fixup: add release title underline for 2.3.0

fix sys.path for local workers  Fixes #421

.appveyor.yml

@@ -1,20 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: "py35-pytestlatest"
-  - TOXENV: "py36-pytestlatest"
-  - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py38-pytestlatest"
-  - TOXENV: "py38-pytestmaster"
-
-install:
-  - C:\Python38\python -m pip install -U pip setuptools virtualenv
-  - C:\Python38\python -m pip install -U tox setuptools_scm
-
-build: false  # Not a C# project, build stuff at the test step instead.
-
-test_script:
-  - C:\Python38\python -m tox
-
-# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
-# might as well save resources
-skip_tags: true

.github/workflows/main.yml

@@ -0,0 +1,75 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py36-pytestlatest"
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py38-pytestmain"
+          - "py38-psutil"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py36-pytestlatest"
+            python: "3.6"
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py38-pytestmain"
+            python: "3.8"
+          - tox_env: "py38-psutil"
+            python: "3.8"
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}
+
+  deploy:
+
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    runs-on: ubuntu-latest
+
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: "3.7"
+    - name: Install wheel
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,20 +1,23 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 19.10b0
+-   repo: https://github.com/psf/black
+    rev: 21.6b0
     hooks:
     -   id: black
         args: [--safe, --quiet, --target-version, py35]
         language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 3.9.2
+    hooks:
     -   id: flake8
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.2
+    rev: v2.19.4
     hooks:
     -   id: pyupgrade
         args: [--py3-plus]

.travis.yml

@@ -1,62 +0,0 @@
-dist: xenial
-language: python
-
-notifications:
-  irc:
-    channels:
-      - 'chat.freenode.net#pytest'
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-  - pytest-commit@python.org
-
-install:
-  - pip install -U pip setuptools
-  - pip install tox setuptools_scm
-script: tox
-
-stages:
-  - baseline
-  - test
-  - name: deploy
-    if: repo = pytest-dev/pytest-xdist AND tag IS present
-
-jobs:
-  include:
-    - stage: baseline
-      python: '3.7'
-      env: TOXENV=linting
-      cache:
-        directories:
-          - $HOME/.cache/pre-commit
-    - python: '3.8'
-      env: TOXENV=py38-pytestlatest
-
-    - stage: test
-      python: "3.5"
-      env: TOXENV=py35-pytestlatest
-    - python: "3.6"
-      env: TOXENV=py36-pytestlatest
-    - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-    - python: "3.9-dev"
-      env: TOXENV=py39-pytestlatest
-    - python: "3.8"
-      env: TOXENV=py38-pytestmaster
-
-    - stage: deploy
-      python: '3.8'
-      env:
-      install: pip install -U setuptools setuptools_scm
-      script: skip
-      deploy:
-        provider: pypi
-        user: ronny
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,73 @@
+pytest-xdist 2.3.0 (2021-06-16)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#654 <https://github.com/pytest-dev/pytest-xdist/issues/654>`_: Python 3.5 is no longer supported.
+
+
+Features
+--------
+
+- `#646 <https://github.com/pytest-dev/pytest-xdist/issues/646>`_: Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+  This is very useful for test suites which are not CPU-bound.
+
+- `#650 <https://github.com/pytest-dev/pytest-xdist/issues/650>`_: Added new ``pytest_handlecrashitem`` hook to allow handling and rescheduling crashed items.
+
+
+Bug Fixes
+---------
+
+- `#421 <https://github.com/pytest-dev/pytest-xdist/issues/421>`_: Copy the parent process sys.path into local workers, to work around execnet's python -c adding the current directory to sys.path.
+
+- `#638 <https://github.com/pytest-dev/pytest-xdist/issues/638>`_: Fix issue caused by changing the branch name of the pytest repository.
+
+
+Trivial Changes
+---------------
+
+- `#592 <https://github.com/pytest-dev/pytest-xdist/issues/592>`_: Replace master with controller where ever possible.
+
+- `#643 <https://github.com/pytest-dev/pytest-xdist/issues/643>`_: Use 'main' to refer to pytest default branch in tox env names.
+
+
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
 pytest-xdist 2.0.0 (2020-08-12)
 ===============================
 

README.rst

@@ -11,13 +11,8 @@
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
-    :alt: Travis CI build status
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
-
-.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
-    :alt: AppVeyor build status
-    :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
@@ -50,6 +45,9 @@ If you would like to know how pytest-xdist works under the covers, checkout
 `OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
 
 
+**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
 Installation
 ------------
 
@@ -57,17 +55,18 @@ Install the plugin with::
 
     pip install pytest-xdist
 
-or use the package in develop/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
 
-    pip install --editable .
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
 
 .. _parallelization:
 
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
 
     pytest -n NUMCPUS
 
@@ -126,7 +125,7 @@ Here is the code:
 
     @pytest.fixture(scope="session")
     def session_data(tmp_path_factory, worker_id):
-        if not worker_id:
+        if worker_id == "master":
             # not executing in with multiple workers, just produce the data and let
             # pytest's fixture caching do its job
             return produce_expensive_data()
@@ -153,16 +152,16 @@ where executing a high-scope fixture exactly once is important.
 Running tests in a Python subprocess
 ------------------------------------
 
-To instantiate a python3.5 subprocess and send tests to it, you may type::
+To instantiate a python3.9 subprocess and send tests to it, you may type::
 
-    pytest -d --tx popen//python=python3.5
+    pytest -d --tx popen//python=python3.9
 
-This will start a subprocess which is run with the ``python3.5``
+This will start a subprocess which is run with the ``python3.9``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python3.5
+    --tx 3*popen//python=python3.9
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -246,7 +245,7 @@ at once. The specifications strings use the `xspec syntax`_.
 
 .. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
 
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
 
 .. _`execnet`: http://codespeak.net/execnet
 
@@ -289,7 +288,18 @@ Since version 2.0, the following functions are also available in the ``xdist`` m
         """
 
     def is_xdist_master(request_or_session) -> bool:
-        """Return `True` if this is the xdist master, `False` otherwise
+        """Return `True` if this is the xdist controller, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        deprecated alias for is_xdist_controller
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+     def is_xdist_controller(request_or_session) -> bool:
+        """Return `True` if this is the xdist controller, `False` otherwise
 
         Note: this method also returns `False` when distribution has not been
         activated at all.
@@ -299,7 +309,7 @@ Since version 2.0, the following functions are also available in the ``xdist`` m
 
     def get_xdist_worker_id(request_or_session) -> str:
         """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-        if running on the 'master' node.
+        if running on the controller node.
 
         If not distributing tests (for example passing `-n0` or not passing `-n` at all) also return 'master'.
 
@@ -364,7 +374,7 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
 
 and then just type::
 

setup.cfg

@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal = 1
-
 [metadata]
 license_file = LICENSE
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=6.0.0", "pytest-forked"]
+install_requires = ["execnet>=1.1", "pytest>=6.0.0", "pytest-forked"]
 
 
 with open("README.rst") as f:
@@ -18,12 +18,12 @@ setup(
     platforms=["linux", "osx", "win32"],
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    extras_require={"testing": ["filelock"]},
+    extras_require={"testing": ["filelock"], "psutil": ["psutil>=3.0"]},
     entry_points={
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },
     zip_safe=False,
-    python_requires=">=3.5",
+    python_requires=">=3.6",
     install_requires=install_requires,
     setup_requires=["setuptools_scm"],
     classifiers=[
@@ -40,9 +40,9 @@ setup(
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
 )

src/xdist/__init__.py

@@ -1,4 +1,15 @@
-from xdist.plugin import is_xdist_worker, is_xdist_master, get_xdist_worker_id
+from xdist.plugin import (
+    is_xdist_worker,
+    is_xdist_master,
+    get_xdist_worker_id,
+    is_xdist_controller,
+)
 from xdist._version import version as __version__
 
-__all__ = ["__version__", "is_xdist_worker", "is_xdist_master", "get_xdist_worker_id"]
+__all__ = [
+    "__version__",
+    "is_xdist_worker",
+    "is_xdist_master",
+    "is_xdist_controller",
+    "get_xdist_worker_id",
+]

src/xdist/dsession.py

@@ -14,7 +14,7 @@ from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
+    """signals an immediate interruption."""
 
 
 class DSession:
@@ -87,7 +87,7 @@ class DSession:
         self._session = None
 
     def pytest_collection(self):
-        # prohibit collection of test items in master process
+        # prohibit collection of test items in controller process
         return True
 
     @pytest.mark.trylast
@@ -174,6 +174,24 @@ class DSession:
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
     def worker_errordown(self, node, error):
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
@@ -222,7 +240,7 @@ class DSession:
             return
         self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
         # tell session which items were effectively collected otherwise
-        # the master node will finish the session with EXIT_NOTESTSCOLLECTED
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
         self.sched.add_node_collection(node, ids)
         if self.terminal:
@@ -325,6 +343,12 @@ class DSession:
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
         rep.node = worker
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 

src/xdist/looponfail.py

@@ -61,7 +61,7 @@ class RemoteControl:
 
     def trace(self, *args):
         if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
+            msg = " ".join(str(x) for x in args)
             print("RemoteControl:", msg)
 
     def initgateway(self):

src/xdist/newhooks.py

@@ -15,43 +15,69 @@ import pytest
 
 
 def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
+    """called before any remote node is set up."""
 
 
 def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
+    """called on new raw gateway creation."""
 
 
 def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
+    """called before rsyncing a directory to remote gateways takes place."""
 
 
 def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
+    """called after rsyncing a directory to remote gateways takes place."""
 
 
 @pytest.mark.firstresult
 def pytest_xdist_getremotemodule():
-    """ called when creating remote node"""
+    """called when creating remote node"""
 
 
 def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
+    """configure node information before it gets instantiated."""
 
 
 def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
+    """Test Node is ready to operate."""
 
 
 def pytest_testnodedown(node, error):
-    """ Test Node is down. """
+    """Test Node is down."""
 
 
 def pytest_xdist_node_collection_finished(node, ids):
-    """called by the master node when a node finishes collecting.
-    """
+    """called by the controller node when a worker node finishes collecting."""
 
 
 @pytest.mark.firstresult
 def pytest_xdist_make_scheduler(config, log):
-    """ return a node scheduler implementation """
+    """return a node scheduler implementation"""
+
+
+@pytest.mark.firstresult
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """
+
+
+@pytest.mark.firstresult
+def pytest_handlecrashitem(crashitem, report, sched):
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -1,21 +1,47 @@
+import os
 import uuid
+import sys
 
-import psutil
 import py
 import pytest
 
-
-def auto_detect_cpus():
-    return psutil.cpu_count(logical=False) or psutil.cpu_count() or 1
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
 
 
-class AutoInt(int):
-    """Mark value as auto-detected."""
+def pytest_xdist_auto_num_workers(config):
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
+    try:
+        from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
+    except ImportError:
+        if os.environ.get("TRAVIS") == "true":
+            # workaround https://bitbucket.org/pypy/pypy/issues/2375
+            return 2
+        try:
+            from os import cpu_count
+        except ImportError:
+            from multiprocessing import cpu_count
+    try:
+        n = cpu_count()
+    except NotImplementedError:
+        return 1
+    return n if n else 1
 
 
 def parse_numprocesses(s):
-    if s == "auto":
-        return AutoInt(auto_detect_cpus())
+    if s in ("auto", "logical"):
+        return s
     elif s is not None:
         return int(s)
 
@@ -29,9 +55,10 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="shortcut for '--dist=load --tx=NUM*popen', "
-        "you can use 'auto' here for auto detection CPUs number on "
-        "host system and it will be 0 when used with --pdb",
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -168,12 +195,13 @@ def pytest_configure(config):
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
-    if isinstance(config.option.numprocesses, AutoInt):
+    if config.option.numprocesses in ("auto", "logical"):
         if usepdb:
             config.option.numprocesses = 0
             config.option.dist = "no"
         else:
-            config.option.numprocesses = int(config.option.numprocesses)
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -204,8 +232,8 @@ def is_xdist_worker(request_or_session) -> bool:
     return hasattr(request_or_session.config, "workerinput")
 
 
-def is_xdist_master(request_or_session) -> bool:
-    """Return `True` if this is the xdist master, `False` otherwise
+def is_xdist_controller(request_or_session) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise
 
     Note: this method also returns `False` when distribution has not been
     activated at all.
@@ -218,9 +246,13 @@ def is_xdist_master(request_or_session) -> bool:
     )
 
 
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
 def get_xdist_worker_id(request_or_session) -> str:
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-    if running on the 'master' node.
+    if running on the controller node.
 
     If not distributing tests (for example passing `-n0` or not passing `-n` at all)
     also return 'master'.
@@ -230,6 +262,7 @@ def get_xdist_worker_id(request_or_session) -> str:
     if hasattr(request_or_session.config, "workerinput"):
         return request_or_session.config.workerinput["workerid"]
     else:
+        # TODO: remove "master", ideally for a None
         return "master"
 
 
@@ -238,6 +271,7 @@ def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
+    # TODO: remove "master", ideally for a None
     return get_xdist_worker_id(request)
 
 

src/xdist/remote.py

@@ -33,8 +33,10 @@ class WorkerInteractor:
         self.channel.send((name, kwargs))
 
     def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
 
     def pytest_sessionstart(self, session):
         self.session = session
@@ -91,9 +93,14 @@ class WorkerInteractor:
         )
 
     def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
         self.sendevent(
             "collectionfinish",
-            topdir=str(session.fspath),
+            topdir=topdir,
             ids=[item.nodeid for item in session.items],
         )
 
@@ -114,7 +121,7 @@ class WorkerInteractor:
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
-        # send only reports that have not passed to master as optimization (#330)
+        # send only reports that have not passed to controller as optimization (#330)
         if not report.passed:
             data = self.config.hook.pytest_report_to_serializable(
                 config=self.config, report=report
@@ -137,7 +144,7 @@ def serialize_warning_message(warning_message):
         message_class_name = type(warning_message.message).__name__
         message_str = str(warning_message.message)
         # check now if we can serialize the warning arguments (#349)
-        # if not, we will just use the exception message on the master node
+        # if not, we will just use the exception message on the controller node
         try:
             dumps(warning_message.message.args)
         except DumpError:
@@ -212,12 +219,14 @@ if __name__ == "__channelexec__":
     channel = channel  # noqa
     workerinput, args, option_dict, change_sys_path = channel.receive()
 
-    if change_sys_path:
+    if change_sys_path is None:
         importpath = os.getcwd()
         sys.path.insert(0, importpath)
         os.environ["PYTHONPATH"] = (
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
+    else:
+        sys.path = change_sys_path
 
     os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]

src/xdist/report.py

@@ -16,5 +16,5 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
         "The difference is:\n"
         "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/each.py

@@ -101,6 +101,14 @@ class EachScheduling:
     def mark_test_complete(self, node, item_index, duration=0):
         self.node2pending[node].remove(item_index)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def remove_node(self, node):
         # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)

src/xdist/scheduler/load.py

@@ -151,6 +151,14 @@ class LoadScheduling:
         self.node2pending[node].remove(item_index)
         self.check_schedule(node, duration=duration)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def check_schedule(self, node, duration=0):
         """Maybe schedule new items on the node
 

src/xdist/scheduler/loadscope.py

@@ -243,6 +243,9 @@ class LoadScopeScheduling:
         self.assigned_work[node][scope][nodeid] = True
         self._reschedule(node)
 
+    def mark_test_pending(self, item):
+        raise NotImplementedError()
+
     def _assign_work_unit(self, node):
         """Assign a work unit to a node."""
         assert self.workqueue

src/xdist/workermanage.py

@@ -9,6 +9,7 @@ import pytest
 import execnet
 
 import xdist.remote
+from xdist.plugin import _sys_path
 
 
 def parse_spec_config(config):
@@ -157,8 +158,7 @@ class NodeManager:
 
 
 class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
-    """
+    """RSyncer that filters out common files"""
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
@@ -262,7 +262,8 @@ class WorkerController:
         remote_module = self.config.hook.pytest_xdist_getremotemodule()
         self.channel = self.gateway.remote_exec(remote_module)
         # change sys.path only for remote workers
-        change_sys_path = not self.gateway.spec.popen
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
 
         if self.putevent:
@@ -294,7 +295,7 @@ class WorkerController:
             self._shutdown_sent = True
 
     def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
+        """send a named parametrized command to the other side."""
         self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
@@ -303,12 +304,12 @@ class WorkerController:
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
-        """ this gets called for each object we receive from
-            the other side and if the channel closes.
+        """this gets called for each object we receive from
+        the other side and if the channel closes.
 
-            Note that channel callbacks run in the receiver
-            thread of execnet gateways - we need to
-            avoid raising exceptions or doing heavy work.
+        Note that channel callbacks run in the receiver
+        thread of execnet gateways - we need to
+        avoid raising exceptions or doing heavy work.
         """
         try:
             if eventcall == self.ENDMARK:
@@ -324,6 +325,8 @@ class WorkerController:
                 self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
                 self.workeroutput = kwargs["workeroutput"]

testing/acceptance_test.py

@@ -244,7 +244,7 @@ class TestDistribution:
     def test_data_exchange(self, testdir):
         testdir.makeconftest(
             """
-            # This hook only called on master.
+            # This hook only called on the controlling process.
             def pytest_configure_node(node):
                 node.workerinput['a'] = 42
                 node.workerinput['b'] = 7
@@ -257,7 +257,7 @@ class TestDistribution:
                     r = a + b
                     config.workeroutput['r'] = r
 
-            # This hook only called on master.
+            # This hook only called on the controlling process.
             def pytest_testnodedown(node, error):
                 node.config.calc_result = node.workeroutput['r']
 
@@ -289,7 +289,7 @@ class TestDistribution:
                 # on the worker
                 if hasattr(session.config, 'workeroutput'):
                     session.config.workeroutput['s2'] = 42
-            # on the master
+            # on the controller
             def pytest_testnodedown(node, error):
                 assert node.workeroutput['s2'] == 42
                 print ("s2call-finished")
@@ -503,7 +503,7 @@ def test_session_hooks(testdir):
             if hasattr(session.config, 'workerinput'):
                 name = "worker"
             else:
-                name = "master"
+                name = "controller"
             with open(name, "w") as f:
                 f.write("xy")
             # let's fail on the worker
@@ -524,12 +524,12 @@ def test_session_hooks(testdir):
     d = result.parseoutcomes()
     assert d["passed"] == 1
     assert testdir.tmpdir.join("worker").check()
-    assert testdir.tmpdir.join("master").check()
+    assert testdir.tmpdir.join("controller").check()
 
 
 def test_session_testscollected(testdir):
     """
-    Make sure master node is updating the session object with the number
+    Make sure controller node is updating the session object with the number
     of tests collected from the workers.
     """
     testdir.makepyfile(
@@ -574,7 +574,7 @@ def test_fixture_teardown_failure(testdir):
 
 
 def test_config_initialization(testdir, monkeypatch, pytestconfig):
-    """Ensure workers and master are initialized consistently. Integration test for #445"""
+    """Ensure workers and controller are initialized consistently. Integration test for #445"""
     testdir.makepyfile(
         **{
             "dir_a/test_foo.py": """
@@ -718,8 +718,7 @@ def test_issue_594_random_parametrize(testdir):
 
 
 def test_tmpdir_disabled(testdir):
-    """Test xdist doesn't break if internal tmpdir plugin is disabled (#22).
-    """
+    """Test xdist doesn't break if internal tmpdir plugin is disabled (#22)."""
     p1 = testdir.makepyfile(
         """
         def test_ok():
@@ -733,8 +732,7 @@ def test_tmpdir_disabled(testdir):
 
 @pytest.mark.parametrize("plugin", ["xdist.looponfail", "xdist.boxed"])
 def test_sub_plugins_disabled(testdir, plugin):
-    """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)
-    """
+    """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)"""
     p1 = testdir.makepyfile(
         """
         def test_ok():
@@ -772,19 +770,23 @@ class TestWarnings:
 
         testdir.makeconftest(
             """
-            def pytest_warning_captured():
-                assert False, "this hook should not be called in this version"
+            def pytest_warning_captured(warning_message):
+                if warning_message == "my custom worker warning":
+                    assert False, (
+                        "this hook should not be called from workers "
+                        "in this version: {}"
+                    ).format(warning_message)
         """
         )
         testdir.makepyfile(
             """
             import warnings
             def test():
-                warnings.warn("custom warning")
+                warnings.warn("my custom worker warning")
         """
         )
         result = testdir.runpytest("-n1")
-        result.stdout.fnmatch_lines(["* 1 passed in *"])
+        result.stdout.fnmatch_lines(["*1 passed*"])
         result.stdout.no_fnmatch_line("*this hook should not be called in this version")
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
@@ -1134,6 +1136,18 @@ def test_internal_error_with_maxfail(testdir):
     assert "INTERNALERROR" not in result.stderr.str()
 
 
+def test_internal_errors_propagate_to_controller(testdir):
+    testdir.makeconftest(
+        """
+        def pytest_collection_modifyitems():
+            raise RuntimeError("Some runtime error")
+        """
+    )
+    testdir.makepyfile("def test(): pass")
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines(["*RuntimeError: Some runtime error*"])
+
+
 class TestLoadScope:
     def test_by_module(self, testdir):
         test_file = """
@@ -1223,14 +1237,22 @@ class TestFileScope:
             "test_b.py::TestB", result.outlines
         )
 
-        assert test_a_workers_and_test_count in (
-            {"gw0": 10},
-            {"gw1": 0},
-        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        assert test_b_workers_and_test_count in (
-            {"gw0": 10},
-            {"gw1": 0},
-        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert (
+            test_a_workers_and_test_count
+            in (
+                {"gw0": 10},
+                {"gw1": 0},
+            )
+            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
+        assert (
+            test_b_workers_and_test_count
+            in (
+                {"gw0": 10},
+                {"gw1": 0},
+            )
+            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
 
     def test_by_class(self, testdir):
         testdir.makepyfile(
@@ -1255,14 +1277,22 @@ class TestFileScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert test_a_workers_and_test_count in (
-            {"gw0": 10},
-            {"gw1": 0},
-        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        assert test_b_workers_and_test_count in (
-            {"gw0": 10},
-            {"gw1": 0},
-        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert (
+            test_a_workers_and_test_count
+            in (
+                {"gw0": 10},
+                {"gw1": 0},
+            )
+            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
+        assert (
+            test_b_workers_and_test_count
+            in (
+                {"gw0": 10},
+                {"gw1": 0},
+            )
+            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
 
     def test_module_single_start(self, testdir):
         """Fix test suite never finishing in case all workers start with a single test (#277)."""
@@ -1392,12 +1422,18 @@ class TestAPI:
         del fake_request.config.workerinput
         assert not xdist.is_xdist_worker(fake_request)
 
-    def test_is_xdist_master(self, fake_request):
+    def test_is_xdist_controller(self, fake_request):
+
         assert not xdist.is_xdist_master(fake_request)
+        assert not xdist.is_xdist_controller(fake_request)
+
         del fake_request.config.workerinput
         assert xdist.is_xdist_master(fake_request)
+        assert xdist.is_xdist_controller(fake_request)
+
         fake_request.config.option.dist = "no"
         assert not xdist.is_xdist_master(fake_request)
+        assert not xdist.is_xdist_controller(fake_request)
 
     def test_get_xdist_worker_id(self, fake_request):
         assert xdist.get_xdist_worker_id(fake_request) == "gw5"

testing/test_newhooks.py

@@ -46,8 +46,7 @@ class TestHooks:
         )
 
     def test_node_collection_finished(self, testdir):
-        """Test pytest_xdist_node_collection_finished hook (#8).
-        """
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
         testdir.makeconftest(
             """
             def pytest_xdist_node_collection_finished(node, ids):
@@ -61,3 +60,37 @@ class TestHooks:
             ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
         )
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, testdir):
+        testdir.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, testdir):
+        """Test pytest_handlecrashitem hook."""
+        testdir.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = testdir.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])

testing/test_plugin.py

@@ -1,7 +1,11 @@
+from contextlib import suppress
+
 import py
 import execnet
 from xdist.workermanage import NodeManager
 
+import pytest
+
 
 def test_dist_incompatibility_messages(testdir):
     result = testdir.runpytest("--pdb", "--looponfail")
@@ -35,15 +39,28 @@ def test_dist_options(testdir):
 
 
 def test_auto_detect_cpus(testdir, monkeypatch):
-    import psutil
+    import os
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 99)
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+
+    if hasattr(os, "sched_getaffinity"):
+        monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
+    elif hasattr(os, "cpu_count"):
+        monkeypatch.setattr(os, "cpu_count", lambda: 99)
+    else:
+        import multiprocessing
+
+        monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
 
     config = testdir.parseconfigure("-n2")
     assert config.getoption("numprocesses") == 2
 
     config = testdir.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 99
 
     config = testdir.parseconfigure("-nauto", "--pdb")
@@ -52,9 +69,51 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption("numprocesses") == 0
     assert config.getoption("dist") == "no"
 
-    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+    config = testdir.parseconfigure("-nlogical", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setenv("TRAVIS", "true")
     config = testdir.parseconfigure("-nauto")
-    assert config.getoption("numprocesses") == 1
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+
+def test_auto_detect_cpus_psutil(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_hook_auto_num_workers(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    testdir.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
 
 
 def test_boxed_with_collect_only(testdir):

testing/test_remote.py

@@ -37,7 +37,9 @@ class WorkerSetup:
         self.testdir = testdir
         self.events = Queue()
 
-    def setup(self,):
+    def setup(
+        self,
+    ):
         self.testdir.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
         self.gateway = execnet.makegateway()
@@ -290,3 +292,17 @@ def test_remote_usage_prog(testdir, request):
     result = testdir.runpytest_subprocess("-n1")
     assert result.ret == 1
     result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(testdir):
+    """Work around sys.path differences due to execnet using `python -c`."""
+    testdir.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
+    )
+    result = testdir.runpytest("-n1")
+    assert result.ret == 0

tox.ini

@@ -1,24 +1,25 @@
 [tox]
 envlist=
   linting
-  py{35,36,37,38,39}-pytestlatest
-  py38-pytestmaster
+  py{36,37,38,39}-pytestlatest
+  py38-pytestmain
+  py38-psutil
 
 [testenv]
-passenv = USER USERNAME
 extras = testing
 deps =
   pytestlatest: pytest
-  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
 commands=
   pytest {posargs}
 
-[testenv:linting]
-skipsdist = True
-usedevelop = True
-deps =
-  pre-commit
-commands = pre-commit run --all-files --show-diff-on-failure
+[testenv:py38-psutil]
+extras =
+  testing
+  psutil
+deps = pytest
+commands =
+  pytest {posargs:-k psutil}
 
 [testenv:release]
 changedir=
@@ -28,12 +29,14 @@ skipsdist = True
 usedevelop = True
 passenv = *
 deps =
-    towncrier
+  towncrier
 commands =
-    towncrier --version {posargs} --yes
+  towncrier --version {posargs} --yes
 
 [pytest]
-addopts = -ra
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
 testpaths = testing
 
 [flake8]