Manually fix changelog title

[pre-commit.ci] pre-commit autoupdate

.appveyor.yml

@@ -1,21 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: "py27-pytestlatest"
-  - TOXENV: "py35-pytestlatest"
-  - TOXENV: "py36-pytestlatest"
-  - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py38-pytestlatest"
-  - TOXENV: "py38-pytestmaster"
-
-install:
-  - C:\Python38\python -m pip install -U pip setuptools virtualenv
-  - C:\Python38\python -m pip install -U tox setuptools_scm
-
-build: false  # Not a C# project, build stuff at the test step instead.
-
-test_script:
-  - C:\Python38\python -m tox
-
-# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
-# might as well save resources
-skip_tags: true

.github/workflows/main.yml

@@ -0,0 +1,81 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py36-pytestlatest"
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py38-pytestmain"
+          - "py38-psutil"
+          - "py38-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py36-pytestlatest"
+            python: "3.6"
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10-dev"
+          - tox_env: "py38-pytestmain"
+            python: "3.8"
+          - tox_env: "py38-psutil"
+            python: "3.8"
+          - tox_env: "py38-setproctitle"
+            python: "3.8"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}
+
+  deploy:
+
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    runs-on: ubuntu-latest
+
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+    - name: Install wheel
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,18 +1,26 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 19.10b0
+-   repo: https://github.com/psf/black
+    rev: 21.9b0
     hooks:
     -   id: black
-        args: [--safe, --quiet]
+        args: [--safe, --quiet, --target-version, py35]
         language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 3.9.2
+    hooks:
     -   id: flake8
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.26.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py3-plus]
 -   repo: local
     hooks:
     -   id: rst

.travis.yml

@@ -1,62 +0,0 @@
-dist: xenial
-language: python
-
-notifications:
-  irc:
-    channels:
-      - 'chat.freenode.net#pytest'
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-  - pytest-commit@python.org
-
-install:
-  - pip install -U pip setuptools
-  - pip install tox setuptools_scm
-script: tox
-
-stages:
-  - baseline
-  - test
-  - name: deploy
-    if: repo = pytest-dev/pytest-xdist AND tag IS present
-
-jobs:
-  include:
-    - stage: baseline
-      python: '3.7'
-      env: TOXENV=linting
-      cache:
-        directories:
-          - $HOME/.cache/pre-commit
-    - python: '3.8'
-      env: TOXENV=py38-pytestlatest
-    - python: '2.7'
-      env: TOXENV=py27-pytestlatest
-
-    - stage: test
-      python: "3.5"
-      env: TOXENV=py35-pytestlatest
-    - python: "3.6"
-      env: TOXENV=py36-pytestlatest
-    - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-    - python: "3.8"
-      env: TOXENV=py38-pytestmaster
-
-    - stage: deploy
-      python: '3.8'
-      env:
-      install: pip install -U setuptools setuptools_scm
-      script: skip
-      deploy:
-        provider: pypi
-        user: ronny
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,117 @@
+pytest-xdist 2.4.0 (2021-09-20)
+===============================
+
+Features
+--------
+
+- `#696 <https://github.com/pytest-dev/pytest-xdist/issues/696>`_: On Linux, the process title now changes to indicate the current worker state (running/idle).
+
+  Depends on the `setproctitle <https://pypi.org/project/setproctitle/>`__ package, which can be installed with ``pip install pytest-xdist[setproctitle]``.
+
+- `#704 <https://github.com/pytest-dev/pytest-xdist/issues/704>`_: Add support for Python 3.10.
+
+
+pytest-xdist 2.3.0 (2021-06-16)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#654 <https://github.com/pytest-dev/pytest-xdist/issues/654>`_: Python 3.5 is no longer supported.
+
+
+Features
+--------
+
+- `#646 <https://github.com/pytest-dev/pytest-xdist/issues/646>`_: Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+  This is very useful for test suites which are not CPU-bound.
+
+- `#650 <https://github.com/pytest-dev/pytest-xdist/issues/650>`_: Added new ``pytest_handlecrashitem`` hook to allow handling and rescheduling crashed items.
+
+
+Bug Fixes
+---------
+
+- `#421 <https://github.com/pytest-dev/pytest-xdist/issues/421>`_: Copy the parent process sys.path into local workers, to work around execnet's python -c adding the current directory to sys.path.
+
+- `#638 <https://github.com/pytest-dev/pytest-xdist/issues/638>`_: Fix issue caused by changing the branch name of the pytest repository.
+
+
+Trivial Changes
+---------------
+
+- `#592 <https://github.com/pytest-dev/pytest-xdist/issues/592>`_: Replace master with controller where ever possible.
+
+- `#643 <https://github.com/pytest-dev/pytest-xdist/issues/643>`_: Use 'main' to refer to pytest default branch in tox env names.
+
+
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
+pytest-xdist 2.0.0 (2020-08-12)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: Drop backward-compatibility "slave" aliases related to worker nodes.  We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.
+
+- `#569 <https://github.com/pytest-dev/pytest-xdist/issues/569>`_: ``pytest-xdist`` no longer supports Python 2.7.
+
+
+Features
+--------
+
+- `#504 <https://github.com/pytest-dev/pytest-xdist/issues/504>`_: New functions ``xdist.is_xdist_worker``, ``xdist.is_xdist_master``, ``xdist.get_xdist_worker_id``, to easily identify the current node.
+
+
+Bug Fixes
+---------
+
+- `#471 <https://github.com/pytest-dev/pytest-xdist/issues/471>`_: Fix issue with Rsync reporting in quiet mode.
+
+- `#553 <https://github.com/pytest-dev/pytest-xdist/issues/553>`_: When using ``-n auto``, count the number of physical CPU cores instead of logical ones.
+
+
+Trivial Changes
+---------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: ``pytest-xdist`` now requires ``pytest>=6.0``.
+
+
 pytest-xdist 1.34.0 (2020-07-27)
 ================================
 

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no pytest plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

README.rst

@@ -11,13 +11,8 @@
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
-    :alt: Travis CI build status
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
-
-.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
-    :alt: AppVeyor build status
-    :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
@@ -50,6 +45,9 @@ If you would like to know how pytest-xdist works under the covers, checkout
 `OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
 
 
+**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
 Installation
 ------------
 
@@ -57,17 +55,18 @@ Install the plugin with::
 
     pip install pytest-xdist
 
-or use the package in develop/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
 
-    pip install --editable .
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
 
 .. _parallelization:
 
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
 
     pytest -n NUMCPUS
 
@@ -126,7 +125,7 @@ Here is the code:
 
     @pytest.fixture(scope="session")
     def session_data(tmp_path_factory, worker_id):
-        if not worker_id:
+        if worker_id == "master":
             # not executing in with multiple workers, just produce the data and let
             # pytest's fixture caching do its job
             return produce_expensive_data()
@@ -153,16 +152,16 @@ where executing a high-scope fixture exactly once is important.
 Running tests in a Python subprocess
 ------------------------------------
 
-To instantiate a python3.5 subprocess and send tests to it, you may type::
+To instantiate a python3.9 subprocess and send tests to it, you may type::
 
-    pytest -d --tx popen//python=python3.5
+    pytest -d --tx popen//python=python3.9
 
-This will start a subprocess which is run with the ``python3.5``
+This will start a subprocess which is run with the ``python3.9``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python3.5
+    --tx 3*popen//python=python3.9
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -246,7 +245,7 @@ at once. The specifications strings use the `xspec syntax`_.
 
 .. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
 
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
 
 .. _`execnet`: http://codespeak.net/execnet
 
@@ -268,7 +267,7 @@ a test or fixture, you may use the ``worker_id`` fixture to do so:
 When ``xdist`` is disabled (running with ``-n0`` for example), then
 ``worker_id`` will return ``"master"``.
 
-Additionally, worker processes have the following environment variables
+Worker processes also have the following environment variables
 defined:
 
 * ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
@@ -278,6 +277,65 @@ defined:
 The information about the worker_id in a test is stored in the ``TestReport`` as
 well, under the ``worker_id`` attribute.
 
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+.. code-block:: python
+
+    def is_xdist_worker(request_or_session) -> bool:
+        """Return `True` if this is an xdist worker, `False` otherwise
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def is_xdist_master(request_or_session) -> bool:
+        """Return `True` if this is the xdist controller, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        deprecated alias for is_xdist_controller
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+     def is_xdist_controller(request_or_session) -> bool:
+        """Return `True` if this is the xdist controller, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def get_xdist_worker_id(request_or_session) -> str:
+        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+        if running on the controller node.
+
+        If not distributing tests (for example passing `-n0` or not passing `-n` at all) also return 'master'.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+
+Identifying workers from the system environment
+-----------------------------------------------
+
+*New in version UNRELEASED TBD FIXME*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
 
 Uniquely identifying the current test run
 -----------------------------------------
@@ -336,7 +394,7 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
 
 and then just type::
 

setup.cfg

@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal = 1
-
 [metadata]
 license_file = LICENSE
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-install_requires = ["execnet>=1.1", "pytest>=4.4.0", "pytest-forked", "six"]
+install_requires = ["execnet>=1.1", "pytest>=6.0.0", "pytest-forked"]
 
 
 with open("README.rst") as f:
@@ -9,8 +9,7 @@ with open("README.rst") as f:
 setup(
     name="pytest-xdist",
     use_scm_version={"write_to": "src/xdist/_version.py"},
-    description="pytest xdist plugin for distributed testing"
-    " and loop-on-failing modes",
+    description="pytest xdist plugin for distributed testing and loop-on-failing modes",
     long_description=long_description,
     license="MIT",
     author="holger krekel and contributors",
@@ -19,12 +18,16 @@ setup(
     platforms=["linux", "osx", "win32"],
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    extras_require={"testing": ["filelock"]},
+    extras_require={
+        "testing": ["filelock"],
+        "psutil": ["psutil>=3.0"],
+        "setproctitle": ["setproctitle"],
+    },
     entry_points={
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
+    python_requires=">=3.6",
     install_requires=install_requires,
     setup_requires=["setuptools_scm"],
     classifiers=[
@@ -39,12 +42,12 @@ setup(
         "Topic :: Software Development :: Quality Assurance",
         "Topic :: Utilities",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
     ],
 )

src/xdist/__init__.py

@@ -1,3 +1,15 @@
+from xdist.plugin import (
+    is_xdist_worker,
+    is_xdist_master,
+    get_xdist_worker_id,
+    is_xdist_controller,
+)
 from xdist._version import version as __version__
 
-__all__ = ["__version__"]
+__all__ = [
+    "__version__",
+    "is_xdist_worker",
+    "is_xdist_master",
+    "is_xdist_controller",
+    "get_xdist_worker_id",
+]

src/xdist/dsession.py

@@ -10,14 +10,14 @@ from xdist.scheduler import (
 )
 
 
-from six.moves.queue import Empty, Queue
+from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
+    """signals an immediate interruption."""
 
 
-class DSession(object):
+class DSession:
     """A pytest plugin which runs a distributed test session
 
     At the beginning of the test session this creates a NodeManager
@@ -87,7 +87,7 @@ class DSession(object):
         self._session = None
 
     def pytest_collection(self):
-        # prohibit collection of test items in master process
+        # prohibit collection of test items in controller process
         return True
 
     @pytest.mark.trylast
@@ -150,9 +150,6 @@ class DSession(object):
         node.workerinfo["id"] = node.gateway.id
         node.workerinfo["spec"] = node.gateway.spec
 
-        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
-        node.slaveinfo = node.workerinfo
-
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
@@ -177,6 +174,24 @@ class DSession(object):
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
     def worker_errordown(self, node, error):
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
@@ -225,7 +240,7 @@ class DSession(object):
             return
         self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
         # tell session which items were effectively collected otherwise
-        # the master node will finish the session with EXIT_NOTESTSCOLLECTED
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
         self.sched.add_node_collection(node, ids)
         if self.terminal:
@@ -271,11 +286,6 @@ class DSession(object):
         assert not rep.passed
         self._failed_worker_collectreport(node, rep)
 
-    def worker_logwarning(self, message, code, nodeid, fslocation):
-        """Emitted when a node calls the pytest_logwarning hook."""
-        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
-        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
-
     def worker_warning_captured(self, warning_message, when, item):
         """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
         kwargs = dict(warning_message=warning_message, when=when, item=item)
@@ -333,10 +343,16 @@ class DSession(object):
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
         rep.node = worker
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
-class TerminalDistReporter(object):
+class TerminalDistReporter:
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")

src/xdist/looponfail.py

@@ -6,7 +6,6 @@
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-from __future__ import print_function
 import py
 import pytest
 import sys
@@ -55,14 +54,14 @@ def looponfail_main(config):
         print()
 
 
-class RemoteControl(object):
+class RemoteControl:
     def __init__(self, config):
         self.config = config
         self.failures = []
 
     def trace(self, *args):
         if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
+            msg = " ".join(str(x) for x in args)
             print("RemoteControl:", msg)
 
     def initgateway(self):
@@ -165,7 +164,7 @@ def init_worker_session(channel, args, option_dict):
     WorkerFailSession(config, channel).main()
 
 
-class WorkerFailSession(object):
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -220,7 +219,7 @@ class WorkerFailSession(object):
         self.channel.send((trails, failreports, self.collection_failed))
 
 
-class StatRecorder(object):
+class StatRecorder:
     def __init__(self, rootdirlist):
         self.rootdirlist = rootdirlist
         self.statcache = {}

src/xdist/newhooks.py

@@ -15,43 +15,69 @@ import pytest
 
 
 def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
+    """called before any remote node is set up."""
 
 
 def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
+    """called on new raw gateway creation."""
 
 
 def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
+    """called before rsyncing a directory to remote gateways takes place."""
 
 
 def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
+    """called after rsyncing a directory to remote gateways takes place."""
 
 
 @pytest.mark.firstresult
 def pytest_xdist_getremotemodule():
-    """ called when creating remote node"""
+    """called when creating remote node"""
 
 
 def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
+    """configure node information before it gets instantiated."""
 
 
 def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
+    """Test Node is ready to operate."""
 
 
 def pytest_testnodedown(node, error):
-    """ Test Node is down. """
+    """Test Node is down."""
 
 
 def pytest_xdist_node_collection_finished(node, ids):
-    """called by the master node when a node finishes collecting.
-    """
+    """called by the controller node when a worker node finishes collecting."""
 
 
 @pytest.mark.firstresult
 def pytest_xdist_make_scheduler(config, log):
-    """ return a node scheduler implementation """
+    """return a node scheduler implementation"""
+
+
+@pytest.mark.firstresult
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """
+
+
+@pytest.mark.firstresult
+def pytest_handlecrashitem(crashitem, report, sched):
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -1,13 +1,29 @@
 import os
 import uuid
+import sys
 
 import py
 import pytest
 
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
 
-def auto_detect_cpus():
+
+def pytest_xdist_auto_num_workers(config):
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
     try:
         from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
     except ImportError:
         if os.environ.get("TRAVIS") == "true":
             # workaround https://bitbucket.org/pypy/pypy/issues/2375
@@ -16,11 +32,6 @@ def auto_detect_cpus():
             from os import cpu_count
         except ImportError:
             from multiprocessing import cpu_count
-    else:
-
-        def cpu_count():
-            return len(sched_getaffinity(0))
-
     try:
         n = cpu_count()
     except NotImplementedError:
@@ -28,13 +39,9 @@ def auto_detect_cpus():
     return n if n else 1
 
 
-class AutoInt(int):
-    """Mark value as auto-detected."""
-
-
 def parse_numprocesses(s):
-    if s == "auto":
-        return AutoInt(auto_detect_cpus())
+    if s in ("auto", "logical"):
+        return s
     elif s is not None:
         return int(s)
 
@@ -48,9 +55,10 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="shortcut for '--dist=load --tx=NUM*popen', "
-        "you can use 'auto' here for auto detection CPUs number on "
-        "host system and it will be 0 when used with --pdb",
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -62,14 +70,11 @@ def pytest_addoption(parser):
     )
     group.addoption(
         "--max-worker-restart",
-        "--max-slave-restart",
         action="store",
         default=None,
         dest="maxworkerrestart",
         help="maximum number of workers that can be restarted "
-        "when crashed (set to zero to disable this feature)\n"
-        "'--max-slave-restart' option is deprecated and will be removed in "
-        "a future release",
+        "when crashed (set to zero to disable this feature)",
     )
     group.addoption(
         "--dist",
@@ -190,12 +195,13 @@ def pytest_configure(config):
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
-    if isinstance(config.option.numprocesses, AutoInt):
+    if config.option.numprocesses in ("auto", "logical"):
         if usepdb:
             config.option.numprocesses = 0
             config.option.dist = "no"
         else:
-            config.option.numprocesses = int(config.option.numprocesses)
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -214,19 +220,59 @@ def pytest_cmdline_main(config):
 
 
 # -------------------------------------------------------------------------
-# fixtures
+# fixtures and API to easily know the role of current node
 # -------------------------------------------------------------------------
 
 
+def is_xdist_worker(request_or_session) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_controller(request_or_session) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
+def get_xdist_worker_id(request_or_session) -> str:
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the controller node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        return request_or_session.config.workerinput["workerid"]
+    else:
+        # TODO: remove "master", ideally for a None
+        return "master"
+
+
 @pytest.fixture(scope="session")
 def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
-    if hasattr(request.config, "workerinput"):
-        return request.config.workerinput["workerid"]
-    else:
-        return "master"
+    # TODO: remove "master", ideally for a None
+    return get_xdist_worker_id(request)
 
 
 @pytest.fixture(scope="session")

src/xdist/remote.py

@@ -11,14 +11,28 @@ import os
 import time
 
 import py
-import _pytest.hookspec
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
 from _pytest.config import _prepareconfig, Config
 
+try:
+    from setproctitle import setproctitle
+except ImportError:
 
-class WorkerInteractor(object):
+    def setproctitle(title):
+        pass
+
+
+def worker_title(title):
+    try:
+        setproctitle(title)
+    except Exception:
+        # changing the process name is very optional, no errors please
+        pass
+
+
+class WorkerInteractor:
     def __init__(self, config, channel):
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
@@ -34,8 +48,10 @@ class WorkerInteractor(object):
         self.channel.send((name, kwargs))
 
     def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
 
     def pytest_sessionstart(self, session):
         self.session = session
@@ -84,26 +100,33 @@ class WorkerInteractor(object):
         else:
             nextitem = None
 
+        worker_title("[pytest-xdist running] %s" % item.nodeid)
+
         start = time.time()
         self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         duration = time.time() - start
+
+        worker_title("[pytest-xdist idle]")
+
         self.sendevent(
             "runtest_protocol_complete", item_index=self.item_index, duration=duration
         )
 
     def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
         self.sendevent(
             "collectionfinish",
-            topdir=str(session.fspath),
+            topdir=topdir,
             ids=[item.nodeid for item in session.items],
         )
 
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
-    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
-    if hasattr(_pytest.hookspec, "pytest_runtest_logfinish"):
-
     def pytest_runtest_logfinish(self, nodeid, location):
         self.sendevent("logfinish", nodeid=nodeid, location=location)
 
@@ -118,30 +141,13 @@ class WorkerInteractor(object):
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
-        # send only reports that have not passed to master as optimization (#330)
+        # send only reports that have not passed to controller as optimization (#330)
         if not report.passed:
             data = self.config.hook.pytest_report_to_serializable(
                 config=self.config, report=report
             )
             self.sendevent("collectreport", data=data)
 
-    # the pytest_logwarning hook was deprecated since pytest 4.0
-    if hasattr(
-        _pytest.hookspec, "pytest_logwarning"
-    ) and not _pytest.hookspec.pytest_logwarning.pytest_spec.get("warn_on_impl"):
-
-        def pytest_logwarning(self, message, code, nodeid, fslocation):
-            self.sendevent(
-                "logwarning",
-                message=message,
-                code=code,
-                nodeid=nodeid,
-                fslocation=str(fslocation),
-            )
-
-    # the pytest_warning_recorded hook was introduced in pytest 6.0
-    if hasattr(_pytest.hookspec, "pytest_warning_recorded"):
-
     def pytest_warning_recorded(self, warning_message, when, nodeid, location):
         self.sendevent(
             "warning_recorded",
@@ -151,18 +157,6 @@ class WorkerInteractor(object):
             location=location,
         )
 
-    # the pytest_warning_captured hook was introduced in pytest 3.8
-    elif hasattr(_pytest.hookspec, "pytest_warning_captured"):
-
-        def pytest_warning_captured(self, warning_message, when, item):
-            self.sendevent(
-                "warning_captured",
-                warning_message_data=serialize_warning_message(warning_message),
-                when=when,
-                # item cannot be serialized and will always be None when used with xdist
-                item=None,
-            )
-
 
 def serialize_warning_message(warning_message):
     if isinstance(warning_message.message, Warning):
@@ -170,7 +164,7 @@ def serialize_warning_message(warning_message):
         message_class_name = type(warning_message.message).__name__
         message_str = str(warning_message.message)
         # check now if we can serialize the warning arguments (#349)
-        # if not, we will just use the exception message on the master node
+        # if not, we will just use the exception message on the controller node
         try:
             dumps(warning_message.message.args)
         except DumpError:
@@ -245,12 +239,14 @@ if __name__ == "__channelexec__":
     channel = channel  # noqa
     workerinput, args, option_dict, change_sys_path = channel.receive()
 
-    if change_sys_path:
+    if change_sys_path is None:
         importpath = os.getcwd()
         sys.path.insert(0, importpath)
         os.environ["PYTHONPATH"] = (
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
+    else:
+        sys.path = change_sys_path
 
     os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
@@ -266,8 +262,5 @@ if __name__ == "__channelexec__":
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
     config.workerinput = workerinput
     config.workeroutput = {}
-    # TODO: deprecated name, backward compatibility only. Remove it in future
-    config.slaveinput = config.workerinput
-    config.slaveoutput = config.workeroutput
     interactor = WorkerInteractor(config, channel)
     config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 from difflib import unified_diff
 
 
@@ -13,9 +12,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
 
     diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
     error_message = (
-        u"Different tests were collected between {from_id} and {to_id}. "
-        u"The difference is:\n"
-        u"{diff}"
+        "Different tests were collected between {from_id} and {to_id}. "
+        "The difference is:\n"
+        "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/each.py

@@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class EachScheduling(object):
+class EachScheduling:
     """Implement scheduling of test items on all nodes
 
     If a node gets added after the test run is started then it is
@@ -101,6 +101,14 @@ class EachScheduling(object):
     def mark_test_complete(self, node, item_index, duration=0):
         self.node2pending[node].remove(item_index)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def remove_node(self, node):
         # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)

src/xdist/scheduler/load.py

@@ -7,7 +7,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class LoadScheduling(object):
+class LoadScheduling:
     """Implement load scheduling across nodes.
 
     This distributes the tests collected across all nodes so each test
@@ -151,6 +151,14 @@ class LoadScheduling(object):
         self.node2pending[node].remove(item_index)
         self.check_schedule(node, duration=duration)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def check_schedule(self, node, duration=0):
         """Maybe schedule new items on the node
 

src/xdist/scheduler/loadfile.py

@@ -21,7 +21,7 @@ class LoadFileScheduling(LoadScopeScheduling):
     """
 
     def __init__(self, config, log=None):
-        super(LoadFileScheduling, self).__init__(config, log)
+        super().__init__(config, log)
         if log is None:
             self.log = Producer("loadfilesched")
         else:

src/xdist/scheduler/loadscope.py

@@ -6,7 +6,7 @@ from xdist.report import report_collection_diff
 from xdist.workermanage import parse_spec_config
 
 
-class LoadScopeScheduling(object):
+class LoadScopeScheduling:
     """Implement load scheduling across nodes, but grouping test by scope.
 
     This distributes the tests collected across all nodes so each test is run
@@ -243,6 +243,9 @@ class LoadScopeScheduling(object):
         self.assigned_work[node][scope][nodeid] = True
         self._reschedule(node)
 
+    def mark_test_pending(self, item):
+        raise NotImplementedError()
+
     def _assign_work_unit(self, node):
         """Assign a work unit to a node."""
         assert self.workqueue

src/xdist/workermanage.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import fnmatch
 import os
 import re
@@ -10,6 +9,7 @@ import pytest
 import execnet
 
 import xdist.remote
+from xdist.plugin import _sys_path
 
 
 def parse_spec_config(config):
@@ -29,7 +29,7 @@ def parse_spec_config(config):
     return xspeclist
 
 
-class NodeManager(object):
+class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
@@ -123,7 +123,7 @@ class NodeManager(object):
 
         return {
             "ignores": ignores,
-            "verbose": getattr(self.config.option, "verbose", False),
+            "verbose": getattr(self.config.option, "verbose", 0),
         }
 
     def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
@@ -158,8 +158,7 @@ class NodeManager(object):
 
 
 class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
-    """
+    """RSyncer that filters out common files"""
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
@@ -167,7 +166,7 @@ class HostRSync(execnet.RSync):
         self._ignores = [
             re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
         ]
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
+        super().__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
         path = py.path.local(path)
@@ -179,12 +178,10 @@ class HostRSync(execnet.RSync):
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(
-            gateway, remotepath, finishedcallback=finished, delete=True
-        )
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
 
     def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
+        if self._verbose > 0:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
             print("{}:{} <= {}".format(gateway.spec, remotepath, path))
@@ -211,7 +208,7 @@ def make_reltoroot(roots, args):
     return result
 
 
-class WorkerController(object):
+class WorkerController:
     ENDMARK = -1
 
     class RemoteHook:
@@ -228,13 +225,9 @@ class WorkerController(object):
         self.workerinput = {
             "workerid": gateway.id,
             "workercount": len(nodemanager.specs),
-            "slaveid": gateway.id,
-            "slavecount": len(nodemanager.specs),
             "testrunuid": nodemanager.testrunuid,
             "mainargv": sys.argv,
         }
-        # TODO: deprecated name, backward compatibility only. Remove it in future
-        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
         self.log = py.log.Producer("workerctl-%s" % gateway.id)
@@ -269,7 +262,8 @@ class WorkerController(object):
         remote_module = self.config.hook.pytest_xdist_getremotemodule()
         self.channel = self.gateway.remote_exec(remote_module)
         # change sys.path only for remote workers
-        change_sys_path = not self.gateway.spec.popen
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
 
         if self.putevent:
@@ -296,12 +290,12 @@ class WorkerController(object):
         if not self._down:
             try:
                 self.sendcommand("shutdown")
-            except (IOError, OSError):
+            except OSError:
                 pass
             self._shutdown_sent = True
 
     def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
+        """send a named parametrized command to the other side."""
         self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
@@ -310,7 +304,7 @@ class WorkerController(object):
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
-        """ this gets called for each object we receive from
+        """this gets called for each object we receive from
         the other side and if the channel closes.
 
         Note that channel callbacks run in the receiver
@@ -331,9 +325,11 @@ class WorkerController(object):
                 self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
-                self.workeroutput = self.slaveoutput = kwargs["workeroutput"]
+                self.workeroutput = kwargs["workeroutput"]
                 self.notify_inproc("workerfinished", node=self)
             elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
@@ -386,11 +382,7 @@ class WorkerController(object):
         except:  # noqa
             from _pytest._code import ExceptionInfo
 
-            # ExceptionInfo API changed in pytest 4.1
-            if hasattr(ExceptionInfo, "from_current"):
             excinfo = ExceptionInfo.from_current()
-            else:
-                excinfo = ExceptionInfo()
             print("!" * 20, excinfo)
             self.config.notify_exception(excinfo)
             self.shutdown()

testing/acceptance_test.py

@@ -1,10 +1,9 @@
 import os
 import re
-import sys
-import textwrap
 
 import py
 import pytest
+import xdist
 
 
 class TestDistribution:
@@ -115,9 +114,6 @@ class TestDistribution:
         assert result.ret == 2
         result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"])
 
-    @pytest.mark.xfail(
-        reason="#527: Python 3.8 failure in pytest where testdir.tmpdir returns an unexpected value"
-    )
     def test_basetemp_in_subprocesses(self, testdir):
         p1 = testdir.makepyfile(
             """
@@ -245,26 +241,10 @@ class TestDistribution:
         result.stderr.fnmatch_lines(["--foobar=123 active! *"])
         assert dest.join(subdir.basename).check(dir=1)
 
-    def test_backward_compatibility_worker_terminology(self, testdir):
-        """Ensure that we still support "config.slaveinput" for backward compatibility (#234).
-
-        Keep in mind that removing this compatibility will break a ton of plugins and user code.
-        """
-        testdir.makepyfile(
-            """
-            def test(pytestconfig):
-                assert hasattr(pytestconfig, 'slaveinput')
-                assert hasattr(pytestconfig, 'workerinput')
-        """
-        )
-        result = testdir.runpytest("-n1")
-        result.stdout.fnmatch_lines("*1 passed*")
-        assert result.ret == 0
-
     def test_data_exchange(self, testdir):
         testdir.makeconftest(
             """
-            # This hook only called on master.
+            # This hook only called on the controlling process.
             def pytest_configure_node(node):
                 node.workerinput['a'] = 42
                 node.workerinput['b'] = 7
@@ -277,7 +257,7 @@ class TestDistribution:
                     r = a + b
                     config.workeroutput['r'] = r
 
-            # This hook only called on master.
+            # This hook only called on the controlling process.
             def pytest_testnodedown(node, error):
                 node.config.calc_result = node.workeroutput['r']
 
@@ -309,7 +289,7 @@ class TestDistribution:
                 # on the worker
                 if hasattr(session.config, 'workeroutput'):
                     session.config.workeroutput['s2'] = 42
-            # on the master
+            # on the controller
             def pytest_testnodedown(node, error):
                 assert node.workeroutput['s2'] == 42
                 print ("s2call-finished")
@@ -523,7 +503,7 @@ def test_session_hooks(testdir):
             if hasattr(session.config, 'workerinput'):
                 name = "worker"
             else:
-                name = "master"
+                name = "controller"
             with open(name, "w") as f:
                 f.write("xy")
             # let's fail on the worker
@@ -544,12 +524,12 @@ def test_session_hooks(testdir):
     d = result.parseoutcomes()
     assert d["passed"] == 1
     assert testdir.tmpdir.join("worker").check()
-    assert testdir.tmpdir.join("master").check()
+    assert testdir.tmpdir.join("controller").check()
 
 
 def test_session_testscollected(testdir):
     """
-    Make sure master node is updating the session object with the number
+    Make sure controller node is updating the session object with the number
     of tests collected from the workers.
     """
     testdir.makepyfile(
@@ -588,21 +568,13 @@ def test_fixture_teardown_failure(testdir):
             pass
     """
     )
-    result = testdir.runpytest_subprocess("--debug", p)  # , "-n1")
+    result = testdir.runpytest_subprocess(p, "-n1")
     result.stdout.fnmatch_lines(["*ValueError*42*", "*1 passed*1 error*"])
     assert result.ret
 
 
-@pytest.mark.skipif(
-    sys.version_info[:2] == (2, 7),
-    reason="Only available in pytest 5.0+ (Python 3 only)",
-)
 def test_config_initialization(testdir, monkeypatch, pytestconfig):
-    """Ensure workers and master are initialized consistently. Integration test for #445"""
-    if not hasattr(pytestconfig, "invocation_params"):
-        pytest.skip(
-            "requires pytest >=5.1 (config has no attribute 'invocation_params')"
-        )
+    """Ensure workers and controller are initialized consistently. Integration test for #445"""
     testdir.makepyfile(
         **{
             "dir_a/test_foo.py": """
@@ -699,32 +671,6 @@ def test_skipping(testdir):
     result.stdout.fnmatch_lines(["*hello*", "*1 skipped*"])
 
 
-def test_issue34_pluginloading_in_subprocess(testdir):
-    import _pytest.hookspec
-
-    if not hasattr(_pytest.hookspec, "pytest_namespace"):
-        pytest.skip("this pytest version no longer supports pytest_namespace()")
-
-    testdir.tmpdir.join("plugin123.py").write(
-        textwrap.dedent(
-            """
-        def pytest_namespace():
-            return {'sample_variable': 'testing'}
-    """
-        )
-    )
-    testdir.makepyfile(
-        """
-        import pytest
-        def test_hello():
-            assert pytest.sample_variable == "testing"
-    """
-    )
-    result = testdir.runpytest_subprocess("-n1", "-p", "plugin123")
-    assert result.ret == 0
-    result.stdout.fnmatch_lines(["*1 passed*"])
-
-
 def test_fixture_scope_caching_issue503(testdir):
     p1 = testdir.makepyfile(
         """
@@ -772,8 +718,7 @@ def test_issue_594_random_parametrize(testdir):
 
 
 def test_tmpdir_disabled(testdir):
-    """Test xdist doesn't break if internal tmpdir plugin is disabled (#22).
-    """
+    """Test xdist doesn't break if internal tmpdir plugin is disabled (#22)."""
     p1 = testdir.makepyfile(
         """
         def test_ok():
@@ -787,8 +732,7 @@ def test_tmpdir_disabled(testdir):
 
 @pytest.mark.parametrize("plugin", ["xdist.looponfail", "xdist.boxed"])
 def test_sub_plugins_disabled(testdir, plugin):
-    """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)
-    """
+    """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)"""
     p1 = testdir.makepyfile(
         """
         def test_ok():
@@ -802,27 +746,15 @@ def test_sub_plugins_disabled(testdir, plugin):
 
 class TestWarnings:
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
-    def test_warnings(self, testdir, n, request, warn_type):
-        if warn_type == "builtin":
-            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
-        elif warn_type == "pytest":
-            if not hasattr(request.config, "warn"):
-                pytest.skip("config.warn has been removed in pytest 4.1")
-            warn_code = """request.config.warn('', 'this is a warning',
-                           fslocation=py.path.local())"""
-        else:
-            assert False
+    def test_warnings(self, testdir, n):
         testdir.makepyfile(
             """
             import warnings, py, pytest
 
             @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
             def test_func(request):
-                {warn_code}
-        """.format(
-                warn_code=warn_code
-            )
+                warnings.warn(UserWarning('this is a warning'))
+            """
         )
         result = testdir.runpytest(n)
         result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warning*"])
@@ -838,19 +770,23 @@ class TestWarnings:
 
         testdir.makeconftest(
             """
-            def pytest_warning_captured():
-                assert False, "this hook should not be called in this version"
+            def pytest_warning_captured(warning_message):
+                if warning_message == "my custom worker warning":
+                    assert False, (
+                        "this hook should not be called from workers "
+                        "in this version: {}"
+                    ).format(warning_message)
         """
         )
         testdir.makepyfile(
             """
             import warnings
             def test():
-                warnings.warn("custom warning")
+                warnings.warn("my custom worker warning")
         """
         )
         result = testdir.runpytest("-n1")
-        result.stdout.fnmatch_lines(["* 1 passed in *"])
+        result.stdout.fnmatch_lines(["*1 passed*"])
         result.stdout.no_fnmatch_line("*this hook should not be called in this version")
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
@@ -899,9 +835,6 @@ class TestWarnings:
         """Check that warnings with unserializable _WARNING_DETAILS are
         handled correctly (#379).
         """
-        if sys.version_info[0] < 3:
-            # The issue is only present in Python 3 warnings
-            return
         testdir.makepyfile(
             """
             import warnings, pytest
@@ -1203,6 +1136,18 @@ def test_internal_error_with_maxfail(testdir):
     assert "INTERNALERROR" not in result.stderr.str()
 
 
+def test_internal_errors_propagate_to_controller(testdir):
+    testdir.makeconftest(
+        """
+        def pytest_collection_modifyitems():
+            raise RuntimeError("Some runtime error")
+        """
+    )
+    testdir.makepyfile("def test(): pass")
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines(["*RuntimeError: Some runtime error*"])
+
+
 class TestLoadScope:
     def test_by_module(self, testdir):
         test_file = """
@@ -1292,14 +1237,22 @@ class TestFileScope:
             "test_b.py::TestB", result.outlines
         )
 
-        assert test_a_workers_and_test_count in (
+        assert (
+            test_a_workers_and_test_count
+            in (
                 {"gw0": 10},
                 {"gw1": 0},
-        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        assert test_b_workers_and_test_count in (
+            )
+            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
+        assert (
+            test_b_workers_and_test_count
+            in (
                 {"gw0": 10},
                 {"gw1": 0},
-        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+            )
+            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
 
     def test_by_class(self, testdir):
         testdir.makepyfile(
@@ -1324,14 +1277,22 @@ class TestFileScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert test_a_workers_and_test_count in (
+        assert (
+            test_a_workers_and_test_count
+            in (
                 {"gw0": 10},
                 {"gw1": 0},
-        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        assert test_b_workers_and_test_count in (
+            )
+            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
+        assert (
+            test_b_workers_and_test_count
+            in (
                 {"gw0": 10},
                 {"gw1": 0},
-        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+            )
+            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        )
 
     def test_module_single_start(self, testdir):
         """Fix test suite never finishing in case all workers start with a single test (#277)."""
@@ -1417,7 +1378,7 @@ def parse_tests_and_workers_from_output(lines):
             r"""
             \[(gw\d)\]  # worker
             \s*
-            (?:\[\s*\d+%\])? # progress indicator (pytest >=3.3)
+            (?:\[\s*\d+%\])? # progress indicator
             \s(.*?)     # status string ("PASSED")
             \s(.*::.*)  # nodeid
         """,
@@ -1436,3 +1397,45 @@ def get_workers_and_test_count_by_prefix(prefix, lines, expected_status="PASSED"
         if expected_status == status and nodeid.startswith(prefix):
             result[worker] = result.get(worker, 0) + 1
     return result
+
+
+class TestAPI:
+    @pytest.fixture
+    def fake_request(self):
+        class FakeOption:
+            def __init__(self):
+                self.dist = "load"
+
+        class FakeConfig:
+            def __init__(self):
+                self.workerinput = {"workerid": "gw5"}
+                self.option = FakeOption()
+
+        class FakeRequest:
+            def __init__(self):
+                self.config = FakeConfig()
+
+        return FakeRequest()
+
+    def test_is_xdist_worker(self, fake_request):
+        assert xdist.is_xdist_worker(fake_request)
+        del fake_request.config.workerinput
+        assert not xdist.is_xdist_worker(fake_request)
+
+    def test_is_xdist_controller(self, fake_request):
+
+        assert not xdist.is_xdist_master(fake_request)
+        assert not xdist.is_xdist_controller(fake_request)
+
+        del fake_request.config.workerinput
+        assert xdist.is_xdist_master(fake_request)
+        assert xdist.is_xdist_controller(fake_request)
+
+        fake_request.config.option.dist = "no"
+        assert not xdist.is_xdist_master(fake_request)
+        assert not xdist.is_xdist_controller(fake_request)
+
+    def test_get_xdist_worker_id(self, fake_request):
+        assert xdist.get_xdist_worker_id(fake_request) == "gw5"
+        del fake_request.config.workerinput
+        assert xdist.get_xdist_worker_id(fake_request) == "master"

testing/conftest.py

@@ -1,20 +1,9 @@
-import six
 import py
 import pytest
 import execnet
 
 pytest_plugins = "pytester"
 
-if six.PY2:
-
-    @pytest.fixture(scope="session", autouse=True)
-    def _ensure_imports():
-        # we import some modules because pytest-2.8's testdir fixture
-        # will unload all modules after each test and this cause
-        # (unknown) problems with execnet.Group()
-        execnet.Group
-        execnet.makegateway
-
 
 @pytest.fixture(autouse=True)
 def _divert_atexit(request, monkeypatch):

testing/test_dsession.py

@@ -203,7 +203,7 @@ class TestLoadScheduling:
         different test ids are collected by workers.
         """
 
-        class CollectHook(object):
+        class CollectHook:
             """
             Dummy hook that stores collection reports.
             """

testing/test_newhooks.py

@@ -46,8 +46,7 @@ class TestHooks:
         )
 
     def test_node_collection_finished(self, testdir):
-        """Test pytest_xdist_node_collection_finished hook (#8).
-        """
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
         testdir.makeconftest(
             """
             def pytest_xdist_node_collection_finished(node, ids):
@@ -61,3 +60,37 @@ class TestHooks:
             ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
         )
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, testdir):
+        testdir.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, testdir):
+        """Test pytest_handlecrashitem hook."""
+        testdir.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = testdir.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])

testing/test_plugin.py

@@ -1,7 +1,11 @@
+from contextlib import suppress
+
 import py
 import execnet
 from xdist.workermanage import NodeManager
 
+import pytest
+
 
 def test_dist_incompatibility_messages(testdir):
     result = testdir.runpytest("--pdb", "--looponfail")
@@ -38,6 +42,11 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     import os
     from xdist.plugin import pytest_cmdline_main as check_options
 
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+
     if hasattr(os, "sched_getaffinity"):
         monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
     elif hasattr(os, "cpu_count"):
@@ -51,6 +60,7 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption("numprocesses") == 2
 
     config = testdir.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 99
 
     config = testdir.parseconfigure("-nauto", "--pdb")
@@ -59,12 +69,53 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption("numprocesses") == 0
     assert config.getoption("dist") == "no"
 
+    config = testdir.parseconfigure("-nlogical", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
     config = testdir.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 2
 
 
+def test_auto_detect_cpus_psutil(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_hook_auto_num_workers(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    testdir.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = testdir.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
 def test_boxed_with_collect_only(testdir):
     from xdist.plugin import pytest_cmdline_main as check_options
 

testing/test_remote.py

@@ -8,7 +8,7 @@ from xdist.workermanage import WorkerController
 import execnet
 import marshal
 
-from six.moves.queue import Queue
+from queue import Queue
 
 WAIT_TIMEOUT = 10.0
 
@@ -37,7 +37,9 @@ class WorkerSetup:
         self.testdir = testdir
         self.events = Queue()
 
-    def setup(self,):
+    def setup(
+        self,
+    ):
         self.testdir.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
         self.gateway = execnet.makegateway()
@@ -290,3 +292,17 @@ def test_remote_usage_prog(testdir, request):
     result = testdir.runpytest_subprocess("-n1")
     assert result.ret == 1
     result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(testdir):
+    """Work around sys.path differences due to execnet using `python -c`."""
+    testdir.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
+    )
+    result = testdir.runpytest("-n1")
+    assert result.ret == 0

testing/test_workermanage.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import py
 import pytest
 import textwrap
@@ -35,7 +34,7 @@ def mysetup(tmpdir):
 
 @pytest.fixture
 def workercontroller(monkeypatch):
-    class MockController(object):
+    class MockController:
         def __init__(self, *args):
             pass
 
@@ -197,6 +196,25 @@ class TestNodeManager:
             assert dest.join("dir1", "dir2", "hello").check()
             nodemanager.teardown_nodes()
 
+    @pytest.mark.parametrize(
+        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+    )
+    def test_rsync_report(
+        self, testdir, mysetup, workercontroller, capsys, flag, expects_report
+    ):
+        source, dest = mysetup.source, mysetup.dest
+        dir1 = mysetup.source.mkdir("dir1")
+        args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source
+        if flag:
+            args += (flag,)
+        nodemanager = NodeManager(testdir.parseconfig(*args))
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
     def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)

tox.ini

@@ -1,24 +1,34 @@
 [tox]
 envlist=
   linting
-  py{27,35,36,37,38}-pytestlatest
-  py38-pytestmaster
+  py{36,37,38,39,310}-pytestlatest
+  py38-pytestmain
+  py38-psutil
+  py38-setproctitle
 
 [testenv]
-passenv = USER USERNAME
 extras = testing
 deps =
   pytestlatest: pytest
-  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
 commands=
   pytest {posargs}
 
-[testenv:linting]
-skipsdist = True
-usedevelop = True
-deps =
-  pre-commit
-commands = pre-commit run --all-files --show-diff-on-failure
+[testenv:py38-psutil]
+extras =
+  testing
+  psutil
+deps = pytest
+commands =
+  pytest {posargs:-k psutil}
+
+[testenv:py38-setproctitle]
+extras =
+  testing
+  setproctitle
+deps = pytest
+commands =
+  pytest {posargs}
 
 [testenv:release]
 changedir=
@@ -33,7 +43,9 @@ commands =
   towncrier --version {posargs} --yes
 
 [pytest]
-addopts = -ra
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
 testpaths = testing
 
 [flake8]