Remove unnecessary skip from test_logfinish_hook as we require pytest>=6.2

updates:
- [github.com/psf/black: 21.11b1 → 21.12b0](https://github.com/psf/black/compare/21.11b1...21.12b0)

.appveyor.yml

@@ -1,20 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: "py35-pytestlatest"
-  - TOXENV: "py36-pytestlatest"
-  - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py38-pytestlatest"
-  - TOXENV: "py38-pytestmaster"
-
-install:
-  - C:\Python38\python -m pip install -U pip setuptools virtualenv
-  - C:\Python38\python -m pip install -U tox setuptools_scm
-
-build: false  # Not a C# project, build stuff at the test step instead.
-
-test_script:
-  - C:\Python38\python -m tox
-
-# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
-# might as well save resources
-skip_tags: true

.github/workflows/main.yml

@@ -0,0 +1,81 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py36-pytestlatest"
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py38-pytestmain"
+          - "py38-psutil"
+          - "py38-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py36-pytestlatest"
+            python: "3.6"
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10-dev"
+          - tox_env: "py38-pytestmain"
+            python: "3.8"
+          - tox_env: "py38-psutil"
+            python: "3.8"
+          - tox_env: "py38-setproctitle"
+            python: "3.8"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}
+
+  deploy:
+
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    runs-on: ubuntu-latest
+
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+    - name: Install wheel
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,20 +1,22 @@
 repos:
--   repo: https://github.com/ambv/black
+-   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 21.12b0
     hooks:
     -   id: black
         args: [--safe, --quiet, --target-version, py35]
-        language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.4.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
     -   id: flake8
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.7.2
+    rev: v2.29.1
     hooks:
     -   id: pyupgrade
         args: [--py3-plus]
@@ -26,4 +28,12 @@ repos:
         files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
-        language_version: python3.7
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910-1
+    hooks:
+    -   id: mypy
+        files: ^(src/|testing/)
+        args: []
+        additional_dependencies:
+          - pytest>=6.2.0
+          - py>=1.10.0

.travis.yml

@@ -1,62 +0,0 @@
-dist: xenial
-language: python
-
-notifications:
-  irc:
-    channels:
-      - 'chat.freenode.net#pytest'
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-  - pytest-commit@python.org
-
-install:
-  - pip install -U pip setuptools
-  - pip install tox setuptools_scm
-script: tox
-
-stages:
-  - baseline
-  - test
-  - name: deploy
-    if: repo = pytest-dev/pytest-xdist AND tag IS present
-
-jobs:
-  include:
-    - stage: baseline
-      python: '3.7'
-      env: TOXENV=linting
-      cache:
-        directories:
-          - $HOME/.cache/pre-commit
-    - python: '3.8'
-      env: TOXENV=py38-pytestlatest
-
-    - stage: test
-      python: "3.5"
-      env: TOXENV=py35-pytestlatest
-    - python: "3.6"
-      env: TOXENV=py36-pytestlatest
-    - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-    - python: "3.9-dev"
-      env: TOXENV=py39-pytestlatest
-    - python: "3.8"
-      env: TOXENV=py38-pytestmaster
-
-    - stage: deploy
-      python: '3.8'
-      env:
-      install: pip install -U setuptools setuptools_scm
-      script: skip
-      deploy:
-        provider: pypi
-        user: ronny
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,109 @@
+pytest-xdist 2.5.0 (2021-12-10)
+===============================
+
+Features
+--------
+
+- `#722 <https://github.com/pytest-dev/pytest-xdist/issues/722>`_: Full compatibility with pytest 7 - no deprecation warnings or use of legacy features.
+
+- `#733 <https://github.com/pytest-dev/pytest-xdist/issues/733>`_: New ``--dist=loadgroup`` option, which ensures all tests marked with ``@pytest.mark.xdist_group`` run in the same session/worker. Other tests run distributed as in ``--dist=load``.
+
+
+Trivial Changes
+---------------
+
+- `#708 <https://github.com/pytest-dev/pytest-xdist/issues/708>`_: Use ``@pytest.hookspec`` decorator to declare hook options in ``newhooks.py`` to avoid warnings in ``pytest 7.0``.
+
+- `#719 <https://github.com/pytest-dev/pytest-xdist/issues/719>`_: Use up-to-date ``setup.cfg``/``pyproject.toml`` packaging setup.
+
+- `#720 <https://github.com/pytest-dev/pytest-xdist/issues/720>`_: Require pytest>=6.2.0.
+
+- `#721 <https://github.com/pytest-dev/pytest-xdist/issues/721>`_: Started using type annotations and mypy checking internally. The types are incomplete and not published.
+
+
+pytest-xdist 2.4.0 (2021-09-20)
+===============================
+
+Features
+--------
+
+- `#696 <https://github.com/pytest-dev/pytest-xdist/issues/696>`_: On Linux, the process title now changes to indicate the current worker state (running/idle).
+
+  Depends on the `setproctitle <https://pypi.org/project/setproctitle/>`__ package, which can be installed with ``pip install pytest-xdist[setproctitle]``.
+
+- `#704 <https://github.com/pytest-dev/pytest-xdist/issues/704>`_: Add support for Python 3.10.
+
+
+pytest-xdist 2.3.0 (2021-06-16)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#654 <https://github.com/pytest-dev/pytest-xdist/issues/654>`_: Python 3.5 is no longer supported.
+
+
+Features
+--------
+
+- `#646 <https://github.com/pytest-dev/pytest-xdist/issues/646>`_: Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+  This is very useful for test suites which are not CPU-bound.
+
+- `#650 <https://github.com/pytest-dev/pytest-xdist/issues/650>`_: Added new ``pytest_handlecrashitem`` hook to allow handling and rescheduling crashed items.
+
+
+Bug Fixes
+---------
+
+- `#421 <https://github.com/pytest-dev/pytest-xdist/issues/421>`_: Copy the parent process sys.path into local workers, to work around execnet's python -c adding the current directory to sys.path.
+
+- `#638 <https://github.com/pytest-dev/pytest-xdist/issues/638>`_: Fix issue caused by changing the branch name of the pytest repository.
+
+
+Trivial Changes
+---------------
+
+- `#592 <https://github.com/pytest-dev/pytest-xdist/issues/592>`_: Replace master with controller where ever possible.
+
+- `#643 <https://github.com/pytest-dev/pytest-xdist/issues/643>`_: Use 'main' to refer to pytest default branch in tox env names.
+
+
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
 pytest-xdist 2.0.0 (2020-08-12)
 ===============================
 

OVERVIEW.md

@@ -1,76 +0,0 @@
-# Overview #
-
-`xdist` works by spawning one or more **workers**, which are controlled
-by the **master**. Each **worker** is responsible for performing
-a full test collection and afterwards running tests as dictated by the **master**.
-
-The execution flow is:
-
-1. **master** spawns one or more **workers** at the beginning of
-   the test session. The communication between **master** and **worker** nodes makes use of
-   [execnet](http://codespeak.net/execnet/) and its [gateways](http://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters).
-   The actual interpreters executing the code for the **workers** might
-   be remote or local.
-
-1. Each **worker** itself is a mini pytest runner. **workers** at this
-   point perform a full test collection, sending back the collected
-   test-ids back to the **master** which does not
-   perform any collection itself.
-
-1. The **master** receives the result of the collection from all nodes.
-   At this point the **master** performs some sanity check to ensure that
-   all **workers** collected the same tests (including order), bailing out otherwise.
-   If all is well, it converts the list of test-ids into a list of simple
-   indexes, where each index corresponds to the position of that test in the
-   original collection list. This works because all nodes have the same
-   collection list, and saves bandwidth because the **master** can now tell
-   one of the workers to just *execute test index 3* index of passing the
-   full test id.
-
-1. If **dist-mode** is **each**: the **master** just sends the full list
-   of test indexes to each node at this moment.
-
-1. If **dist-mode** is **load**: the **master** takes around 25% of the
-   tests and sends them one by one to each **worker** in a round robin
-   fashion. The rest of the tests will be distributed later as **workers**
-   finish tests (see below).
-
-1. Note that `pytest_xdist_make_scheduler` hook can be used to implement custom tests distribution logic.
-
-1. **workers** re-implement `pytest_runtestloop`: pytest's default implementation
-   basically loops over all collected items in the `session` object and executes
-   the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly
-   waiting for **master** to send tests for execution. As tests are
-   received by **workers**, `pytest_runtest_protocol` is executed for each test.
-   Here it worth noting an implementation detail: **workers** always must keep at
-   least one test item on their queue due to how the `pytest_runtest_protocol(item, nextitem)`
-   hook is defined: in order to pass the `nextitem` to the hook, the worker must wait for more
-   instructions from master before executing that remaining test. If it receives more tests,
-   then it can safely call `pytest_runtest_protocol` because it knows what the `nextitem` parameter will be.
-   If it receives a "shutdown" signal, then it can execute the hook passing `nextitem` as `None`.
-
-1. As tests are started and completed at the **workers**, the results are sent
-   back to the **master**, which then just forwards the results to
-   the appropriate pytest hooks: `pytest_runtest_logstart` and
-   `pytest_runtest_logreport`. This way other plugins (for example `junitxml`)
-   can work normally. The **master** (when in dist-mode **load**)
-   decides to send more tests to a node when a test completes, using
-   some heuristics such as test durations and how many tests each **worker**
-   still has to run.
-
-1. When the **master** has no more pending tests it will
-   send a "shutdown" signal to all **workers**, which will then run their
-   remaining tests to completion and shut down. At this point the
-   **master** will sit waiting for **workers** to shut down, still
-   processing events such as `pytest_runtest_logreport`.
-
-## FAQ ##
-
-> Why does each worker do its own collection, as opposed to having
-the master collect once and distribute from that collection to the workers?
-
-If collection was performed by master then it would have to
-serialize collected items to send them through the wire, as workers live in another process.
-The problem is that test items are not easily (impossible?) to serialize, as they contain references to
-the test functions, fixture managers, config objects, etc. Even if one manages to serialize it,
-it seems it would be very hard to get it right and easy to break by any small change in pytest.

README.rst

@@ -1,4 +1,6 @@
-
+============
+pytest-xdist
+============
 
 .. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
     :alt: PyPI version
@@ -11,27 +13,42 @@
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
-    :alt: Travis CI build status
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
-
-.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
-    :alt: AppVeyor build status
-    :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
 
-xdist: pytest distributed testing plugin
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
-========================================
+tests across multiple CPUs to speed up test execution::
 
-The `pytest-xdist`_ plugin extends pytest with some unique
+    pytest -n auto
-test execution modes:
 
-* test run parallelization_: if you have multiple CPUs or hosts you can use
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
-  those for a combined test run.  This allows to speed up
+the tests randomly across them. There is also a number of `distribution modes`_ to choose from.
-  development or to use special resources of `remote machines`_.
 
+**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+.. contents:: **Table of Contents**
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+
+Features
+--------
+
+* Test run parallelization_: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of `remote machines`_.
 
 * ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
   pytest waits until a file in your project changes and then re-runs
@@ -41,50 +58,33 @@ test execution modes:
 * `Multi-Platform`_ coverage: you can specify different Python interpreters
   or different platforms and run tests in parallel on all of them.
 
-Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
-program source code to the remote place.  All test results
+  program source code to the remote place.
-are reported back and displayed to your local terminal.
+  You may specify different Python versions and interpreters. It does not
-You may specify different Python versions and interpreters.
+  installs/synchronize dependencies however.
 
-If you would like to know how pytest-xdist works under the covers, checkout
+  **Note**: this mode exists mostly for backward compatibility, as modern development
-`OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
+  relies on continuous integration for multi-platform testing.
-
-
-Installation
-------------
-
-Install the plugin with::
-
-    pip install pytest-xdist
-
-or use the package in develop/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
-
-    pip install --editable .
 
 .. _parallelization:
 
-Speed up test runs by sending tests to multiple CPUs
+Running tests across multiple CPUs
-----------------------------------------------------
+----------------------------------
 
-To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
 
-    pytest -n NUMCPUS
+    pytest -n 8
 
 Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
 can lead to considerable speed ups, especially if your test suite takes a
 noticeable amount of time.
 
-If a test crashes a worker, pytest-xdist will automatically restart that worker
+The test distribution algorithm is configured with the ``--dist`` command-line option:
-and report the test’s failure. You can use the ``--max-worker-restart`` option
-to limit the number of worker restarts that are allowed, or disable restarting
-altogether using ``--max-worker-restart 0``.
 
-By default, using ``--numprocesses`` will send pending tests to any worker that
+.. _distribution modes:
-is available, without any guaranteed order. You can change the test
-distribution algorithm this with the ``--dist`` option. It takes these values:
 
-* ``--dist no``: The default algorithm, distributing one test at a time.
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order.
 
 * ``--dist loadscope``: Tests are grouped by **module** for *test functions*
   and by **class** for *test methods*. Groups are distributed to available
@@ -97,72 +97,40 @@ distribution algorithm this with the ``--dist`` option. It takes these values:
   distributed to available workers as whole units. This guarantees that all
   tests in a file run in the same worker.
 
-Making session-scoped fixtures execute only once
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker.
 
-``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+  .. code-block:: python
-a subset of all tests. This means that tests in different processes requesting a high-level
-scoped fixture (for example ``session``) will execute the fixture code more than once, which
-breaks expectations and might be undesired in certain situations.
 
-While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+      @pytest.mark.xdist_group(name="group1")
-executed exactly once, this can be achieved by using a lock file for inter-process communication.
+      def test1():
+          pass
 
-The example below needs to execute the fixture ``session_data`` only once (because it is
+      class TestA:
-resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+          @pytest.mark.xdist_group("group1")
-use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+          def test2():
-when the first process requests the fixture, while the other processes will then read
+              pass
-the data from a file.
 
-Here is the code:
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
 
-.. code-block:: python
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).
 
-    import json
-
-    import pytest
-    from filelock import FileLock
-
-
-    @pytest.fixture(scope="session")
-    def session_data(tmp_path_factory, worker_id):
-        if not worker_id:
-            # not executing in with multiple workers, just produce the data and let
-            # pytest's fixture caching do its job
-            return produce_expensive_data()
-
-        # get the temp directory shared by all workers
-        root_tmp_dir = tmp_path_factory.getbasetemp().parent
-
-        fn = root_tmp_dir / "data.json"
-        with FileLock(str(fn) + ".lock"):
-            if fn.is_file():
-                data = json.loads(fn.read_text())
-            else:
-                data = produce_expensive_data()
-                fn.write_text(json.dumps(data))
-        return data
-
-
-The example above can also be use in cases a fixture needs to execute exactly once per test session, like
-initializing a database service and populating initial tables.
-
-This technique might not work for every case, but should be a starting point for many situations
-where executing a high-scope fixture exactly once is important.
 
 Running tests in a Python subprocess
 ------------------------------------
 
-To instantiate a python3.5 subprocess and send tests to it, you may type::
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
 
-    pytest -d --tx popen//python=python3.5
+    pytest -d --tx popen//python=python3.9
 
-This will start a subprocess which is run with the ``python3.5``
+This will start a subprocess which is run with the ``python3.9``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python3.5
+    --tx 3*popen//python=python3.9
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -244,14 +212,27 @@ environment this command will send each tests to all
 platforms - and report back failures from all platforms
 at once. The specifications strings use the `xspec syntax`_.
 
-.. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
 
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
 
-.. _`execnet`: http://codespeak.net/execnet
+.. _`execnet`: https://codespeak.net/execnet
+
+
+When tests crash
+----------------
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.
+
+
+How-tos
+-------
 
 Identifying the worker process during a test
---------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 *New in version 1.15.*
 
@@ -288,8 +269,8 @@ Since version 2.0, the following functions are also available in the ``xdist`` m
         :param request_or_session: the `pytest` `request` or `session` object
         """
 
-    def is_xdist_master(request_or_session) -> bool:
+     def is_xdist_controller(request_or_session) -> bool:
-        """Return `True` if this is the xdist master, `False` otherwise
+        """Return `True` if this is the xdist controller, `False` otherwise
 
         Note: this method also returns `False` when distribution has not been
         activated at all.
@@ -297,18 +278,42 @@ Since version 2.0, the following functions are also available in the ``xdist`` m
         :param request_or_session: the `pytest` `request` or `session` object
         """
 
+    def is_xdist_master(request_or_session) -> bool:
+        """Deprecated alias for is_xdist_controller."""
+
     def get_xdist_worker_id(request_or_session) -> str:
         """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-        if running on the 'master' node.
+        if running on the controller node.
 
-        If not distributing tests (for example passing `-n0` or not passing `-n` at all) also return 'master'.
+        If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+        also return 'master'.
 
         :param request_or_session: the `pytest` `request` or `session` object
         """
 
 
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
 Uniquely identifying the current test run
------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 *New in version 1.32.*
 
@@ -341,15 +346,15 @@ Additionally, during a test run, the following environment variable is defined:
 
 * ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
 
-Accessing ``sys.argv`` from the master node in workers
+Accessing ``sys.argv`` from the controller node in workers
-------------------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-To access the ``sys.argv`` passed to the command-line of the master node, use
+To access the ``sys.argv`` passed to the command-line of the controller node, use
 ``request.config.workerinput["mainargv"]``.
 
 
 Specifying test exec environments in an ini file
-------------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can use pytest's ini file configuration to avoid typing common options.
 You can for example make running with three subprocesses your default like this:
@@ -364,7 +369,7 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
 
 and then just type::
 
@@ -374,7 +379,7 @@ to run tests in each of the environments.
 
 
 Specifying "rsync" dirs in an ini-file
---------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
 you may specify directories to include or to exclude in synchronisation:
@@ -391,3 +396,149 @@ where the configuration file was found.
 .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
 .. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
 .. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+How does xdist work?
+--------------------
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* index of passing the full test
+   id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

RELEASING.rst

@@ -41,3 +41,5 @@ To publish a new release ``X.Y.Z``, the steps are as follows:
     $ git push git@github.com:pytest-dev/pytest-xdist.git v$VERSION
 
 That will build the package and publish it on ``PyPI`` automatically.
+
+#. Merge the release PR to `master`.

changelog/468.deprecation.rst

@@ -0,0 +1,3 @@
+The ``--boxed`` commmand line argument is deprecated.
+Install pytest-forked and use ``--forked`` instead.
+pytest-xdist 3.0.0 will remove the ``--boxed`` argument and pytest-forked dependency.

example/boxed.txt

@@ -1,9 +1,9 @@
-.. note::
+.. warning::
 
   Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a
   separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_
-  which can be installed independently. The ``--boxed`` command-line options remains
+  which can be installed independently. The ``--boxed`` command-line option is deprecated
-  for backward compatibility reasons.
+  and will be removed in pytest-xdist 3.0.0; use ``--forked`` from pytest-forked instead.
 
 
 If your testing involves C or C++ libraries you might have to deal

pyproject.toml

@@ -1,3 +1,15 @@
+[build-system]
+requires = [
+  # sync with setup.py until we discard non-pep-517/518
+  "setuptools>=45.0",
+  "setuptools-scm[toml]>=6.2.3",
+  "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+write_to = "src/xdist/_version.py"
+
 [tool.towncrier]
 package = "xdist"
 filename = "CHANGELOG.rst"

setup.cfg

@@ -1,8 +1,76 @@
-[bdist_wheel]
-universal = 1
-
 [metadata]
+name = pytest-xdist
+description = pytest xdist plugin for distributed testing and loop-on-failing modes
+long_description = file: README.rst
+license = MIT
+author = holger krekel and contributors
+author_email = pytest-dev@python.org,holger@merlinux.eu
+url = https://github.com/pytest-dev/pytest-xdist
+platforms =
+    linux
+    osx
+    win32
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Framework :: Pytest
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Operating System :: POSIX
+    Operating System :: Microsoft :: Windows
+    Operating System :: MacOS :: MacOS X
+    Topic :: Software Development :: Testing
+    Topic :: Software Development :: Quality Assurance
+    Topic :: Utilities
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 license_file = LICENSE
 
+[options]
+packages = find:
+package_dir = =src
+zip_safe = False
+python_requires = >=3.6
+install_requires =
+    execnet>=1.1
+    pytest>=6.2.0
+    pytest-forked
+setup_requires = setuptools_scm>=6.0
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+pytest11 =
+    xdist = xdist.plugin
+    xdist.looponfail = xdist.looponfail
+
+[options.extras_require]
+testing =
+    filelock
+psutil = psutil>=3.0
+setproctitle = setproctitle
+
 [flake8]
 max-line-length = 100
+
+[mypy]
+mypy_path = src
+# TODO: Enable this & fix errors.
+# check_untyped_defs = True
+disallow_any_generics = True
+ignore_missing_imports = True
+no_implicit_optional = True
+show_error_codes = True
+strict_equality = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_configs = True
+# TODO: Enable this & fix errors.
+# no_implicit_reexport = True

setup.py

@@ -1,48 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import setup
 
-install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=6.0.0", "pytest-forked"]
+if __name__ == "__main__":
-
+    setup()
-
-with open("README.rst") as f:
-    long_description = f.read()
-
-setup(
-    name="pytest-xdist",
-    use_scm_version={"write_to": "src/xdist/_version.py"},
-    description="pytest xdist plugin for distributed testing and loop-on-failing modes",
-    long_description=long_description,
-    license="MIT",
-    author="holger krekel and contributors",
-    author_email="pytest-dev@python.org,holger@merlinux.eu",
-    url="https://github.com/pytest-dev/pytest-xdist",
-    platforms=["linux", "osx", "win32"],
-    packages=find_packages(where="src"),
-    package_dir={"": "src"},
-    extras_require={"testing": ["filelock"]},
-    entry_points={
-        "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
-    },
-    zip_safe=False,
-    python_requires=">=3.5",
-    install_requires=install_requires,
-    setup_requires=["setuptools_scm"],
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Framework :: Pytest",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: POSIX",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: MacOS :: MacOS X",
-        "Topic :: Software Development :: Testing",
-        "Topic :: Software Development :: Quality Assurance",
-        "Topic :: Utilities",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-    ],
-)

src/xdist/__init__.py

@@ -1,4 +1,15 @@
-from xdist.plugin import is_xdist_worker, is_xdist_master, get_xdist_worker_id
+from xdist.plugin import (
+    is_xdist_worker,
+    is_xdist_master,
+    get_xdist_worker_id,
+    is_xdist_controller,
+)
 from xdist._version import version as __version__
 
-__all__ = ["__version__", "is_xdist_worker", "is_xdist_master", "get_xdist_worker_id"]
+__all__ = [
+    "__version__",
+    "is_xdist_worker",
+    "is_xdist_master",
+    "is_xdist_controller",
+    "get_xdist_worker_id",
+]

src/xdist/dsession.py

@@ -7,6 +7,7 @@ from xdist.scheduler import (
     LoadScheduling,
     LoadScopeScheduling,
     LoadFileScheduling,
+    LoadGroupScheduling,
 )
 
 
@@ -14,7 +15,7 @@ from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
+    """signals an immediate interruption."""
 
 
 class DSession:
@@ -67,7 +68,7 @@ class DSession:
         if self.terminal and self.config.option.verbose >= 0:
             self.terminal.write_line(line)
 
-    @pytest.mark.trylast
+    @pytest.hookimpl(trylast=True)
     def pytest_sessionstart(self, session):
         """Creates and starts the nodes.
 
@@ -79,6 +80,7 @@ class DSession:
         self._active_nodes.update(nodes)
         self._session = session
 
+    @pytest.hookimpl
     def pytest_sessionfinish(self, session):
         """Shutdown all nodes."""
         nm = getattr(self, "nodemanager", None)  # if not fully initialized
@@ -86,11 +88,12 @@ class DSession:
             nm.teardown_nodes()
         self._session = None
 
+    @pytest.hookimpl
     def pytest_collection(self):
-        # prohibit collection of test items in master process
+        # prohibit collection of test items in controller process
         return True
 
-    @pytest.mark.trylast
+    @pytest.hookimpl(trylast=True)
     def pytest_xdist_make_scheduler(self, config, log):
         dist = config.getvalue("dist")
         schedulers = {
@@ -98,9 +101,11 @@ class DSession:
             "load": LoadScheduling,
             "loadscope": LoadScopeScheduling,
             "loadfile": LoadFileScheduling,
+            "loadgroup": LoadGroupScheduling,
         }
         return schedulers[dist](config, log)
 
+    @pytest.hookimpl
     def pytest_runtestloop(self):
         self.sched = self.config.hook.pytest_xdist_make_scheduler(
             config=self.config, log=self.log
@@ -174,6 +179,24 @@ class DSession:
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
     def worker_errordown(self, node, error):
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
@@ -205,6 +228,7 @@ class DSession:
             self._clone_node(node)
         self._active_nodes.remove(node)
 
+    @pytest.hookimpl
     def pytest_terminal_summary(self, terminalreporter):
         if self.config.option.verbose >= 0 and self._summary_report:
             terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
@@ -222,7 +246,7 @@ class DSession:
             return
         self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
         # tell session which items were effectively collected otherwise
-        # the master node will finish the session with EXIT_NOTESTSCOLLECTED
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
         self.sched.add_node_collection(node, ids)
         if self.terminal:
@@ -270,6 +294,8 @@ class DSession:
 
     def worker_warning_captured(self, warning_message, when, item):
         """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
+        # This hook as been removed in pytest 7.1, and we can remove support once we only
+        # support pytest >=7.1.
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
@@ -325,6 +351,12 @@ class DSession:
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
         rep.node = worker
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
@@ -366,6 +398,7 @@ class TerminalDistReporter:
             self._lastlen = len(line)
         self.tr.rewrite(pline, bold=True)
 
+    @pytest.hookimpl
     def pytest_xdist_setupnodes(self, specs):
         self._specs = specs
         for spec in specs:
@@ -373,6 +406,7 @@ class TerminalDistReporter:
         self.setstatus(spec, "I", show=True)
         self.ensure_show_status()
 
+    @pytest.hookimpl
     def pytest_xdist_newgateway(self, gateway):
         if self.config.option.verbose > 0:
             rinfo = gateway._rinfo()
@@ -384,6 +418,7 @@ class TerminalDistReporter:
             )
         self.setstatus(gateway.spec, "C")
 
+    @pytest.hookimpl
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
             d = node.workerinfo
@@ -393,6 +428,7 @@ class TerminalDistReporter:
             self.rewrite(infoline, newline=True)
         self.setstatus(node.gateway.spec, "ok")
 
+    @pytest.hookimpl
     def pytest_testnodedown(self, node, error):
         if not error:
             return

src/xdist/looponfail.py

@@ -13,6 +13,7 @@ import time
 import execnet
 
 
+@pytest.hookimpl
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
@@ -26,6 +27,7 @@ def pytest_addoption(parser):
     )
 
 
+@pytest.hookimpl
 def pytest_cmdline_main(config):
 
     if config.getoption("looponfail"):
@@ -38,7 +40,7 @@ def pytest_cmdline_main(config):
 
 def looponfail_main(config):
     remotecontrol = RemoteControl(config)
-    rootdirs = config.getini("looponfailroots")
+    rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")]
     statrecorder = StatRecorder(rootdirs)
     try:
         while 1:
@@ -61,7 +63,7 @@ class RemoteControl:
 
     def trace(self, *args):
         if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
+            msg = " ".join(str(x) for x in args)
             print("RemoteControl:", msg)
 
     def initgateway(self):
@@ -178,6 +180,7 @@ class WorkerFailSession:
         if self.config.option.debug:
             print(" ".join(map(str, args)))
 
+    @pytest.hookimpl
     def pytest_collection(self, session):
         self.session = session
         self.trails = self.current_command
@@ -192,10 +195,12 @@ class WorkerFailSession:
         hook.pytest_collection_finish(session=session)
         return True
 
+    @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)
 
+    @pytest.hookimpl
     def pytest_collectreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)

src/xdist/newhooks.py

@@ -14,44 +14,78 @@ must be taken in plugins in case ``xdist`` is not installed. Please see:
 import pytest
 
 
+@pytest.hookspec()
 def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
+    """called before any remote node is set up."""
 
 
+@pytest.hookspec()
 def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
+    """called on new raw gateway creation."""
 
 
+@pytest.hookspec()
 def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
+    """called before rsyncing a directory to remote gateways takes place."""
 
 
+@pytest.hookspec()
 def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
+    """called after rsyncing a directory to remote gateways takes place."""
 
 
-@pytest.mark.firstresult
+@pytest.hookspec(firstresult=True)
 def pytest_xdist_getremotemodule():
-    """ called when creating remote node"""
+    """called when creating remote node"""
 
 
+@pytest.hookspec()
 def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
+    """configure node information before it gets instantiated."""
 
 
+@pytest.hookspec()
 def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
+    """Test Node is ready to operate."""
 
 
+@pytest.hookspec()
 def pytest_testnodedown(node, error):
-    """ Test Node is down. """
+    """Test Node is down."""
 
 
+@pytest.hookspec()
 def pytest_xdist_node_collection_finished(node, ids):
-    """called by the master node when a node finishes collecting.
+    """called by the controller node when a worker node finishes collecting."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_make_scheduler(config, log):
+    """return a node scheduler implementation"""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
     """
 
 
-@pytest.mark.firstresult
+@pytest.hookspec(firstresult=True)
-def pytest_xdist_make_scheduler(config, log):
+def pytest_handlecrashitem(crashitem, report, sched):
-    """ return a node scheduler implementation """
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -1,25 +1,57 @@
+import os
 import uuid
+import sys
+from pathlib import Path
 
-import psutil
 import py
 import pytest
 
 
-def auto_detect_cpus():
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
-    return psutil.cpu_count(logical=False) or psutil.cpu_count() or 1
+
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
 
 
-class AutoInt(int):
+@pytest.hookimpl
-    """Mark value as auto-detected."""
+def pytest_xdist_auto_num_workers(config):
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
+    try:
+        from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
+    except ImportError:
+        if os.environ.get("TRAVIS") == "true":
+            # workaround https://bitbucket.org/pypy/pypy/issues/2375
+            return 2
+        try:
+            from os import cpu_count
+        except ImportError:
+            from multiprocessing import cpu_count
+    try:
+        n = cpu_count()
+    except NotImplementedError:
+        return 1
+    return n if n else 1
 
 
 def parse_numprocesses(s):
-    if s == "auto":
+    if s in ("auto", "logical"):
-        return AutoInt(auto_detect_cpus())
+        return s
     elif s is not None:
         return int(s)
 
 
+@pytest.hookimpl
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
@@ -29,9 +61,10 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="shortcut for '--dist=load --tx=NUM*popen', "
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
-        "you can use 'auto' here for auto detection CPUs number on "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
-        "host system and it will be 0 when used with --pdb",
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -53,7 +86,7 @@ def pytest_addoption(parser):
         "--dist",
         metavar="distmode",
         action="store",
-        choices=["each", "load", "loadscope", "loadfile", "no"],
+        choices=["each", "load", "loadscope", "loadfile", "loadgroup", "no"],
         dest="dist",
         default="no",
         help=(
@@ -65,6 +98,7 @@ def pytest_addoption(parser):
             " the same scope to any available environment.\n\n"
             "loadfile: load balance by sending test grouped by file"
             " to any available environment.\n\n"
+            "loadgroup: like load, but sends tests marked with 'xdist_group' to the same worker.\n\n"
             "(default) no: run tests inprocess, don't distribute."
         ),
     )
@@ -120,18 +154,18 @@ def pytest_addoption(parser):
     parser.addini(
         "rsyncdirs",
         "list of (relative) paths to be rsynced for remote distributed testing.",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
     )
     parser.addini(
         "rsyncignore",
         "list of (relative) glob-style paths to be ignored for rsyncing.",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
     )
     parser.addini(
         "looponfailroots",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
         help="directories to check for changes",
-        default=[py.path.local()],
+        default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()],
     )
 
 
@@ -140,6 +174,7 @@ def pytest_addoption(parser):
 # -------------------------------------------------------------------------
 
 
+@pytest.hookimpl
 def pytest_addhooks(pluginmanager):
     from xdist import newhooks
 
@@ -151,7 +186,7 @@ def pytest_addhooks(pluginmanager):
 # -------------------------------------------------------------------------
 
 
-@pytest.mark.trylast
+@pytest.hookimpl(trylast=True)
 def pytest_configure(config):
     if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
         from xdist.dsession import DSession
@@ -162,18 +197,31 @@ def pytest_configure(config):
         if tr:
             tr.showfspath = False
     if config.getoption("boxed"):
+        warning = DeprecationWarning(
+            "The --boxed commmand line argument is deprecated. "
+            "Install pytest-forked and use --forked instead. "
+            "pytest-xdist 3.0.0 will remove the --boxed argument and pytest-forked dependency."
+        )
+        config.issue_config_time_warning(warning, 2)
         config.option.forked = True
 
+    config_line = (
+        "xdist_group: specify group for tests should run in same session."
+        "in relation to one another. " + "Provided by pytest-xdist."
+    )
+    config.addinivalue_line("markers", config_line)
 
-@pytest.mark.tryfirst
+
+@pytest.hookimpl(tryfirst=True)
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
-    if isinstance(config.option.numprocesses, AutoInt):
+    if config.option.numprocesses in ("auto", "logical"):
         if usepdb:
             config.option.numprocesses = 0
             config.option.dist = "no"
         else:
-            config.option.numprocesses = int(config.option.numprocesses)
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -204,8 +252,8 @@ def is_xdist_worker(request_or_session) -> bool:
     return hasattr(request_or_session.config, "workerinput")
 
 
-def is_xdist_master(request_or_session) -> bool:
+def is_xdist_controller(request_or_session) -> bool:
-    """Return `True` if this is the xdist master, `False` otherwise
+    """Return `True` if this is the xdist controller, `False` otherwise
 
     Note: this method also returns `False` when distribution has not been
     activated at all.
@@ -218,9 +266,13 @@ def is_xdist_master(request_or_session) -> bool:
     )
 
 
-def get_xdist_worker_id(request_or_session) -> str:
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
+def get_xdist_worker_id(request_or_session):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-    if running on the 'master' node.
+    if running on the controller node.
 
     If not distributing tests (for example passing `-n0` or not passing `-n` at all)
     also return 'master'.
@@ -230,6 +282,7 @@ def get_xdist_worker_id(request_or_session) -> str:
     if hasattr(request_or_session.config, "workerinput"):
         return request_or_session.config.workerinput["workerid"]
     else:
+        # TODO: remove "master", ideally for a None
         return "master"
 
 
@@ -238,6 +291,7 @@ def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
+    # TODO: remove "master", ideally for a None
     return get_xdist_worker_id(request)
 
 

src/xdist/remote.py

@@ -16,6 +16,21 @@ from execnet.gateway_base import dumps, DumpError
 
 from _pytest.config import _prepareconfig, Config
 
+try:
+    from setproctitle import setproctitle
+except ImportError:
+
+    def setproctitle(title):
+        pass
+
+
+def worker_title(title):
+    try:
+        setproctitle(title)
+    except Exception:
+        # changing the process name is very optional, no errors please
+        pass
+
 
 class WorkerInteractor:
     def __init__(self, config, channel):
@@ -32,10 +47,14 @@ class WorkerInteractor:
         self.log("sending", name, kwargs)
         self.channel.send((name, kwargs))
 
+    @pytest.hookimpl
     def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
 
+    @pytest.hookimpl
     def pytest_sessionstart(self, session):
         self.session = session
         workerinfo = getinfodict()
@@ -48,9 +67,11 @@ class WorkerInteractor:
         yield
         self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
+    @pytest.hookimpl
     def pytest_collection(self, session):
         self.sendevent("collectionstart")
 
+    @pytest.hookimpl
     def pytest_runtestloop(self, session):
         self.log("entering main loop")
         torun = []
@@ -83,26 +104,54 @@ class WorkerInteractor:
         else:
             nextitem = None
 
+        worker_title("[pytest-xdist running] %s" % item.nodeid)
+
         start = time.time()
         self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         duration = time.time() - start
+
+        worker_title("[pytest-xdist idle]")
+
         self.sendevent(
             "runtest_protocol_complete", item_index=self.item_index, duration=duration
         )
 
+    def pytest_collection_modifyitems(self, session, config, items):
+        # add the group name to nodeid as suffix if --dist=loadgroup
+        if config.getvalue("loadgroup"):
+            for item in items:
+                mark = item.get_closest_marker("xdist_group")
+                if not mark:
+                    continue
+                gname = (
+                    mark.args[0]
+                    if len(mark.args) > 0
+                    else mark.kwargs.get("name", "default")
+                )
+                item._nodeid = "{}@{}".format(item.nodeid, gname)
+
+    @pytest.hookimpl
     def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
         self.sendevent(
             "collectionfinish",
-            topdir=str(session.fspath),
+            topdir=topdir,
             ids=[item.nodeid for item in session.items],
         )
 
+    @pytest.hookimpl
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
+    @pytest.hookimpl
     def pytest_runtest_logfinish(self, nodeid, location):
         self.sendevent("logfinish", nodeid=nodeid, location=location)
 
+    @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
         data = self.config.hook.pytest_report_to_serializable(
             config=self.config, report=report
@@ -113,14 +162,16 @@ class WorkerInteractor:
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
+    @pytest.hookimpl
     def pytest_collectreport(self, report):
-        # send only reports that have not passed to master as optimization (#330)
+        # send only reports that have not passed to controller as optimization (#330)
         if not report.passed:
             data = self.config.hook.pytest_report_to_serializable(
                 config=self.config, report=report
             )
             self.sendevent("collectreport", data=data)
 
+    @pytest.hookimpl
     def pytest_warning_recorded(self, warning_message, when, nodeid, location):
         self.sendevent(
             "warning_recorded",
@@ -137,7 +188,7 @@ def serialize_warning_message(warning_message):
         message_class_name = type(warning_message.message).__name__
         message_str = str(warning_message.message)
         # check now if we can serialize the warning arguments (#349)
-        # if not, we will just use the exception message on the master node
+        # if not, we will just use the exception message on the controller node
         try:
             dumps(warning_message.message.args)
         except DumpError:
@@ -199,6 +250,7 @@ def remote_initconfig(option_dict, args):
 
 
 def setup_config(config, basetemp):
+    config.option.loadgroup = config.getvalue("dist") == "loadgroup"
     config.option.looponfail = False
     config.option.usepdb = False
     config.option.dist = "no"
@@ -209,15 +261,17 @@ def setup_config(config, basetemp):
 
 
 if __name__ == "__channelexec__":
-    channel = channel  # noqa
+    channel = channel  # type: ignore[name-defined] # noqa: F821
-    workerinput, args, option_dict, change_sys_path = channel.receive()
+    workerinput, args, option_dict, change_sys_path = channel.receive()  # type: ignore[name-defined]
 
-    if change_sys_path:
+    if change_sys_path is None:
         importpath = os.getcwd()
         sys.path.insert(0, importpath)
         os.environ["PYTHONPATH"] = (
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
+    else:
+        sys.path = change_sys_path
 
     os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
@@ -231,7 +285,7 @@ if __name__ == "__channelexec__":
 
     setup_config(config, option_dict.get("basetemp"))
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
-    config.workerinput = workerinput
+    config.workerinput = workerinput  # type: ignore[attr-defined]
-    config.workeroutput = {}
+    config.workeroutput = {}  # type: ignore[attr-defined]
-    interactor = WorkerInteractor(config, channel)
+    interactor = WorkerInteractor(config, channel)  # type: ignore[name-defined]
     config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -16,5 +16,5 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
         "The difference is:\n"
         "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/__init__.py

@@ -2,3 +2,4 @@ from xdist.scheduler.each import EachScheduling  # noqa
 from xdist.scheduler.load import LoadScheduling  # noqa
 from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
+from xdist.scheduler.loadgroup import LoadGroupScheduling  # noqa

src/xdist/scheduler/each.py

@@ -101,6 +101,14 @@ class EachScheduling:
     def mark_test_complete(self, node, item_index, duration=0):
         self.node2pending[node].remove(item_index)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def remove_node(self, node):
         # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)

src/xdist/scheduler/load.py

@@ -151,6 +151,14 @@ class LoadScheduling:
         self.node2pending[node].remove(item_index)
         self.check_schedule(node, duration=duration)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def check_schedule(self, node, duration=0):
         """Maybe schedule new items on the node
 

src/xdist/scheduler/loadgroup.py

@@ -0,0 +1,54 @@
+from .loadscope import LoadScopeScheduling
+from py.log import Producer
+
+
+class LoadGroupScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test by xdist_group mark.
+
+    This class behaves very much like LoadScopeScheduling, but it groups tests by xdist_group mark
+    instead of the module or class to which they belong to.
+    """
+
+    def __init__(self, config, log=None):
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadgroupsched")
+        else:
+            self.log = log.loadgroupsched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        With loadgroup, two cases are added::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            example/loadsuite/test/test_gamma.py::test_beta0@gname
+            example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname
+
+        This function will group tests with the scope determined by splitting the first ``@``
+        from the right. That is, test will be grouped in a single work unit when they have
+        same group name. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            gname
+            gname
+        """
+        if nodeid.rfind("@") > nodeid.rfind("]"):
+            # check the index of ']' to avoid the case: parametrize mark value has '@'
+            return nodeid.split("@")[-1]
+        else:
+            return nodeid

src/xdist/scheduler/loadscope.py

@@ -243,6 +243,9 @@ class LoadScopeScheduling:
         self.assigned_work[node][scope][nodeid] = True
         self._reschedule(node)
 
+    def mark_test_pending(self, item):
+        raise NotImplementedError()
+
     def _assign_work_unit(self, node):
         """Assign a work unit to a node."""
         assert self.workqueue

src/xdist/workermanage.py

@@ -9,6 +9,7 @@ import pytest
 import execnet
 
 import xdist.remote
+from xdist.plugin import _sys_path
 
 
 def parse_spec_config(config):
@@ -117,8 +118,8 @@ class NodeManager:
     def _getrsyncoptions(self):
         """Get options to be passed for rsync."""
         ignores = list(self.DEFAULT_IGNORES)
-        ignores += self.config.option.rsyncignore
+        ignores += [str(path) for path in self.config.option.rsyncignore]
-        ignores += self.config.getini("rsyncignore")
+        ignores += [str(path) for path in self.config.getini("rsyncignore")]
 
         return {
             "ignores": ignores,
@@ -157,8 +158,7 @@ class NodeManager:
 
 
 class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
+    """RSyncer that filters out common files"""
-    """
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
@@ -212,7 +212,7 @@ class WorkerController:
     ENDMARK = -1
 
     class RemoteHook:
-        @pytest.mark.trylast
+        @pytest.hookimpl(trylast=True)
         def pytest_xdist_getremotemodule(self):
             return xdist.remote
 
@@ -254,15 +254,16 @@ class WorkerController:
             args = make_reltoroot(self.nodemanager.roots, args)
         if spec.popen:
             name = "popen-%s" % self.gateway.id
-            if hasattr(self.config, "_tmpdirhandler"):
+            if hasattr(self.config, "_tmp_path_factory"):
-                basetemp = self.config._tmpdirhandler.getbasetemp()
+                basetemp = self.config._tmp_path_factory.getbasetemp()
-                option_dict["basetemp"] = str(basetemp.join(name))
+                option_dict["basetemp"] = str(basetemp / name)
         self.config.hook.pytest_configure_node(node=self)
 
         remote_module = self.config.hook.pytest_xdist_getremotemodule()
         self.channel = self.gateway.remote_exec(remote_module)
         # change sys.path only for remote workers
-        change_sys_path = not self.gateway.spec.popen
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
 
         if self.putevent:
@@ -294,7 +295,7 @@ class WorkerController:
             self._shutdown_sent = True
 
     def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
+        """send a named parametrized command to the other side."""
         self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
@@ -303,12 +304,12 @@ class WorkerController:
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
-        """ this gets called for each object we receive from
+        """this gets called for each object we receive from
-            the other side and if the channel closes.
+        the other side and if the channel closes.
 
-            Note that channel callbacks run in the receiver
+        Note that channel callbacks run in the receiver
-            thread of execnet gateways - we need to
+        thread of execnet gateways - we need to
-            avoid raising exceptions or doing heavy work.
+        avoid raising exceptions or doing heavy work.
         """
         try:
             if eventcall == self.ENDMARK:
@@ -324,6 +325,8 @@ class WorkerController:
                 self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
                 self.workeroutput = kwargs["workeroutput"]
@@ -420,9 +423,9 @@ def unserialize_warning_message(data):
 
     kwargs = {"message": message, "category": category}
     # access private _WARNING_DETAILS because the attributes vary between Python versions
-    for attr_name in warnings.WarningMessage._WARNING_DETAILS:
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:  # type: ignore[attr-defined]
         if attr_name in ("message", "category"):
             continue
         kwargs[attr_name] = data[attr_name]
 
-    return warnings.WarningMessage(**kwargs)
+    return warnings.WarningMessage(**kwargs)  # type: ignore[arg-type]

testing/conftest.py

@@ -1,12 +1,13 @@
-import py
-import pytest
 import execnet
+import pytest
+import shutil
+from typing import List
 
 pytest_plugins = "pytester"
 
 
 @pytest.fixture(autouse=True)
-def _divert_atexit(request, monkeypatch):
+def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
     import atexit
 
     finalizers = []
@@ -23,7 +24,7 @@ def _divert_atexit(request, monkeypatch):
         func(*args, **kwargs)
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser) -> None:
     parser.addoption(
         "--gx",
         action="append",
@@ -33,28 +34,28 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture
-def specssh(request):
+def specssh(request) -> str:
     return getspecssh(request.config)
 
 
 # configuration information for tests
-def getgspecs(config):
+def getgspecs(config) -> List[execnet.XSpec]:
     return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
 
 
-def getspecssh(config):
+def getspecssh(config) -> str:  # type: ignore[return]
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.ssh:
-            if not py.path.local.sysfind("ssh"):
+            if not shutil.which("ssh"):
-                py.test.skip("command not found: ssh")
+                pytest.skip("command not found: ssh")
             return str(spec)
-    py.test.skip("need '--gx ssh=...'")
+    pytest.skip("need '--gx ssh=...'")
 
 
-def getsocketspec(config):
+def getsocketspec(config) -> execnet.XSpec:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.socket:
             return spec
-    py.test.skip("need '--gx socket=...'")
+    pytest.skip("need '--gx socket=...'")

testing/test_dsession.py

@@ -1,59 +1,44 @@
 from xdist.dsession import DSession, get_default_max_worker_restart
 from xdist.report import report_collection_diff
 from xdist.scheduler import EachScheduling, LoadScheduling
+from typing import Optional
 
-import py
 import pytest
 import execnet
 
-XSpec = execnet.XSpec
-
-
-def run(item, node, excinfo=None):
-    runner = item.config.pluginmanager.getplugin("runner")
-    rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call")
-    rep.node = node
-    return rep
-
 
 class MockGateway:
-    _count = 0
+    def __init__(self) -> None:
-
+        self._count = 0
-    def __init__(self):
         self.id = str(self._count)
         self._count += 1
 
 
 class MockNode:
-    def __init__(self):
+    def __init__(self) -> None:
-        self.sent = []
+        self.sent = []  # type: ignore[var-annotated]
         self.gateway = MockGateway()
         self._shutdown = False
 
-    def send_runtest_some(self, indices):
+    def send_runtest_some(self, indices) -> None:
         self.sent.extend(indices)
 
-    def send_runtest_all(self):
+    def send_runtest_all(self) -> None:
         self.sent.append("ALL")
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self._shutdown = True
 
     @property
-    def shutting_down(self):
+    def shutting_down(self) -> bool:
         return self._shutdown
 
 
-def dumpqueue(queue):
-    while queue.qsize():
-        print(queue.get())
-
-
 class TestEachScheduling:
-    def test_schedule_load_simple(self, testdir):
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
         node2 = MockNode()
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         sched = EachScheduling(config)
         sched.add_node(node1)
         sched.add_node(node2)
@@ -74,9 +59,9 @@ class TestEachScheduling:
         sched.mark_test_complete(node2, 0)
         assert sched.tests_finished
 
-    def test_schedule_remove_node(self, testdir):
+    def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
-        config = testdir.parseconfig("--tx=popen")
+        config = pytester.parseconfig("--tx=popen")
         sched = EachScheduling(config)
         sched.add_node(node1)
         collection = ["a.py::test_1"]
@@ -93,8 +78,8 @@ class TestEachScheduling:
 
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self, testdir):
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -117,8 +102,8 @@ class TestLoadScheduling:
         sched.mark_test_complete(node1, node1.sent[0])
         assert sched.tests_finished
 
-    def test_schedule_batch_size(self, testdir):
+    def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -144,8 +129,8 @@ class TestLoadScheduling:
         assert node1.sent == [0, 2, 4, 5]
         assert not sched.pending
 
-    def test_schedule_fewer_tests_than_nodes(self, testdir):
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -164,8 +149,10 @@ class TestLoadScheduling:
         assert sent3 == []
         assert not sched.pending
 
-    def test_schedule_fewer_than_two_tests_per_node(self, testdir):
+    def test_schedule_fewer_than_two_tests_per_node(
-        config = testdir.parseconfig("--tx=2*popen")
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -184,9 +171,9 @@ class TestLoadScheduling:
         assert sent3 == [2]
         assert not sched.pending
 
-    def test_add_remove_node(self, testdir):
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
         node = MockNode()
-        config = testdir.parseconfig("--tx=popen")
+        config = pytester.parseconfig("--tx=popen")
         sched = LoadScheduling(config)
         sched.add_node(node)
         collection = ["test_file.py::test_func"]
@@ -197,7 +184,7 @@ class TestLoadScheduling:
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
 
-    def test_different_tests_collected(self, testdir):
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
         """
         Test that LoadScheduling is reporting collection errors when
         different test ids are collected by workers.
@@ -215,7 +202,7 @@ class TestLoadScheduling:
                 self.reports.append(report)
 
         collect_hook = CollectHook()
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         config.pluginmanager.register(collect_hook, "collect_hook")
         node1 = MockNode()
         node2 = MockNode()
@@ -231,9 +218,9 @@ class TestLoadScheduling:
 
 
 class TestDistReporter:
-    @py.test.mark.xfail
+    @pytest.mark.xfail
-    def test_rsync_printing(self, testdir, linecomp):
+    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
-        config = testdir.parseconfig()
+        config = pytester.parseconfig()
         from _pytest.pytest_terminal import TerminalReporter
 
         rep = TerminalReporter(config, file=linecomp.stringio)
@@ -258,21 +245,21 @@ class TestDistReporter:
         # linecomp.assert_contains_lines([
         #     "*X1*popen*xyz*2.5*"
         # ])
-        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
+        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])  # type: ignore[attr-defined]
         linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
 
 
-def test_report_collection_diff_equal():
+def test_report_collection_diff_equal() -> None:
     """Test reporting of equal collections."""
     from_collection = to_collection = ["aaa", "bbb", "ccc"]
     assert report_collection_diff(from_collection, to_collection, 1, 2) is None
 
 
-def test_default_max_worker_restart():
+def test_default_max_worker_restart() -> None:
     class config:
         class option:
-            maxworkerrestart = None
+            maxworkerrestart: Optional[str] = None
-            numprocesses = 0
+            numprocesses: int = 0
 
     assert get_default_max_worker_restart(config) is None
 
@@ -286,7 +273,7 @@ def test_default_max_worker_restart():
     assert get_default_max_worker_restart(config) == 0
 
 
-def test_report_collection_diff_different():
+def test_report_collection_diff_different() -> None:
     """Test reporting of different collections."""
     from_collection = ["aaa", "bbb", "ccc", "YYY"]
     to_collection = ["aZa", "bbb", "XXX", "ccc"]
@@ -311,8 +298,8 @@ def test_report_collection_diff_different():
 
 
 @pytest.mark.xfail(reason="duplicate test ids not supported yet")
-def test_pytest_issue419(testdir):
+def test_pytest_issue419(pytester: pytest.Pytester) -> None:
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import pytest
 
@@ -321,6 +308,6 @@ def test_pytest_issue419(testdir):
             pass
     """
     )
-    reprec = testdir.inline_run("-n1")
+    reprec = pytester.inline_run("-n1")
     reprec.assertoutcome(passed=2)
     assert 0

testing/test_looponfail.py

@@ -1,90 +1,106 @@
 import py
 import pytest
-from pkg_resources import parse_version
+import shutil
+import textwrap
+from pathlib import Path
 
 from xdist.looponfail import RemoteControl
 from xdist.looponfail import StatRecorder
 
 
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
+
+
 class TestStatRecorder:
-    def test_filechange(self, tmpdir):
+    def test_filechange(self, tmp_path: Path) -> None:
-        tmp = tmpdir
+        tmp = tmp_path
-        hello = tmp.ensure("hello.py")
+        hello = tmp / "hello.py"
-        sd = StatRecorder([tmp])
+        hello.touch()
+        sd = StatRecorder([py.path.local(tmp)])
         changed = sd.check()
         assert not changed
 
-        hello.write("world")
+        hello.write_text("world")
         changed = sd.check()
         assert changed
 
-        (hello + "c").write("hello")
+        hello.with_suffix(".pyc").write_text("hello")
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp / "new.py"
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "b", "c.py").ensure()
+        tmp.joinpath("a", "b").mkdir(parents=True)
+        tmp.joinpath("a", "b", "c.py").touch()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "c.txt").ensure()
+        tmp.joinpath("a", "c.txt").touch()
         changed = sd.check()
         assert changed
         changed = sd.check()
         assert not changed
 
-        tmp.join("a").remove()
+        shutil.rmtree(str(tmp.joinpath("a")))
         changed = sd.check()
         assert changed
 
-    def test_dirchange(self, tmpdir):
+    def test_dirchange(self, tmp_path: Path) -> None:
-        tmp = tmpdir
+        tmp = tmp_path
-        tmp.ensure("dir", "hello.py")
+        tmp.joinpath("dir").mkdir()
-        sd = StatRecorder([tmp])
+        tmp.joinpath("dir", "hello.py").touch()
-        assert not sd.fil(tmp.join("dir"))
+        sd = StatRecorder([py.path.local(tmp)])
+        assert not sd.fil(py.path.local(tmp / "dir"))
 
-    def test_filechange_deletion_race(self, tmpdir, monkeypatch):
+    def test_filechange_deletion_race(
-        tmp = tmpdir
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-        sd = StatRecorder([tmp])
+    ) -> None:
+        tmp = tmp_path
+        pytmp = py.path.local(tmp)
+        sd = StatRecorder([pytmp])
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp.joinpath("new.py")
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(tmp, "visit", lambda *args: [p])
+        monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
 
         changed = sd.check()
         assert changed
 
-    def test_pycremoval(self, tmpdir):
+    def test_pycremoval(self, tmp_path: Path) -> None:
-        tmp = tmpdir
+        tmp = tmp_path
-        hello = tmp.ensure("hello.py")
+        hello = tmp / "hello.py"
-        sd = StatRecorder([tmp])
+        hello.touch()
+        sd = StatRecorder([py.path.local(tmp)])
         changed = sd.check()
         assert not changed
 
-        pycfile = hello + "c"
+        pycfile = hello.with_suffix(".pyc")
-        pycfile.ensure()
+        pycfile.touch()
-        hello.write("world")
+        hello.write_text("world")
         changed = sd.check()
         assert changed
-        assert not pycfile.check()
+        assert not pycfile.exists()
 
-    def test_waitonchange(self, tmpdir, monkeypatch):
+    def test_waitonchange(
-        tmp = tmpdir
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-        sd = StatRecorder([tmp])
+    ) -> None:
+        tmp = tmp_path
+        sd = StatRecorder([py.path.local(tmp)])
 
         ret_values = [True, False]
         monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -93,63 +109,74 @@ class TestStatRecorder:
 
 
 class TestRemoteControl:
-    def test_nofailures(self, testdir):
+    def test_nofailures(self, pytester: pytest.Pytester) -> None:
-        item = testdir.getitem("def test_func(): pass\n")
+        item = pytester.getitem("def test_func(): pass\n")
         control = RemoteControl(item.config)
         control.setup()
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failures_somewhere(self, testdir):
+    def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
-        item = testdir.getitem("def test_func():\n assert 0\n")
+        item = pytester.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
         failures = control.runsession()
         assert failures
         control.setup()
-        item.fspath.write("def test_func():\n assert 1\n")
+        item_path = item.path if PYTEST_GTE_7 else Path(item.fspath)  # type: ignore[attr-defined]
-        removepyc(item.fspath)
+        item_path.write_text("def test_func():\n assert 1\n")
+        removepyc(item_path)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failure_change(self, testdir):
+    def test_failure_change(self, pytester: pytest.Pytester) -> None:
-        modcol = testdir.getitem(
+        modcol = pytester.getitem(
-            """
+            textwrap.dedent(
-            def test_func():
+                """
-                assert 0
+                def test_func():
-        """
+                    assert 0
+                """
+            )
         )
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol.fspath.write(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)  # type: ignore[attr-defined]
-            py.code.Source(
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 1
+                def test_new():
+                    assert 0
                 """
-            def test_func():
-                assert 1
-            def test_new():
-                assert 0
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         control.loop_once()
         assert not control.failures
         control.loop_once()
         assert control.failures
         assert str(control.failures).find("test_new") != -1
 
-    def test_failure_subdir_no_init(self, testdir):
+    def test_failure_subdir_no_init(
-        modcol = testdir.getitem(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
-            """
+    ) -> None:
-            def test_func():
+        modcol = pytester.getitem(
-                assert 0
+            textwrap.dedent(
-        """
+                """
+                def test_func():
+                    assert 0
+                """
+            )
         )
-        parent = modcol.fspath.dirpath().dirpath()
+        if PYTEST_GTE_7:
-        parent.chdir()
+            parent = modcol.path.parent.parent  # type: ignore[attr-defined]
+        else:
+            parent = Path(modcol.fspath.dirpath().dirpath())
+        monkeypatch.chdir(parent)
         modcol.config.args = [
-            py.path.local(x).relto(parent) for x in modcol.config.args
+            str(Path(x).relative_to(parent)) for x in modcol.config.args
         ]
         control = RemoteControl(modcol.config)
         control.loop_once()
@@ -159,57 +186,63 @@ class TestRemoteControl:
 
 
 class TestLooponFailing:
-    def test_looponfail_from_fail_to_ok(self, testdir):
+    def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
-        modcol = testdir.getmodulecol(
+        modcol = pytester.getmodulecol(
-            """
+            textwrap.dedent(
-            def test_one():
+                """
-                x = 0
+                def test_one():
-                assert x == 1
+                    x = 0
-            def test_two():
+                    assert x == 1
-                assert 1
+                def test_two():
-        """
+                    assert 1
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-        modcol.fspath.write(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
-            py.code.Source(
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1
+                def test_two():
+                    assert 1
                 """
-            def test_one():
-                assert 1
-            def test_two():
-                assert 1
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert not remotecontrol.failures
 
-    def test_looponfail_from_one_to_two_tests(self, testdir):
+    def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
-        modcol = testdir.getmodulecol(
+        modcol = pytester.getmodulecol(
-            """
+            textwrap.dedent(
-            def test_one():
+                """
-                assert 0
+                def test_one():
-        """
+                    assert 0
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
         assert "test_one" in remotecontrol.failures[0]
 
-        modcol.fspath.write(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
-            py.code.Source(
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1 # passes now
+                def test_two():
+                    assert 0 # new and fails
                 """
-            def test_one():
-                assert 1 # passes now
-            def test_two():
-                assert 0 # new and fails
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
         remotecontrol.loop_once()
@@ -217,47 +250,49 @@ class TestLooponFailing:
         assert "test_one" not in remotecontrol.failures[0]
         assert "test_two" in remotecontrol.failures[0]
 
-    @pytest.mark.xfail(
+    @pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
-        parse_version(pytest.__version__) >= parse_version("3.1"),
+    def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
-        reason="broken by pytest 3.1+",
+        modcol = pytester.getmodulecol(
-        strict=True,
+            textwrap.dedent(
-    )
+                """
-    def test_looponfail_removed_test(self, testdir):
+                def test_one():
-        modcol = testdir.getmodulecol(
+                    assert 0
-            """
+                def test_two():
-            def test_one():
+                    assert 0
-                assert 0
+                """
-            def test_two():
+            )
-                assert 0
-        """
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 2
 
-        modcol.fspath.write(
+        modcol.path.write_text(
-            py.code.Source(
+            textwrap.dedent(
+                """
+                def test_xxx(): # renamed test
+                    assert 0
+                def test_two():
+                    assert 1 # pass now
                 """
-            def test_xxx(): # renamed test
-                assert 0
-            def test_two():
-                assert 1 # pass now
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
 
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-    def test_looponfail_multiple_errors(self, testdir, monkeypatch):
+    def test_looponfail_multiple_errors(
-        modcol = testdir.getmodulecol(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
-            """
+    ) -> None:
-            def test_one():
+        modcol = pytester.getmodulecol(
-                assert 0
+            textwrap.dedent(
-        """
+                """
+                def test_one():
+                    assert 0
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         orig_runsession = remotecontrol.runsession
@@ -274,55 +309,59 @@ class TestLooponFailing:
 
 
 class TestFunctional:
-    def test_fail_to_ok(self, testdir):
+    def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
-        p = testdir.makepyfile(
+        p = pytester.makepyfile(
-            """
+            textwrap.dedent(
-            def test_one():
+                """
-                x = 0
+                def test_one():
-                assert x == 1
+                    x = 0
-        """
+                    assert x == 1
+                """
+            )
         )
-        # p = testdir.mkdir("sub").join(p1.basename)
+        # p = pytester.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
+        child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
         child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
-        p.write(
+        p.write_text(
-            py.code.Source(
+            textwrap.dedent(
                 """
-            def test_one():
+                def test_one():
-                x = 1
+                    x = 1
-                assert x == 1
+                    assert x == 1
-        """
+                """
-            )
+            ),
         )
         child.expect(".*1 passed.*")
         child.kill(15)
 
-    def test_xfail_passes(self, testdir):
+    def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
-        p = testdir.makepyfile(
+        p = pytester.makepyfile(
-            """
+            textwrap.dedent(
-            import py
+                """
-            @py.test.mark.xfail
+                import pytest
-            def test_one():
+                @pytest.mark.xfail
-                pass
+                def test_one():
-        """
+                    pass
+                """
+            )
         )
-        child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
+        child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
         child.kill(15)
 
 
-def removepyc(path):
+def removepyc(path: Path) -> None:
     # XXX damn those pyc files
-    pyc = path + "c"
+    pyc = path.with_suffix(".pyc")
-    if pyc.check():
+    if pyc.exists():
-        pyc.remove()
+        pyc.unlink()
-    c = path.dirpath("__pycache__")
+    c = path.parent / "__pycache__"
-    if c.check():
+    if c.exists():
-        c.remove()
+        shutil.rmtree(c)

testing/test_newhooks.py

@@ -3,8 +3,8 @@ import pytest
 
 class TestHooks:
     @pytest.fixture(autouse=True)
-    def create_test_file(self, testdir):
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
-        testdir.makepyfile(
+        pytester.makepyfile(
             """
             import os
             def test_a(): pass
@@ -13,11 +13,11 @@ class TestHooks:
         """
         )
 
-    def test_runtest_logreport(self, testdir):
+    def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
         """Test that log reports from pytest_runtest_logreport when running
         with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
         """
-        testdir.makeconftest(
+        pytester.makeconftest(
             """
             def pytest_runtest_logreport(report):
                 if hasattr(report, 'node'):
@@ -35,7 +35,7 @@ class TestHooks:
                                    % (report.nodeid, report.worker_id, report.testrun_uid))
         """
         )
-        res = testdir.runpytest("-n1", "-s")
+        res = pytester.runpytest("-n1", "-s")
         res.stdout.fnmatch_lines(
             [
                 "*HOOK: test_runtest_logreport.py::test_a gw0 *",
@@ -45,10 +45,9 @@ class TestHooks:
             ]
         )
 
-    def test_node_collection_finished(self, testdir):
+    def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
-        """Test pytest_xdist_node_collection_finished hook (#8).
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
-        """
+        pytester.makeconftest(
-        testdir.makeconftest(
             """
             def pytest_xdist_node_collection_finished(node, ids):
                 workerid = node.workerinput['workerid']
@@ -56,8 +55,42 @@ class TestHooks:
                 print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
         """
         )
-        res = testdir.runpytest("-n2", "-s")
+        res = pytester.runpytest("-n2", "-s")
         res.stdout.fnmatch_lines_random(
             ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
         )
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])

testing/test_plugin.py

@@ -1,108 +1,173 @@
-import py
+from contextlib import suppress
+from pathlib import Path
+
 import execnet
 from xdist.workermanage import NodeManager
 
+import pytest
 
-def test_dist_incompatibility_messages(testdir):
+
-    result = testdir.runpytest("--pdb", "--looponfail")
+def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
+    result = pytester.runpytest("--pdb", "--looponfail")
     assert result.ret != 0
-    result = testdir.runpytest("--pdb", "-n", "3")
+    result = pytester.runpytest("--pdb", "-n", "3")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
-    result = testdir.runpytest("--pdb", "-d", "--tx", "popen")
+    result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
 
 
-def test_dist_options(testdir):
+def test_dist_options(pytester: pytest.Pytester) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = testdir.parseconfigure("-n 2")
+    config = pytester.parseconfigure("-n 2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("--numprocesses", "2")
+    config = pytester.parseconfigure("--numprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("-d")
+    config = pytester.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"
 
 
-def test_auto_detect_cpus(testdir, monkeypatch):
+def test_auto_detect_cpus(
-    import psutil
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    import os
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 99)
+    with suppress(ImportError):
+        import psutil
 
-    config = testdir.parseconfigure("-n2")
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+
+    if hasattr(os, "sched_getaffinity"):
+        monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
+    elif hasattr(os, "cpu_count"):
+        monkeypatch.setattr(os, "cpu_count", lambda: 99)
+    else:
+        import multiprocessing
+
+        monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
+
+    config = pytester.parseconfigure("-n2")
     assert config.getoption("numprocesses") == 2
 
-    config = testdir.parseconfigure("-nauto")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 99
 
-    config = testdir.parseconfigure("-nauto", "--pdb")
+    config = pytester.parseconfigure("-nauto", "--pdb")
     check_options(config)
     assert config.getoption("usepdb")
     assert config.getoption("numprocesses") == 0
     assert config.getoption("dist") == "no"
 
-    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+    config = pytester.parseconfigure("-nlogical", "--pdb")
-    config = testdir.parseconfigure("-nauto")
+    check_options(config)
-    assert config.getoption("numprocesses") == 1
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setenv("TRAVIS", "true")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
 
 
-def test_boxed_with_collect_only(testdir):
+def test_auto_detect_cpus_psutil(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = testdir.parseconfigure("-n1", "--boxed")
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_hook_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
+def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    config = pytester.parseconfigure("-n1", "--boxed")
     check_options(config)
     assert config.option.forked
 
-    config = testdir.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--collect-only")
     check_options(config)
     assert not config.option.forked
 
-    config = testdir.parseconfigure("-n1", "--boxed", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
     check_options(config)
     assert config.option.forked
 
 
-def test_dsession_with_collect_only(testdir):
+def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
     from xdist.plugin import pytest_configure as configure
 
-    config = testdir.parseconfigure("-n1")
+    config = pytester.parseconfigure("-n1")
     check_options(config)
     configure(config)
     assert config.pluginmanager.hasplugin("dsession")
 
-    config = testdir.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--collect-only")
     check_options(config)
     configure(config)
     assert not config.pluginmanager.hasplugin("dsession")
 
 
-def test_testrunuid_provided(testdir):
+def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
-    config = testdir.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
     nm = NodeManager(config)
     assert nm.testrunuid == "test123"
 
 
-def test_testrunuid_generated(testdir):
+def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
-    config = testdir.parseconfigure("--tx=popen")
+    config = pytester.parseconfigure("--tx=popen")
     nm = NodeManager(config)
     assert len(nm.testrunuid) == 32
 
 
 class TestDistOptions:
-    def test_getxspecs(self, testdir):
+    def test_getxspecs(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
+        config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
         nodemanager = NodeManager(config)
         xspecs = nodemanager._getxspecs()
         assert len(xspecs) == 2
@@ -110,39 +175,39 @@ class TestDistOptions:
         assert xspecs[0].popen
         assert xspecs[1].ssh == "xyz"
 
-    def test_xspecs_multiplied(self, testdir):
+    def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfigure("--tx=3*popen")
+        config = pytester.parseconfigure("--tx=3*popen")
         xspecs = NodeManager(config)._getxspecs()
         assert len(xspecs) == 3
         assert xspecs[1].popen
 
-    def test_getrsyncdirs(self, testdir):
+    def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfigure("--rsyncdir=" + str(testdir.tmpdir))
+        config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
         nm = NodeManager(config, specs=[execnet.XSpec("popen")])
         assert not nm._getrsyncdirs()
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert nm.roots
-        assert testdir.tmpdir in nm.roots
+        assert pytester.path in nm.roots
 
-    def test_getrsyncignore(self, testdir):
+    def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
-        config = testdir.parseconfigure("--rsyncignore=fo*")
+        config = pytester.parseconfigure("--rsyncignore=fo*")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert "fo*" in nm.rsyncoptions["ignores"]
 
-    def test_getrsyncdirs_with_conftest(self, testdir):
+    def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
-        p = py.path.local()
+        p = Path.cwd()
-        for bn in "x y z".split():
+        for bn in ("x", "y", "z"):
-            p.mkdir(bn)
+            p.joinpath(bn).mkdir()
-        testdir.makeini(
+        pytester.makeini(
             """
             [pytest]
             rsyncdirs= x
         """
         )
-        config = testdir.parseconfigure(testdir.tmpdir, "--rsyncdir=y", "--rsyncdir=z")
+        config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
         roots = nm._getrsyncdirs()
         # assert len(roots) == 3 + 1 # pylib
-        assert py.path.local("y") in roots
+        assert Path("y").resolve() in roots
-        assert py.path.local("z") in roots
+        assert Path("z").resolve() in roots
-        assert testdir.tmpdir.join("x") in roots
+        assert pytester.path.joinpath("x") in roots

testing/test_remote.py

@@ -1,5 +1,5 @@
-import py
 import pprint
+import py
 import pytest
 import sys
 import uuid
@@ -32,16 +32,16 @@ class EventCall:
 class WorkerSetup:
     use_callback = False
 
-    def __init__(self, request, testdir):
+    def __init__(self, request, pytester: pytest.Pytester) -> None:
         self.request = request
-        self.testdir = testdir
+        self.pytester = pytester
-        self.events = Queue()
+        self.events = Queue()  # type: ignore[var-annotated]
 
-    def setup(self,):
+    def setup(self) -> None:
-        self.testdir.chdir()
+        self.pytester.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
         self.gateway = execnet.makegateway()
-        self.config = config = self.testdir.parseconfigure()
+        self.config = config = self.pytester.parseconfigure()
         putevent = self.use_callback and self.events.put or None
 
         class DummyMananger:
@@ -68,15 +68,15 @@ class WorkerSetup:
 
 
 @pytest.fixture
-def worker(request, testdir):
+def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
-    return WorkerSetup(request, testdir)
+    return WorkerSetup(request, pytester)
 
 
 @pytest.mark.xfail(reason="#59")
-def test_remoteinitconfig(testdir):
+def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
     from xdist.remote import remote_initconfig
 
-    config1 = testdir.parseconfig()
+    config1 = pytester.parseconfig()
     config2 = remote_initconfig(config1.option.__dict__, config1.args)
     assert config2.option.__dict__ == config1.option.__dict__
     assert config2.pluginmanager.getplugin("terminal") in (-1, None)
@@ -92,8 +92,10 @@ class TestWorkerInteractor:
 
         return unserialize
 
-    def test_basic_collect_and_runtests(self, worker, unserialize_report):
+    def test_basic_collect_and_runtests(
-        worker.testdir.makepyfile(
+        self, worker: WorkerSetup, unserialize_report
+    ) -> None:
+        worker.pytester.makepyfile(
             """
             def test_func():
                 pass
@@ -106,7 +108,7 @@ class TestWorkerInteractor:
         assert ev.name == "collectionstart"
         assert not ev.kwargs
         ev = worker.popevent("collectionfinish")
-        assert ev.kwargs["topdir"] == worker.testdir.tmpdir
+        assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
         ids = ev.kwargs["ids"]
         assert len(ids) == 1
         worker.sendcommand("runtests", indices=list(range(len(ids))))
@@ -124,8 +126,8 @@ class TestWorkerInteractor:
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, worker, unserialize_report):
+    def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
-        worker.testdir.makepyfile(
+        worker.pytester.makepyfile(
             """
             import pytest
             pytest.skip("hello", allow_module_level=True)
@@ -142,8 +144,8 @@ class TestWorkerInteractor:
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, worker, unserialize_report):
+    def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
-        worker.testdir.makepyfile("""aasd qwe""")
+        worker.pytester.makepyfile("""aasd qwe""")
         worker.setup()
         ev = worker.popevent("collectionstart")
         assert not ev.kwargs
@@ -154,8 +156,8 @@ class TestWorkerInteractor:
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, worker, unserialize_report):
+    def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
-        worker.testdir.makepyfile(
+        worker.pytester.makepyfile(
             """
             def test_func(): pass
             def test_func2(): pass
@@ -181,17 +183,19 @@ class TestWorkerInteractor:
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, worker):
+    def test_happy_run_events_converted(
-        py.test.xfail("implement a simple test for event production")
+        self, pytester: pytest.Pytester, worker: WorkerSetup
-        assert not worker.use_callback
+    ) -> None:
-        worker.testdir.makepyfile(
+        pytest.xfail("implement a simple test for event production")
+        assert not worker.use_callback  # type: ignore[unreachable]
+        worker.pytester.makepyfile(
             """
             def test_func():
                 pass
         """
         )
         worker.setup()
-        hookrec = testdir.getreportrecorder(worker.config)
+        hookrec = pytester.getreportrecorder(worker.config)
         for data in worker.slp.channel:
             worker.slp.process_from_remote(data)
         worker.slp.process_from_remote(worker.slp.ENDMARK)
@@ -207,7 +211,9 @@ class TestWorkerInteractor:
             ]
         )
 
-    def test_process_from_remote_error_handling(self, worker, capsys):
+    def test_process_from_remote_error_handling(
+        self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
+    ) -> None:
         worker.use_callback = True
         worker.setup()
         worker.slp.process_from_remote(("<nonono>", ()))
@@ -217,8 +223,8 @@ class TestWorkerInteractor:
         assert ev.name == "errordown"
 
 
-def test_remote_env_vars(testdir):
+def test_remote_env_vars(pytester: pytest.Pytester) -> None:
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import os
         def test():
@@ -227,13 +233,13 @@ def test_remote_env_vars(testdir):
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     """
     )
-    result = testdir.runpytest("-n2", "--max-worker-restart=0")
+    result = pytester.runpytest("-n2", "--max-worker-restart=0")
     assert result.ret == 0
 
 
-def test_remote_inner_argv(testdir):
+def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
     """Test/document the behavior due to execnet using `python -c`."""
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import sys
 
@@ -241,14 +247,14 @@ def test_remote_inner_argv(testdir):
             assert sys.argv == ["-c"]
         """
     )
-    result = testdir.runpytest("-n1")
+    result = pytester.runpytest("-n1")
     assert result.ret == 0
 
 
-def test_remote_mainargv(testdir):
+def test_remote_mainargv(pytester: pytest.Pytester) -> None:
     outer_argv = sys.argv
 
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         def test_mainargv(request):
             assert request.config.workerinput["mainargv"] == {!r}
@@ -256,14 +262,14 @@ def test_remote_mainargv(testdir):
             outer_argv
         )
     )
-    result = testdir.runpytest("-n1")
+    result = pytester.runpytest("-n1")
     assert result.ret == 0
 
 
-def test_remote_usage_prog(testdir, request):
+def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
     if not hasattr(request.config._parser, "prog"):
         pytest.skip("prog not available in config parser")
-    testdir.makeconftest(
+    pytester.makeconftest(
         """
         import pytest
 
@@ -278,7 +284,7 @@ def test_remote_usage_prog(testdir, request):
             config_parser = config._parser
     """
     )
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import sys
 
@@ -287,6 +293,20 @@ def test_remote_usage_prog(testdir, request):
     """
     )
 
-    result = testdir.runpytest_subprocess("-n1")
+    result = pytester.runpytest_subprocess("-n1")
     assert result.ret == 1
     result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(pytester: pytest.Pytester) -> None:
+    """Work around sys.path differences due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0

testing/test_workermanage.py

@@ -1,39 +1,42 @@
+import execnet
 import py
 import pytest
+import shutil
 import textwrap
-import execnet
+from pathlib import Path
-from _pytest.pytester import HookRecorder
+from xdist import workermanage
-from xdist import workermanage, newhooks
 from xdist.workermanage import HostRSync, NodeManager
 
 pytest_plugins = "pytester"
 
 
 @pytest.fixture
-def hookrecorder(request, config):
+def hookrecorder(request, config, pytester: pytest.Pytester):
-    hookrecorder = HookRecorder(config.pluginmanager)
+    hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
-    if hasattr(hookrecorder, "start_recording"):
-        hookrecorder.start_recording(newhooks)
-    request.addfinalizer(hookrecorder.finish_recording)
     return hookrecorder
 
 
 @pytest.fixture
-def config(testdir):
+def config(pytester: pytest.Pytester):
-    return testdir.parseconfig()
+    return pytester.parseconfig()
 
 
 @pytest.fixture
-def mysetup(tmpdir):
+def source(tmp_path: Path) -> Path:
-    class mysetup:
+    source = tmp_path / "source"
-        source = tmpdir.mkdir("source")
+    source.mkdir()
-        dest = tmpdir.mkdir("dest")
+    return source
-
-    return mysetup()
 
 
 @pytest.fixture
-def workercontroller(monkeypatch):
+def dest(tmp_path: Path) -> Path:
+    dest = tmp_path / "dest"
+    dest.mkdir()
+    return dest
+
+
+@pytest.fixture
+def workercontroller(monkeypatch: pytest.MonkeyPatch):
     class MockController:
         def __init__(self, *args):
             pass
@@ -46,18 +49,20 @@ def workercontroller(monkeypatch):
 
 
 class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, config):
+    def test_popen_no_default_chdir(self, config) -> None:
         gm = NodeManager(config, ["popen"])
         assert gm.specs[0].chdir is None
 
-    def test_default_chdir(self, config):
+    def test_default_chdir(self, config) -> None:
         specs = ["ssh=noco", "socket=xyz"]
         for spec in NodeManager(config, specs).specs:
             assert spec.chdir == "pyexecnetcache"
         for spec in NodeManager(config, specs, defaultchdir="abc").specs:
             assert spec.chdir == "abc"
 
-    def test_popen_makegateway_events(self, config, hookrecorder, workercontroller):
+    def test_popen_makegateway_events(
+        self, config, hookrecorder, workercontroller
+    ) -> None:
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
         call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -72,15 +77,16 @@ class TestNodeManagerPopen:
         hm.teardown_nodes()
         assert not len(hm.group)
 
-    def test_popens_rsync(self, config, mysetup, workercontroller):
+    def test_popens_rsync(
-        source = mysetup.source
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
         assert len(hm.group) == 2
         for gw in hm.group:
 
             class pseudoexec:
-                args = []
+                args = []  # type: ignore[var-annotated]
 
                 def __init__(self, *args):
                     self.args.extend(args)
@@ -97,30 +103,37 @@ class TestNodeManagerPopen:
         assert not len(hm.group)
         assert "sys.path.insert" in gw.remote_exec.args[0]
 
-    def test_rsync_popen_with_path(self, config, mysetup, workercontroller):
+    def test_rsync_popen_with_path(
-        source, dest = mysetup.source, mysetup.dest
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
         hm.setup_nodes(None)
-        source.ensure("dir1", "dir2", "hello")
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
         notifications = []
         for gw in hm.group:
             hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
         assert len(notifications) == 1
         assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
         hm.teardown_nodes()
-        dest = dest.join(source.basename)
+        dest = dest.joinpath(source.name)
-        assert dest.join("dir1").check()
+        assert dest.joinpath("dir1").exists()
-        assert dest.join("dir1", "dir2").check()
+        assert dest.joinpath("dir1", "dir2").exists()
-        assert dest.join("dir1", "dir2", "hello").check()
+        assert dest.joinpath("dir1", "dir2", "hello").exists()
 
     def test_rsync_same_popen_twice(
-        self, config, mysetup, hookrecorder, workercontroller
+        self,
-    ):
+        config,
-        source, dest = mysetup.source, mysetup.dest
+        source: Path,
+        dest: Path,
+        hookrecorder,
+        workercontroller,
+    ) -> None:
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
         hm.roots = []
         hm.setup_nodes(None)
-        source.ensure("dir1", "dir2", "hello")
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
         gw = hm.group[0]
         hm.rsync(gw, source)
         call = hookrecorder.popcall("pytest_xdist_rsyncstart")
@@ -131,83 +144,98 @@ class TestNodeManagerPopen:
 
 
 class TestHRSync:
-    def test_hrsync_filter(self, mysetup):
+    def test_hrsync_filter(self, source: Path, dest: Path) -> None:
-        source, _ = mysetup.source, mysetup.dest  # noqa
+        source.joinpath("dir").mkdir()
-        source.ensure("dir", "file.txt")
+        source.joinpath("dir", "file.txt").touch()
-        source.ensure(".svn", "entries")
+        source.joinpath(".svn").mkdir()
-        source.ensure(".somedotfile", "moreentries")
+        source.joinpath(".svn", "entries").touch()
-        source.ensure("somedir", "editfile~")
+        source.joinpath(".somedotfile").mkdir()
+        source.joinpath(".somedotfile", "moreentries").touch()
+        source.joinpath("somedir").mkdir()
+        source.joinpath("somedir", "editfile~").touch()
         syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        files = list(source.visit(rec=syncer.filter, fil=syncer.filter))
+        files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
         assert len(files) == 3
         basenames = [x.basename for x in files]
         assert "dir" in basenames
         assert "file.txt" in basenames
         assert "somedir" in basenames
 
-    def test_hrsync_one_host(self, mysetup):
+    def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
-        source, dest = mysetup.source, mysetup.dest
         gw = execnet.makegateway("popen//chdir=%s" % dest)
         finished = []
         rsync = HostRSync(source)
         rsync.add_target_host(gw, finished=lambda: finished.append(1))
-        source.join("hello.py").write("world")
+        source.joinpath("hello.py").write_text("world")
         rsync.send()
         gw.exit()
-        assert dest.join(source.basename, "hello.py").check()
+        assert dest.joinpath(source.name, "hello.py").exists()
         assert len(finished) == 1
 
 
 class TestNodeManager:
-    @py.test.mark.xfail(run=False)
+    @pytest.mark.xfail(run=False)
-    def test_rsync_roots_no_roots(self, testdir, mysetup):
+    def test_rsync_roots_no_roots(
-        mysetup.source.ensure("dir1", "file1").write("hello")
+        self, pytester: pytest.Pytester, source: Path, dest: Path
-        config = testdir.parseconfig(mysetup.source)
+    ) -> None:
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
+        source.joinpath("dir1").mkdir()
+        source.joinpath("dir1", "file1").write_text("hello")
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         # assert nodemanager.config.topdir == source == config.topdir
-        nodemanager.makegateways()
+        nodemanager.makegateways()  # type: ignore[attr-defined]
-        nodemanager.rsync_roots()
+        nodemanager.rsync_roots()  # type: ignore[call-arg]
-        (p,) = nodemanager.gwmanager.multi_exec(
+        (p,) = nodemanager.gwmanager.multi_exec(  # type: ignore[attr-defined]
             "import os ; channel.send(os.getcwd())"
         ).receive_each()
-        p = py.path.local(p)
+        p = Path(p)
         print("remote curdir", p)
-        assert p == mysetup.dest.join(config.topdir.basename)
+        assert p == dest.joinpath(config.rootpath.name)
-        assert p.join("dir1").check()
+        assert p.joinpath("dir1").check()
-        assert p.join("dir1", "file1").check()
+        assert p.joinpath("dir1", "file1").check()
 
-    def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller):
+    def test_popen_rsync_subdir(
-        source, dest = mysetup.source, mysetup.dest
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
-        dir1 = mysetup.source.mkdir("dir1")
+    ) -> None:
-        dir2 = dir1.mkdir("dir2")
+        dir1 = source / "dir1"
-        dir2.ensure("hello")
+        dir1.mkdir()
+        dir2 = dir1 / "dir2"
+        dir2.mkdir()
+        dir2.joinpath("hello").touch()
         for rsyncroot in (dir1, source):
-            dest.remove()
+            shutil.rmtree(str(dest), ignore_errors=True)
             nodemanager = NodeManager(
-                testdir.parseconfig(
+                pytester.parseconfig(
                     "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
                 )
             )
             nodemanager.setup_nodes(None)  # calls .rsync_roots()
             if rsyncroot == source:
-                dest = dest.join("source")
+                dest = dest.joinpath("source")
-            assert dest.join("dir1").check()
+            assert dest.joinpath("dir1").exists()
-            assert dest.join("dir1", "dir2").check()
+            assert dest.joinpath("dir1", "dir2").exists()
-            assert dest.join("dir1", "dir2", "hello").check()
+            assert dest.joinpath("dir1", "dir2", "hello").exists()
             nodemanager.teardown_nodes()
 
     @pytest.mark.parametrize(
         "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
     )
     def test_rsync_report(
-        self, testdir, mysetup, workercontroller, capsys, flag, expects_report
+        self,
-    ):
+        pytester: pytest.Pytester,
-        source, dest = mysetup.source, mysetup.dest
+        source: Path,
-        dir1 = mysetup.source.mkdir("dir1")
+        dest: Path,
-        args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source
+        workercontroller,
+        capsys: pytest.CaptureFixture[str],
+        flag: str,
+        expects_report: bool,
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
         if flag:
-            args += (flag,)
+            args.append(flag)
-        nodemanager = NodeManager(testdir.parseconfig(*args))
+        nodemanager = NodeManager(pytester.parseconfig(*args))
         nodemanager.setup_nodes(None)  # calls .rsync_roots()
         out, _ = capsys.readouterr()
         if expects_report:
@@ -215,77 +243,86 @@ class TestNodeManager:
         else:
             assert "<= pytest/__init__.py" not in out
 
-    def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
+    def test_init_rsync_roots(
-        source, dest = mysetup.source, mysetup.dest
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
-        dir2 = source.ensure("dir1", "dir2", dir=1)
+    ) -> None:
-        source.ensure("dir1", "somefile", dir=1)
+        dir2 = source.joinpath("dir1", "dir2")
-        dir2.ensure("hello")
+        dir2.mkdir(parents=True)
-        source.ensure("bogusdir", "file")
+        source.joinpath("dir1", "somefile").mkdir()
-        source.join("tox.ini").write(
+        dir2.joinpath("hello").touch()
+        source.joinpath("bogusdir").mkdir()
+        source.joinpath("bogusdir", "file").touch()
+        source.joinpath("tox.ini").write_text(
             textwrap.dedent(
                 """
-            [pytest]
+                [pytest]
-            rsyncdirs=dir1/dir2
+                rsyncdirs=dir1/dir2
-        """
+                """
             )
         )
-        config = testdir.parseconfig(source)
+        config = pytester.parseconfig(source)
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         nodemanager.setup_nodes(None)  # calls .rsync_roots()
-        assert dest.join("dir2").check()
+        assert dest.joinpath("dir2").exists()
-        assert not dest.join("dir1").check()
+        assert not dest.joinpath("dir1").exists()
-        assert not dest.join("bogus").check()
+        assert not dest.joinpath("bogus").exists()
 
-    def test_rsyncignore(self, testdir, mysetup, workercontroller):
+    def test_rsyncignore(
-        source, dest = mysetup.source, mysetup.dest
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
-        dir2 = source.ensure("dir1", "dir2", dir=1)
+    ) -> None:
-        source.ensure("dir5", "dir6", "bogus")
+        dir2 = source.joinpath("dir1", "dir2")
-        source.ensure("dir5", "file")
+        dir2.mkdir(parents=True)
-        dir2.ensure("hello")
+        source.joinpath("dir5", "dir6").mkdir(parents=True)
-        source.ensure("foo", "bar")
+        source.joinpath("dir5", "dir6", "bogus").touch()
-        source.ensure("bar", "foo")
+        source.joinpath("dir5", "file").touch()
-        source.join("tox.ini").write(
+        dir2.joinpath("hello").touch()
+        source.joinpath("foo").mkdir()
+        source.joinpath("foo", "bar").touch()
+        source.joinpath("bar").mkdir()
+        source.joinpath("bar", "foo").touch()
+        source.joinpath("tox.ini").write_text(
             textwrap.dedent(
                 """
-            [pytest]
+                [pytest]
-            rsyncdirs = dir1 dir5
+                rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6 foo*
+                rsyncignore = dir1/dir2 dir5/dir6 foo*
-        """
+                """
             )
         )
-        config = testdir.parseconfig(source)
+        config = pytester.parseconfig(source)
         config.option.rsyncignore = ["bar"]
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         nodemanager.setup_nodes(None)  # calls .rsync_roots()
-        assert dest.join("dir1").check()
+        assert dest.joinpath("dir1").exists()
-        assert not dest.join("dir1", "dir2").check()
+        assert not dest.joinpath("dir1", "dir2").exists()
-        assert dest.join("dir5", "file").check()
+        assert dest.joinpath("dir5", "file").exists()
-        assert not dest.join("dir6").check()
+        assert not dest.joinpath("dir6").exists()
-        assert not dest.join("foo").check()
+        assert not dest.joinpath("foo").exists()
-        assert not dest.join("bar").check()
+        assert not dest.joinpath("bar").exists()
 
-    def test_optimise_popen(self, testdir, mysetup, workercontroller):
+    def test_optimise_popen(
-        source = mysetup.source
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
         specs = ["popen"] * 3
-        source.join("conftest.py").write("rsyncdirs = ['a']")
+        source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
-        source.ensure("a", dir=1)
+        source.joinpath("a").mkdir()
-        config = testdir.parseconfig(source)
+        config = pytester.parseconfig(source)
         nodemanager = NodeManager(config, specs)
         nodemanager.setup_nodes(None)  # calls .rysnc_roots()
         for gwspec in nodemanager.specs:
             assert gwspec._samefilesystem()
             assert not gwspec.chdir
 
-    def test_ssh_setup_nodes(self, specssh, testdir):
+    def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
-        testdir.makepyfile(
+        pytester.makepyfile(
             __init__="",
             test_x="""
             def test_one():
                 pass
         """,
         )
-        reprec = testdir.inline_run(
+        reprec = pytester.inline_run(
-            "-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir
+            "-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
         )
         (rep,) = reprec.getreports("pytest_runtest_logreport")
         assert rep.passed

tox.ini

@@ -1,21 +1,38 @@
 [tox]
 envlist=
   linting
-  py{35,36,37,38,39}-pytestlatest
+  py{36,37,38,39,310}-pytestlatest
-  py38-pytestmaster
+  py38-pytestmain
+  py38-psutil
+  py38-setproctitle
 
 [testenv]
-passenv = USER USERNAME
 extras = testing
 deps =
   pytestlatest: pytest
-  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
 commands=
   pytest {posargs}
 
+[testenv:py38-psutil]
+extras =
+  testing
+  psutil
+commands =
+  pytest {posargs:-k psutil}
+
+[testenv:py38-setproctitle]
+extras =
+  testing
+  setproctitle
+deps = pytest
+commands =
+  pytest {posargs}
+
 [testenv:linting]
-skipsdist = True
+skip_install = True
 usedevelop = True
+passenv = PRE_COMMIT_HOME
 deps =
   pre-commit
 commands = pre-commit run --all-files --show-diff-on-failure
@@ -28,12 +45,14 @@ skipsdist = True
 usedevelop = True
 passenv = *
 deps =
-    towncrier
+  towncrier
 commands =
-    towncrier --version {posargs} --yes
+  towncrier --version {posargs} --yes
 
 [pytest]
-addopts = -ra
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
 testpaths = testing
 
 [flake8]