Remove unnecessary skip from test_logfinish_hook as we require pytest>=6.2

updates:
- [github.com/psf/black: 21.11b1 → 21.12b0](https://github.com/psf/black/compare/21.11b1...21.12b0)

.github/workflows/main.yml

@@ -0,0 +1,81 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py36-pytestlatest"
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py38-pytestmain"
+          - "py38-psutil"
+          - "py38-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py36-pytestlatest"
+            python: "3.6"
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10-dev"
+          - tox_env: "py38-pytestmain"
+            python: "3.8"
+          - tox_env: "py38-psutil"
+            python: "3.8"
+          - tox_env: "py38-setproctitle"
+            python: "3.8"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}
+
+  deploy:
+
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    runs-on: ubuntu-latest
+
+    needs: build
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.7"
+    - name: Install wheel
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.gitignore

@@ -23,7 +23,7 @@ include/
 lib/
 bin/
 env/
-xdist/_version.py*
+src/xdist/_version.py*
 pytest_xdist.egg-info
 issue/
 3rdparty/

.pre-commit-config.yaml

@@ -1,18 +1,25 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: 18.6b4
+-   repo: https://github.com/psf/black
+    rev: 21.12b0
     hooks:
     -   id: black
-        args: [--safe, --quiet]
-        language_version: python3.6
+        args: [--safe, --quiet, --target-version, py35]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+    rev: v4.0.1
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
     -   id: flake8
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.29.1
+    hooks:
+    -   id: pyupgrade
+        args: [--py3-plus]
 -   repo: local
     hooks:
     -   id: rst
@@ -21,4 +28,12 @@ repos:
         files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
-        python_version: python3.6
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910-1
+    hooks:
+    -   id: mypy
+        files: ^(src/|testing/)
+        args: []
+        additional_dependencies:
+          - pytest>=6.2.0
+          - py>=1.10.0

.travis.yml

@@ -1,67 +0,0 @@
-sudo: false
-language: python
-
-notifications:
-  irc:
-    channels:
-    - 'chat.freenode.net#pytest'
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-  - pytest-commit@python.org
-
-install:
-  - pip install -U pip
-  - pip install tox setuptools_scm
-script: tox
-
-stages:
-- baseline
-- test
-- name: deploy
-  if: repo = pytest-dev/pytest-xdist AND tag IS present
-
-jobs:
-  include:
-    - stage: baseline
-      python: '3.6'
-      env: TOXENV=linting
-    - python: '3.6'
-      env: TOXENV=py36-pytestlatest
-    - python: '2.7'
-      env: TOXENV=py27-pytestlatest
-
-    - stage: test
-      python: "3.4"
-      env: TOXENV=py34-pytestlatest
-    - python: "3.5"
-      env: TOXENV=py35-pytestlatest
-    - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-      sudo: required
-      dist: xenial
-    - python: "2.7"
-      env: TOXENV=py27-pytestmaster
-    - python: "2.7"
-      env: TOXENV=py27-pytestfeatures
-    - python: "3.6"
-      env: TOXENV=py36-pytestmaster
-    - python: "3.6"
-      env: TOXENV=py36-pytestfeatures
-
-    - stage: deploy
-      python: '3.6'
-      env:
-      install: pip install -U setuptools setuptools_scm
-      script: skip
-      deploy:
-        provider: pypi
-        user: ronny
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,274 @@
+pytest-xdist 2.5.0 (2021-12-10)
+===============================
+
+Features
+--------
+
+- `#722 <https://github.com/pytest-dev/pytest-xdist/issues/722>`_: Full compatibility with pytest 7 - no deprecation warnings or use of legacy features.
+
+- `#733 <https://github.com/pytest-dev/pytest-xdist/issues/733>`_: New ``--dist=loadgroup`` option, which ensures all tests marked with ``@pytest.mark.xdist_group`` run in the same session/worker. Other tests run distributed as in ``--dist=load``.
+
+
+Trivial Changes
+---------------
+
+- `#708 <https://github.com/pytest-dev/pytest-xdist/issues/708>`_: Use ``@pytest.hookspec`` decorator to declare hook options in ``newhooks.py`` to avoid warnings in ``pytest 7.0``.
+
+- `#719 <https://github.com/pytest-dev/pytest-xdist/issues/719>`_: Use up-to-date ``setup.cfg``/``pyproject.toml`` packaging setup.
+
+- `#720 <https://github.com/pytest-dev/pytest-xdist/issues/720>`_: Require pytest>=6.2.0.
+
+- `#721 <https://github.com/pytest-dev/pytest-xdist/issues/721>`_: Started using type annotations and mypy checking internally. The types are incomplete and not published.
+
+
+pytest-xdist 2.4.0 (2021-09-20)
+===============================
+
+Features
+--------
+
+- `#696 <https://github.com/pytest-dev/pytest-xdist/issues/696>`_: On Linux, the process title now changes to indicate the current worker state (running/idle).
+
+  Depends on the `setproctitle <https://pypi.org/project/setproctitle/>`__ package, which can be installed with ``pip install pytest-xdist[setproctitle]``.
+
+- `#704 <https://github.com/pytest-dev/pytest-xdist/issues/704>`_: Add support for Python 3.10.
+
+
+pytest-xdist 2.3.0 (2021-06-16)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#654 <https://github.com/pytest-dev/pytest-xdist/issues/654>`_: Python 3.5 is no longer supported.
+
+
+Features
+--------
+
+- `#646 <https://github.com/pytest-dev/pytest-xdist/issues/646>`_: Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+  This is very useful for test suites which are not CPU-bound.
+
+- `#650 <https://github.com/pytest-dev/pytest-xdist/issues/650>`_: Added new ``pytest_handlecrashitem`` hook to allow handling and rescheduling crashed items.
+
+
+Bug Fixes
+---------
+
+- `#421 <https://github.com/pytest-dev/pytest-xdist/issues/421>`_: Copy the parent process sys.path into local workers, to work around execnet's python -c adding the current directory to sys.path.
+
+- `#638 <https://github.com/pytest-dev/pytest-xdist/issues/638>`_: Fix issue caused by changing the branch name of the pytest repository.
+
+
+Trivial Changes
+---------------
+
+- `#592 <https://github.com/pytest-dev/pytest-xdist/issues/592>`_: Replace master with controller where ever possible.
+
+- `#643 <https://github.com/pytest-dev/pytest-xdist/issues/643>`_: Use 'main' to refer to pytest default branch in tox env names.
+
+
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
+pytest-xdist 2.0.0 (2020-08-12)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: Drop backward-compatibility "slave" aliases related to worker nodes.  We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.
+
+- `#569 <https://github.com/pytest-dev/pytest-xdist/issues/569>`_: ``pytest-xdist`` no longer supports Python 2.7.
+
+
+Features
+--------
+
+- `#504 <https://github.com/pytest-dev/pytest-xdist/issues/504>`_: New functions ``xdist.is_xdist_worker``, ``xdist.is_xdist_master``, ``xdist.get_xdist_worker_id``, to easily identify the current node.
+
+
+Bug Fixes
+---------
+
+- `#471 <https://github.com/pytest-dev/pytest-xdist/issues/471>`_: Fix issue with Rsync reporting in quiet mode.
+
+- `#553 <https://github.com/pytest-dev/pytest-xdist/issues/553>`_: When using ``-n auto``, count the number of physical CPU cores instead of logical ones.
+
+
+Trivial Changes
+---------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: ``pytest-xdist`` now requires ``pytest>=6.0``.
+
+
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
+pytest-xdist 1.32.0 (2020-05-03)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- `#475 <https://github.com/pytest-dev/pytest-xdist/issues/475>`_: Drop support for EOL Python 3.4.
+
+
+Features
+--------
+
+- `#524 <https://github.com/pytest-dev/pytest-xdist/issues/524>`_: Add `testrun_uid` fixture. This is a shared value that uniquely identifies a test run among all workers.
+  This also adds a `PYTEST_XDIST_TESTRUNUID` environment variable that is accessible within a test as well as a command line option `--testrunuid` to manually set the value from outside.
+
+
+pytest-xdist 1.31.0 (2019-12-19)
+================================
+
+Features
+--------
+
+- `#486 <https://github.com/pytest-dev/pytest-xdist/issues/486>`_: Add support for Python 3.8.
+
+
+Bug Fixes
+---------
+
+- `#491 <https://github.com/pytest-dev/pytest-xdist/issues/491>`_: Fix regression that caused custom plugin command-line arguments to be discarded when using ``--tx`` mode.
+
+
+
+pytest-xdist 1.30.0 (2019-10-01)
+================================
+
+Features
+--------
+
+- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
+  long-standing issues related to startup with the ``-c`` option.
+
+  Issues:
+
+  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
+  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
+
+  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
+
+
+Bug Fixes
+---------
+
+- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
+
+
+pytest-xdist 1.29.0 (2019-06-14)
+================================
+
+Features
+--------
+
+- `#226 <https://github.com/pytest-dev/pytest-xdist/issues/226>`_: ``--max-worker-restart`` now assumes a more reasonable value (4 times the number of
+  nodes) when not given explicitly. This prevents test suites from running forever when the suite crashes during collection.
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: When the test session is interrupted due to running out of workers, the reason is shown in the test summary
+  for easier viewing.
+
+- `#442 <https://github.com/pytest-dev/pytest-xdist/issues/442>`_: Compatibility fix for upcoming pytest 5.0: ``session.exitstatus`` is now an ``IntEnum`` object.
+
+
+Bug Fixes
+---------
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: No longer show an internal error when we run out of workers due to crashes.
+
+
+pytest-xdist 1.28.0 (2019-04-02)
+================================
+
+Features
+--------
+
+- `#426 <https://github.com/pytest-dev/pytest-xdist/issues/426>`_: ``pytest-xdist`` now uses the new ``pytest_report_to_serializable`` and ``pytest_report_from_serializable``
+  hooks from ``pytest 4.4`` (still experimental). This will make report serialization more reliable and
+  extensible.
+
+  This also means that ``pytest-xdist`` now requires ``pytest>=4.4``.
+
+
+pytest-xdist 1.27.0 (2019-02-15)
+================================
+
+Features
+--------
+
+- `#374 <https://github.com/pytest-dev/pytest-xdist/issues/374>`_: The new ``pytest_xdist_getremotemodule`` hook allows overriding the module run on remote nodes.
+
+- `#415 <https://github.com/pytest-dev/pytest-xdist/issues/415>`_: Improve behavior of ``--numprocesses=auto`` to work well with ``--pdb`` option.
+
+
 pytest-xdist 1.26.1 (2019-01-28)
 ================================
 
@@ -356,17 +627,6 @@ Improved Documentation
 - Added ``HOWTORELEASE`` documentation. (#155)
 
 
-..
-    You should *NOT* be adding new change log entries to this file, this
-    file is managed by towncrier. You *may* edit previous change logs to
-    fix problems like typo corrections or such.
-    To add a new change log entry, please see
-    https://pip.pypa.io/en/latest/development/#adding-a-news-entry
-    We named the news folder ``changelog``
-
-.. towncrier release notes start
-
-
 1.17.0
 ------
 

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no pytest plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

OVERVIEW.md

@@ -1,76 +0,0 @@
-# Overview #
-
-`xdist` works by spawning one or more **workers**, which are controlled
-by the **master**. Each **worker** is responsible for performing
-a full test collection and afterwards running tests as dictated by the **master**.
-
-The execution flow is:
-
-1. **master** spawns one or more **workers** at the beginning of
-   the test session. The communication between **master** and **worker** nodes makes use of
-   [execnet](http://codespeak.net/execnet/) and its [gateways](http://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters).
-   The actual interpreters executing the code for the **workers** might
-   be remote or local.
-
-1. Each **worker** itself is a mini pytest runner. **workers** at this
-   point perform a full test collection, sending back the collected
-   test-ids back to the **master** which does not
-   perform any collection itself.
-
-1. The **master** receives the result of the collection from all nodes.
-   At this point the **master** performs some sanity check to ensure that
-   all **workers** collected the same tests (including order), bailing out otherwise.
-   If all is well, it converts the list of test-ids into a list of simple
-   indexes, where each index corresponds to the position of that test in the
-   original collection list. This works because all nodes have the same
-   collection list, and saves bandwidth because the **master** can now tell
-   one of the workers to just *execute test index 3* index of passing the
-   full test id.
-
-1. If **dist-mode** is **each**: the **master** just sends the full list
-   of test indexes to each node at this moment.
-
-1. If **dist-mode** is **load**: the **master** takes around 25% of the
-   tests and sends them one by one to each **worker** in a round robin
-   fashion. The rest of the tests will be distributed later as **workers**
-   finish tests (see below).
-
-1. Note that `pytest_xdist_make_scheduler` hook can be used to implement custom tests distribution logic.
-
-1. **workers** re-implement `pytest_runtestloop`: pytest's default implementation
-   basically loops over all collected items in the `session` object and executes
-   the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly
-   waiting for **master** to send tests for execution. As tests are
-   received by **workers**, `pytest_runtest_protocol` is executed for each test.
-   Here it worth noting an implementation detail: **workers** always must keep at
-   least one test item on their queue due to how the `pytest_runtest_protocol(item, nextitem)`
-   hook is defined: in order to pass the `nextitem` to the hook, the worker must wait for more
-   instructions from master before executing that remaining test. If it receives more tests,
-   then it can safely call `pytest_runtest_protocol` because it knows what the `nextitem` parameter will be.
-   If it receives a "shutdown" signal, then it can execute the hook passing `nextitem` as `None`.
-
-1. As tests are started and completed at the **workers**, the results are sent
-   back to the **master**, which then just forwards the results to
-   the appropriate pytest hooks: `pytest_runtest_logstart` and
-   `pytest_runtest_logreport`. This way other plugins (for example `junitxml`)
-   can work normally. The **master** (when in dist-mode **load**)
-   decides to send more tests to a node when a test completes, using
-   some heuristics such as test durations and how many tests each **worker**
-   still has to run.
-
-1. When the **master** has no more pending tests it will
-   send a "shutdown" signal to all **workers**, which will then run their
-   remaining tests to completion and shut down. At this point the
-   **master** will sit waiting for **workers** to shut down, still
-   processing events such as `pytest_runtest_logreport`.
-
-## FAQ ##
-
-> Why does each worker do its own collection, as opposed to having
-the master collect once and distribute from that collection to the workers?
-
-If collection was performed by master then it would have to
-serialize collected items to send them through the wire, as workers live in another process.
-The problem is that test items are not easily (impossible?) to serialize, as they contain references to
-the test functions, fixture managers, config objects, etc. Even if one manages to serialize it,
-it seems it would be very hard to get it right and easy to break by any small change in pytest.

README.rst

@@ -1,4 +1,6 @@
-
+============
+pytest-xdist
+============
 
 .. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
     :alt: PyPI version
@@ -11,27 +13,42 @@
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
-    :alt: Travis CI build status
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
-
-.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
-    :alt: AppVeyor build status
-    :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
 
-xdist: pytest distributed testing plugin
-========================================
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
 
-The `pytest-xdist`_ plugin extends pytest with some unique
-test execution modes:
+    pytest -n auto
 
-* test run parallelization_: if you have multiple CPUs or hosts you can use
-  those for a combined test run.  This allows to speed up
-  development or to use special resources of `remote machines`_.
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them. There is also a number of `distribution modes`_ to choose from.
 
+**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+.. contents:: **Table of Contents**
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+
+Features
+--------
+
+* Test run parallelization_: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of `remote machines`_.
 
 * ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
   pytest waits until a file in your project changes and then re-runs
@@ -41,73 +58,79 @@ test execution modes:
 * `Multi-Platform`_ coverage: you can specify different Python interpreters
   or different platforms and run tests in parallel on all of them.
 
-Before running tests remotely, ``pytest`` efficiently "rsyncs" your
-program source code to the remote place.  All test results
-are reported back and displayed to your local terminal.
-You may specify different Python versions and interpreters.
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  program source code to the remote place.
+  You may specify different Python versions and interpreters. It does not
+  installs/synchronize dependencies however.
 
-If you would like to know how pytest-xdist works under the covers, checkout
-`OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
-
-
-Installation
-------------
-
-Install the plugin with::
-
-    pip install pytest-xdist
-
-or use the package in develop/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
-
-    pip install --editable .
+  **Note**: this mode exists mostly for backward compatibility, as modern development
+  relies on continuous integration for multi-platform testing.
 
 .. _parallelization:
 
-Speed up test runs by sending tests to multiple CPUs
-----------------------------------------------------
+Running tests across multiple CPUs
+----------------------------------
 
-To send tests to multiple CPUs, type::
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
 
-    pytest -n NUM
+    pytest -n 8
 
-Especially for longer running tests or tests requiring
-a lot of I/O this can lead to considerable speed ups. This option can
-also be set to ``auto`` for automatic detection of the number of CPUs.
+Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
+can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
 
-If a test crashes the interpreter, pytest-xdist will automatically restart
-that worker and report the failure as usual. You can use the
-``--max-worker-restart`` option to limit the number of workers that can
-be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+The test distribution algorithm is configured with the ``--dist`` command-line option:
 
-By default, the ``-n`` option will send pending tests to any worker that is available, without
-any guaranteed order, but you can control this with these options:
+.. _distribution modes:
 
-* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
-  by **class** for *test methods*, then each group will be sent to an available worker,
-  guaranteeing that all tests in a group run in the same process. This can be useful if you have
-  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
-  with grouping by class takes priority over grouping by module.
-  This feature was added in version ``1.19``.
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order.
 
-* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
-  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
-  in version ``1.21``.
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
+
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker.
+
+  .. code-block:: python
+
+      @pytest.mark.xdist_group(name="group1")
+      def test1():
+          pass
+
+      class TestA:
+          @pytest.mark.xdist_group("group1")
+          def test2():
+              pass
+
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
+
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).
 
 
 Running tests in a Python subprocess
 ------------------------------------
 
-To instantiate a python3.5 subprocess and send tests to it, you may type::
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
 
-    pytest -d --tx popen//python=python3.5
+    pytest -d --tx popen//python=python3.9
 
-This will start a subprocess which is run with the ``python3.5``
+This will start a subprocess which is run with the ``python3.9``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python3.5
+    --tx 3*popen//python=python3.9
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -189,14 +212,27 @@ environment this command will send each tests to all
 platforms - and report back failures from all platforms
 at once. The specifications strings use the `xspec syntax`_.
 
-.. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
 
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
 
-.. _`execnet`: http://codespeak.net/execnet
+.. _`execnet`: https://codespeak.net/execnet
+
+
+When tests crash
+----------------
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.
+
+
+How-tos
+-------
 
 Identifying the worker process during a test
---------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 *New in version 1.15.*
 
@@ -213,7 +249,7 @@ a test or fixture, you may use the ``worker_id`` fixture to do so:
 When ``xdist`` is disabled (running with ``-n0`` for example), then
 ``worker_id`` will return ``"master"``.
 
-Additionally, worker processes have the following environment variables
+Worker processes also have the following environment variables
 defined:
 
 * ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
@@ -223,15 +259,102 @@ defined:
 The information about the worker_id in a test is stored in the ``TestReport`` as
 well, under the ``worker_id`` attribute.
 
-Acessing ``sys.argv`` from the master node in workers
------------------------------------------------------
+Since version 2.0, the following functions are also available in the ``xdist`` module:
 
-To access the ``sys.argv`` passed to the command-line of the master node, use
+.. code-block:: python
+
+    def is_xdist_worker(request_or_session) -> bool:
+        """Return `True` if this is an xdist worker, `False` otherwise
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+     def is_xdist_controller(request_or_session) -> bool:
+        """Return `True` if this is the xdist controller, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def is_xdist_master(request_or_session) -> bool:
+        """Deprecated alias for is_xdist_controller."""
+
+    def get_xdist_worker_id(request_or_session) -> str:
+        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+        if running on the controller node.
+
+        If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+        also return 'master'.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
+Uniquely identifying the current test run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """ create a unique database for this particular test run """
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """ retrieve unique database """
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+* ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
+
+Accessing ``sys.argv`` from the controller node in workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the ``sys.argv`` passed to the command-line of the controller node, use
 ``request.config.workerinput["mainargv"]``.
 
 
 Specifying test exec environments in an ini file
-------------------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 You can use pytest's ini file configuration to avoid typing common options.
 You can for example make running with three subprocesses your default like this:
@@ -246,7 +369,7 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
 
 and then just type::
 
@@ -256,7 +379,7 @@ to run tests in each of the environments.
 
 
 Specifying "rsync" dirs in an ini-file
---------------------------------------
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
 you may specify directories to include or to exclude in synchronisation:
@@ -273,3 +396,149 @@ where the configuration file was found.
 .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
 .. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
 .. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+How does xdist work?
+--------------------
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* index of passing the full test
+   id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

RELEASING.rst

@@ -41,3 +41,5 @@ To publish a new release ``X.Y.Z``, the steps are as follows:
     $ git push git@github.com:pytest-dev/pytest-xdist.git v$VERSION
 
 That will build the package and publish it on ``PyPI`` automatically.
+
+#. Merge the release PR to `master`.

appveyor.yml

@@ -1,24 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: "py27-pytestlatest"
-  - TOXENV: "py34-pytestlatest"
-  - TOXENV: "py35-pytestlatest"
-  - TOXENV: "py36-pytestlatest"
-  - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py27-pytestmaster"
-  - TOXENV: "py36-pytestmaster"
-  - TOXENV: "py27-pytestfeatures"
-  - TOXENV: "py36-pytestfeatures"
-
-install:
-  - C:\Python37\python -m pip install -U pip
-  - C:\Python37\python -m pip install -U tox setuptools_scm
-
-build: false  # Not a C# project, build stuff at the test step instead.
-
-test_script:
-  - C:\Python37\python -m tox
-
-# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
-# might as well save resources
-skip_tags: true

changelog/468.deprecation.rst

@@ -0,0 +1,3 @@
+The ``--boxed`` commmand line argument is deprecated.
+Install pytest-forked and use ``--forked`` instead.
+pytest-xdist 3.0.0 will remove the ``--boxed`` argument and pytest-forked dependency.

example/boxed.txt

@@ -1,9 +1,9 @@
-.. note::
+.. warning::
 
   Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a
   separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_
-  which can be installed independently. The ``--boxed`` command-line options remains
-  for backward compatibility reasons.
+  which can be installed independently. The ``--boxed`` command-line option is deprecated
+  and will be removed in pytest-xdist 3.0.0; use ``--forked`` from pytest-forked instead.
 
 
 If your testing involves C or C++ libraries you might have to deal

pyproject.toml

@@ -1,3 +1,15 @@
+[build-system]
+requires = [
+  # sync with setup.py until we discard non-pep-517/518
+  "setuptools>=45.0",
+  "setuptools-scm[toml]>=6.2.3",
+  "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+write_to = "src/xdist/_version.py"
+
 [tool.towncrier]
 package = "xdist"
 filename = "CHANGELOG.rst"

setup.cfg

@@ -1,8 +1,76 @@
-[bdist_wheel]
-universal = 1
-
 [metadata]
+name = pytest-xdist
+description = pytest xdist plugin for distributed testing and loop-on-failing modes
+long_description = file: README.rst
+license = MIT
+author = holger krekel and contributors
+author_email = pytest-dev@python.org,holger@merlinux.eu
+url = https://github.com/pytest-dev/pytest-xdist
+platforms =
+    linux
+    osx
+    win32
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Framework :: Pytest
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Operating System :: POSIX
+    Operating System :: Microsoft :: Windows
+    Operating System :: MacOS :: MacOS X
+    Topic :: Software Development :: Testing
+    Topic :: Software Development :: Quality Assurance
+    Topic :: Utilities
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
 license_file = LICENSE
 
+[options]
+packages = find:
+package_dir = =src
+zip_safe = False
+python_requires = >=3.6
+install_requires =
+    execnet>=1.1
+    pytest>=6.2.0
+    pytest-forked
+setup_requires = setuptools_scm>=6.0
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+pytest11 =
+    xdist = xdist.plugin
+    xdist.looponfail = xdist.looponfail
+
+[options.extras_require]
+testing =
+    filelock
+psutil = psutil>=3.0
+setproctitle = setproctitle
+
 [flake8]
 max-line-length = 100
+
+[mypy]
+mypy_path = src
+# TODO: Enable this & fix errors.
+# check_untyped_defs = True
+disallow_any_generics = True
+ignore_missing_imports = True
+no_implicit_optional = True
+show_error_codes = True
+strict_equality = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_configs = True
+# TODO: Enable this & fix errors.
+# no_implicit_reexport = True

setup.py

@@ -1,48 +1,4 @@
-from setuptools import setup, find_packages
+from setuptools import setup
 
-install_requires = ["execnet>=1.1", "pytest>=3.6.0", "pytest-forked", "six"]
-
-
-with open("README.rst") as f:
-    long_description = f.read()
-
-setup(
-    name="pytest-xdist",
-    use_scm_version={"write_to": "xdist/_version.py"},
-    description="pytest xdist plugin for distributed testing"
-    " and loop-on-failing modes",
-    long_description=long_description,
-    license="MIT",
-    author="holger krekel and contributors",
-    author_email="pytest-dev@python.org,holger@merlinux.eu",
-    url="https://github.com/pytest-dev/pytest-xdist",
-    platforms=["linux", "osx", "win32"],
-    packages=find_packages(exclude=["testing", "example"]),
-    entry_points={
-        "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
-    },
-    zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
-    install_requires=install_requires,
-    setup_requires=["setuptools_scm"],
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Framework :: Pytest",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: MIT License",
-        "Operating System :: POSIX",
-        "Operating System :: Microsoft :: Windows",
-        "Operating System :: MacOS :: MacOS X",
-        "Topic :: Software Development :: Testing",
-        "Topic :: Software Development :: Quality Assurance",
-        "Topic :: Utilities",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
-    ],
-)
+if __name__ == "__main__":
+    setup()

src/xdist/__init__.py

@@ -0,0 +1,15 @@
+from xdist.plugin import (
+    is_xdist_worker,
+    is_xdist_master,
+    get_xdist_worker_id,
+    is_xdist_controller,
+)
+from xdist._version import version as __version__
+
+__all__ = [
+    "__version__",
+    "is_xdist_worker",
+    "is_xdist_master",
+    "is_xdist_controller",
+    "get_xdist_worker_id",
+]

src/xdist/dsession.py

@@ -7,17 +7,18 @@ from xdist.scheduler import (
     LoadScheduling,
     LoadScopeScheduling,
     LoadFileScheduling,
+    LoadGroupScheduling,
 )
 
 
-from six.moves.queue import Empty, Queue
+from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
+    """signals an immediate interruption."""
 
 
-class DSession(object):
+class DSession:
     """A pytest plugin which runs a distributed test session
 
     At the beginning of the test session this creates a NodeManager
@@ -46,14 +47,11 @@ class DSession(object):
         self._failed_collection_errors = {}
         self._active_nodes = set()
         self._failed_nodes_count = 0
-        self._max_worker_restart = self.config.option.maxworkerrestart
-        if self._max_worker_restart is not None:
-            self._max_worker_restart = int(self._max_worker_restart)
-        try:
-            self.terminal = config.pluginmanager.getplugin("terminalreporter")
-        except KeyError:
-            self.terminal = None
-        else:
+        self._max_worker_restart = get_default_max_worker_restart(self.config)
+        # summary message to print at the end of the session
+        self._summary_report = None
+        self.terminal = config.pluginmanager.getplugin("terminalreporter")
+        if self.terminal:
             self.trdist = TerminalDistReporter(config)
             config.pluginmanager.register(self.trdist, "terminaldistreporter")
 
@@ -70,7 +68,7 @@ class DSession(object):
         if self.terminal and self.config.option.verbose >= 0:
             self.terminal.write_line(line)
 
-    @pytest.mark.trylast
+    @pytest.hookimpl(trylast=True)
     def pytest_sessionstart(self, session):
         """Creates and starts the nodes.
 
@@ -82,6 +80,7 @@ class DSession(object):
         self._active_nodes.update(nodes)
         self._session = session
 
+    @pytest.hookimpl
     def pytest_sessionfinish(self, session):
         """Shutdown all nodes."""
         nm = getattr(self, "nodemanager", None)  # if not fully initialized
@@ -89,11 +88,12 @@ class DSession(object):
             nm.teardown_nodes()
         self._session = None
 
+    @pytest.hookimpl
     def pytest_collection(self):
-        # prohibit collection of test items in master process
+        # prohibit collection of test items in controller process
         return True
 
-    @pytest.mark.trylast
+    @pytest.hookimpl(trylast=True)
     def pytest_xdist_make_scheduler(self, config, log):
         dist = config.getvalue("dist")
         schedulers = {
@@ -101,9 +101,11 @@ class DSession(object):
             "load": LoadScheduling,
             "loadscope": LoadScopeScheduling,
             "loadfile": LoadFileScheduling,
+            "loadgroup": LoadGroupScheduling,
         }
         return schedulers[dist](config, log)
 
+    @pytest.hookimpl
     def pytest_runtestloop(self):
         self.sched = self.config.hook.pytest_xdist_make_scheduler(
             config=self.config, log=self.log
@@ -153,9 +155,6 @@ class DSession(object):
         node.workerinfo["id"] = node.gateway.id
         node.workerinfo["spec"] = node.gateway.spec
 
-        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
-        node.slaveinfo = node.workerinfo
-
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
@@ -172,7 +171,7 @@ class DSession(object):
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
         if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
-            self.shouldstop = "%s received keyboard-interrupt" % (node,)
+            self.shouldstop = "{} received keyboard-interrupt".format(node)
             self.worker_errordown(node, "keyboard-interrupt")
             return
         if node in self.sched.nodes:
@@ -180,6 +179,24 @@ class DSession(object):
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
     def worker_errordown(self, node, error):
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
@@ -198,15 +215,24 @@ class DSession(object):
         )
         if maximum_reached:
             if self._max_worker_restart == 0:
-                msg = "Worker restarting disabled"
+                msg = "worker {} crashed and worker restarting disabled".format(
+                    node.gateway.id
+                )
             else:
-                msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
-            self.report_line(msg)
+                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
         else:
-            self.report_line("Replacing crashed worker %s" % node.gateway.id)
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
+    @pytest.hookimpl
+    def pytest_terminal_summary(self, terminalreporter):
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
+
     def worker_collectionfinish(self, node, ids):
         """worker has finished test collection.
 
@@ -220,7 +246,7 @@ class DSession(object):
             return
         self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
         # tell session which items were effectively collected otherwise
-        # the master node will finish the session with EXIT_NOTESTSCOLLECTED
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
         self.sched.add_node_collection(node, ids)
         if self.terminal:
@@ -266,16 +292,20 @@ class DSession(object):
         assert not rep.passed
         self._failed_worker_collectreport(node, rep)
 
-    def worker_logwarning(self, message, code, nodeid, fslocation):
-        """Emitted when a node calls the pytest_logwarning hook."""
-        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
-        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
-
     def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_logwarning hook."""
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
+        # This hook as been removed in pytest 7.1, and we can remove support once we only
+        # support pytest >=7.1.
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 
@@ -316,15 +346,21 @@ class DSession(object):
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
+        msg = "worker {!r} crashed while running {!r}".format(worker.gateway.id, nodeid)
         rep = runner.TestReport(
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
         rep.node = worker
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
-class TerminalDistReporter(object):
+class TerminalDistReporter:
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")
@@ -346,7 +382,9 @@ class TerminalDistReporter(object):
 
     def getstatus(self):
         if self.config.option.verbose >= 0:
-            parts = ["%s %s" % (spec.id, self._status[spec.id]) for spec in self._specs]
+            parts = [
+                "{} {}".format(spec.id, self._status[spec.id]) for spec in self._specs
+            ]
             return " / ".join(parts)
         else:
             return "bringing up nodes..."
@@ -360,6 +398,7 @@ class TerminalDistReporter(object):
             self._lastlen = len(line)
         self.tr.rewrite(pline, bold=True)
 
+    @pytest.hookimpl
     def pytest_xdist_setupnodes(self, specs):
         self._specs = specs
         for spec in specs:
@@ -367,6 +406,7 @@ class TerminalDistReporter(object):
         self.setstatus(spec, "I", show=True)
         self.ensure_show_status()
 
+    @pytest.hookimpl
     def pytest_xdist_newgateway(self, gateway):
         if self.config.option.verbose > 0:
             rinfo = gateway._rinfo()
@@ -378,22 +418,32 @@ class TerminalDistReporter(object):
             )
         self.setstatus(gateway.spec, "C")
 
+    @pytest.hookimpl
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
             d = node.workerinfo
-            infoline = "[%s] Python %s" % (d["id"], d["version"].replace("\n", " -- "))
+            infoline = "[{}] Python {}".format(
+                d["id"], d["version"].replace("\n", " -- ")
+            )
             self.rewrite(infoline, newline=True)
         self.setstatus(node.gateway.spec, "ok")
 
+    @pytest.hookimpl
     def pytest_testnodedown(self, node, error):
         if not error:
             return
-        self.write_line("[%s] node down: %s" % (node.gateway.id, error))
+        self.write_line("[{}] node down: {}".format(node.gateway.id, error))
 
-    # def pytest_xdist_rsyncstart(self, source, gateways):
-    #    targets = ",".join([gw.id for gw in gateways])
-    #    msg = "[%s] rsyncing: %s" %(targets, source)
-    #    self.write_line(msg)
-    # def pytest_xdist_rsyncfinish(self, source, gateways):
-    #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
-    #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
+
+def get_default_max_worker_restart(config):
+    """gets the default value of --max-worker-restart option if it is not provided.
+
+    Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
+    """
+    result = config.option.maxworkerrestart
+    if result is not None:
+        result = int(result)
+    elif config.option.numprocesses:
+        # if --max-worker-restart was not provided, use a reasonable default (#226)
+        result = config.option.numprocesses * 4
+    return result

src/xdist/looponfail.py

@@ -6,7 +6,6 @@
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-from __future__ import print_function
 import py
 import pytest
 import sys
@@ -14,6 +13,7 @@ import time
 import execnet
 
 
+@pytest.hookimpl
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
@@ -27,10 +27,11 @@ def pytest_addoption(parser):
     )
 
 
+@pytest.hookimpl
 def pytest_cmdline_main(config):
 
     if config.getoption("looponfail"):
-        usepdb = config.getoption("usepdb")  # a core option
+        usepdb = config.getoption("usepdb", False)  # a core option
         if usepdb:
             raise pytest.UsageError("--pdb is incompatible with --looponfail.")
         looponfail_main(config)
@@ -39,7 +40,7 @@ def pytest_cmdline_main(config):
 
 def looponfail_main(config):
     remotecontrol = RemoteControl(config)
-    rootdirs = config.getini("looponfailroots")
+    rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")]
     statrecorder = StatRecorder(rootdirs)
     try:
         while 1:
@@ -55,14 +56,14 @@ def looponfail_main(config):
         print()
 
 
-class RemoteControl(object):
+class RemoteControl:
     def __init__(self, config):
         self.config = config
         self.failures = []
 
     def trace(self, *args):
         if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
+            msg = " ".join(str(x) for x in args)
             print("RemoteControl:", msg)
 
     def initgateway(self):
@@ -136,7 +137,7 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
                 tr.line(report, red=True)
     tr.sep("#", "waiting for changes", bold=True)
     for rootdir in rootdirs:
-        tr.line("### Watching:   %s" % (rootdir,), bold=True)
+        tr.line("### Watching:   {}".format(rootdir), bold=True)
 
 
 def init_worker_session(channel, args, option_dict):
@@ -165,7 +166,7 @@ def init_worker_session(channel, args, option_dict):
     WorkerFailSession(config, channel).main()
 
 
-class WorkerFailSession(object):
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -179,6 +180,7 @@ class WorkerFailSession(object):
         if self.config.option.debug:
             print(" ".join(map(str, args)))
 
+    @pytest.hookimpl
     def pytest_collection(self, session):
         self.session = session
         self.trails = self.current_command
@@ -193,10 +195,12 @@ class WorkerFailSession(object):
         hook.pytest_collection_finish(session=session)
         return True
 
+    @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)
 
+    @pytest.hookimpl
     def pytest_collectreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)
@@ -220,7 +224,7 @@ class WorkerFailSession(object):
         self.channel.send((trails, failreports, self.collection_failed))
 
 
-class StatRecorder(object):
+class StatRecorder:
     def __init__(self, rootdirlist):
         self.rootdirlist = rootdirlist
         self.statcache = {}

src/xdist/newhooks.py

@@ -0,0 +1,91 @@
+"""
+xdist hooks.
+
+Additionally, pytest-xdist will also decorate a few other hooks
+with the worker instance that executed the hook originally:
+
+``pytest_runtest_logreport``: ``rep`` parameter has a ``node`` attribute.
+
+You can use this hooks just as you would use normal pytest hooks, but some care
+must be taken in plugins in case ``xdist`` is not installed. Please see:
+
+    http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
+"""
+import pytest
+
+
+@pytest.hookspec()
+def pytest_xdist_setupnodes(config, specs):
+    """called before any remote node is set up."""
+
+
+@pytest.hookspec()
+def pytest_xdist_newgateway(gateway):
+    """called on new raw gateway creation."""
+
+
+@pytest.hookspec()
+def pytest_xdist_rsyncstart(source, gateways):
+    """called before rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec()
+def pytest_xdist_rsyncfinish(source, gateways):
+    """called after rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_getremotemodule():
+    """called when creating remote node"""
+
+
+@pytest.hookspec()
+def pytest_configure_node(node):
+    """configure node information before it gets instantiated."""
+
+
+@pytest.hookspec()
+def pytest_testnodeready(node):
+    """Test Node is ready to operate."""
+
+
+@pytest.hookspec()
+def pytest_testnodedown(node, error):
+    """Test Node is down."""
+
+
+@pytest.hookspec()
+def pytest_xdist_node_collection_finished(node, ids):
+    """called by the controller node when a worker node finishes collecting."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_make_scheduler(config, log):
+    """return a node scheduler implementation"""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_handlecrashitem(crashitem, report, sched):
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -1,12 +1,34 @@
 import os
+import uuid
+import sys
+from pathlib import Path
 
 import py
 import pytest
 
 
-def auto_detect_cpus():
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
+
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
+
+
+@pytest.hookimpl
+def pytest_xdist_auto_num_workers(config):
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
     try:
         from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
     except ImportError:
         if os.environ.get("TRAVIS") == "true":
             # workaround https://bitbucket.org/pypy/pypy/issues/2375
@@ -15,11 +37,6 @@ def auto_detect_cpus():
             from os import cpu_count
         except ImportError:
             from multiprocessing import cpu_count
-    else:
-
-        def cpu_count():
-            return len(sched_getaffinity(0))
-
     try:
         n = cpu_count()
     except NotImplementedError:
@@ -28,12 +45,13 @@ def auto_detect_cpus():
 
 
 def parse_numprocesses(s):
-    if s == "auto":
-        return auto_detect_cpus()
-    else:
+    if s in ("auto", "logical"):
+        return s
+    elif s is not None:
         return int(s)
 
 
+@pytest.hookimpl
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
@@ -43,9 +61,10 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="shortcut for '--dist=load --tx=NUM*popen', "
-        "you can use 'auto' here for auto detection CPUs number on "
-        "host system",
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -57,20 +76,17 @@ def pytest_addoption(parser):
     )
     group.addoption(
         "--max-worker-restart",
-        "--max-slave-restart",
         action="store",
         default=None,
         dest="maxworkerrestart",
         help="maximum number of workers that can be restarted "
-        "when crashed (set to zero to disable this feature)\n"
-        "'--max-slave-restart' option is deprecated and will be removed in "
-        "a future release",
+        "when crashed (set to zero to disable this feature)",
     )
     group.addoption(
         "--dist",
         metavar="distmode",
         action="store",
-        choices=["each", "load", "loadscope", "loadfile", "no"],
+        choices=["each", "load", "loadscope", "loadfile", "loadgroup", "no"],
         dest="dist",
         default="no",
         help=(
@@ -82,6 +98,7 @@ def pytest_addoption(parser):
             " the same scope to any available environment.\n\n"
             "loadfile: load balance by sending test grouped by file"
             " to any available environment.\n\n"
+            "loadgroup: like load, but sends tests marked with 'xdist_group' to the same worker.\n\n"
             "(default) no: run tests inprocess, don't distribute."
         ),
     )
@@ -118,27 +135,37 @@ def pytest_addoption(parser):
         metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.",
     )
-
     group.addoption(
         "--boxed",
         action="store_true",
         help="backward compatibility alias for pytest-forked --forked",
     )
+    group.addoption(
+        "--testrunuid",
+        action="store",
+        help=(
+            "provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture,\n\n,"
+            "if not provided, 'testrun_uid' is filled with a new unique string "
+            "on every test run."
+        ),
+    )
+
     parser.addini(
         "rsyncdirs",
         "list of (relative) paths to be rsynced for remote distributed testing.",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
     )
     parser.addini(
         "rsyncignore",
         "list of (relative) glob-style paths to be ignored for rsyncing.",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
     )
     parser.addini(
         "looponfailroots",
-        type="pathlist",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
         help="directories to check for changes",
-        default=[py.path.local()],
+        default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()],
     )
 
 
@@ -147,14 +174,11 @@ def pytest_addoption(parser):
 # -------------------------------------------------------------------------
 
 
+@pytest.hookimpl
 def pytest_addhooks(pluginmanager):
     from xdist import newhooks
 
-    # avoid warnings with pytest-2.8
-    method = getattr(pluginmanager, "add_hookspecs", None)
-    if method is None:
-        method = pluginmanager.addhooks
-    method(newhooks)
+    pluginmanager.add_hookspecs(newhooks)
 
 
 # -------------------------------------------------------------------------
@@ -162,7 +186,7 @@ def pytest_addhooks(pluginmanager):
 # -------------------------------------------------------------------------
 
 
-@pytest.mark.trylast
+@pytest.hookimpl(trylast=True)
 def pytest_configure(config):
     if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
         from xdist.dsession import DSession
@@ -170,13 +194,35 @@ def pytest_configure(config):
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
         tr = config.pluginmanager.getplugin("terminalreporter")
-        tr.showfspath = False
+        if tr:
+            tr.showfspath = False
     if config.getoption("boxed"):
+        warning = DeprecationWarning(
+            "The --boxed commmand line argument is deprecated. "
+            "Install pytest-forked and use --forked instead. "
+            "pytest-xdist 3.0.0 will remove the --boxed argument and pytest-forked dependency."
+        )
+        config.issue_config_time_warning(warning, 2)
         config.option.forked = True
 
+    config_line = (
+        "xdist_group: specify group for tests should run in same session."
+        "in relation to one another. " + "Provided by pytest-xdist."
+    )
+    config.addinivalue_line("markers", config_line)
 
-@pytest.mark.tryfirst
+
+@pytest.hookimpl(tryfirst=True)
 def pytest_cmdline_main(config):
+    usepdb = config.getoption("usepdb", False)  # a core option
+    if config.option.numprocesses in ("auto", "logical"):
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
+
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
@@ -187,26 +233,72 @@ def pytest_cmdline_main(config):
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue
-    if not val("collectonly"):
-        usepdb = config.getoption("usepdb")  # a core option
-        if val("dist") != "no":
-            if usepdb:
-                raise pytest.UsageError(
-                    "--pdb is incompatible with distributing tests; try using -n0."
-                )  # noqa: E501
+    if not val("collectonly") and val("dist") != "no" and usepdb:
+        raise pytest.UsageError(
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
+        )  # noqa: E501
 
 
 # -------------------------------------------------------------------------
-# fixtures
+# fixtures and API to easily know the role of current node
 # -------------------------------------------------------------------------
 
 
+def is_xdist_worker(request_or_session) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_controller(request_or_session) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
+def get_xdist_worker_id(request_or_session):
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the controller node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        return request_or_session.config.workerinput["workerid"]
+    else:
+        # TODO: remove "master", ideally for a None
+        return "master"
+
+
 @pytest.fixture(scope="session")
 def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
+    # TODO: remove "master", ideally for a None
+    return get_xdist_worker_id(request)
+
+
+@pytest.fixture(scope="session")
+def testrun_uid(request):
+    """Return the unique id of the current test."""
     if hasattr(request.config, "workerinput"):
-        return request.config.workerinput["workerid"]
+        return request.config.workerinput["testrunuid"]
     else:
-        return "master"
+        return uuid.uuid4().hex

src/xdist/remote.py

@@ -10,15 +10,33 @@ import sys
 import os
 import time
 
-import _pytest.hookspec
+import py
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
+from _pytest.config import _prepareconfig, Config
 
-class WorkerInteractor(object):
+try:
+    from setproctitle import setproctitle
+except ImportError:
+
+    def setproctitle(title):
+        pass
+
+
+def worker_title(title):
+    try:
+        setproctitle(title)
+    except Exception:
+        # changing the process name is very optional, no errors please
+        pass
+
+
+class WorkerInteractor:
     def __init__(self, config, channel):
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
+        self.testrunuid = config.workerinput["testrunuid"]
         self.log = py.log.Producer("worker-%s" % self.workerid)
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
@@ -29,10 +47,14 @@ class WorkerInteractor(object):
         self.log("sending", name, kwargs)
         self.channel.send((name, kwargs))
 
+    @pytest.hookimpl
     def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
 
+    @pytest.hookimpl
     def pytest_sessionstart(self, session):
         self.session = session
         workerinfo = getinfodict()
@@ -40,13 +62,16 @@ class WorkerInteractor(object):
 
     @pytest.hookimpl(hookwrapper=True)
     def pytest_sessionfinish(self, exitstatus):
-        self.config.workeroutput["exitstatus"] = exitstatus
+        # in pytest 5.0+, exitstatus is an IntEnum object
+        self.config.workeroutput["exitstatus"] = int(exitstatus)
         yield
         self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
+    @pytest.hookimpl
     def pytest_collection(self, session):
         self.sendevent("collectionstart")
 
+    @pytest.hookimpl
     def pytest_runtestloop(self, session):
         self.log("entering main loop")
         torun = []
@@ -79,108 +104,82 @@ class WorkerInteractor(object):
         else:
             nextitem = None
 
+        worker_title("[pytest-xdist running] %s" % item.nodeid)
+
         start = time.time()
         self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
         duration = time.time() - start
+
+        worker_title("[pytest-xdist idle]")
+
         self.sendevent(
             "runtest_protocol_complete", item_index=self.item_index, duration=duration
         )
 
+    def pytest_collection_modifyitems(self, session, config, items):
+        # add the group name to nodeid as suffix if --dist=loadgroup
+        if config.getvalue("loadgroup"):
+            for item in items:
+                mark = item.get_closest_marker("xdist_group")
+                if not mark:
+                    continue
+                gname = (
+                    mark.args[0]
+                    if len(mark.args) > 0
+                    else mark.kwargs.get("name", "default")
+                )
+                item._nodeid = "{}@{}".format(item.nodeid, gname)
+
+    @pytest.hookimpl
     def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
         self.sendevent(
             "collectionfinish",
-            topdir=str(session.fspath),
+            topdir=topdir,
             ids=[item.nodeid for item in session.items],
         )
 
+    @pytest.hookimpl
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
-    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
-    if hasattr(_pytest.hookspec, "pytest_runtest_logfinish"):
-
-        def pytest_runtest_logfinish(self, nodeid, location):
-            self.sendevent("logfinish", nodeid=nodeid, location=location)
+    @pytest.hookimpl
+    def pytest_runtest_logfinish(self, nodeid, location):
+        self.sendevent("logfinish", nodeid=nodeid, location=location)
 
+    @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
-        data = serialize_report(report)
+        data = self.config.hook.pytest_report_to_serializable(
+            config=self.config, report=report
+        )
         data["item_index"] = self.item_index
         data["worker_id"] = self.workerid
+        data["testrun_uid"] = self.testrunuid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
+    @pytest.hookimpl
     def pytest_collectreport(self, report):
-        # send only reports that have not passed to master as optimization (#330)
+        # send only reports that have not passed to controller as optimization (#330)
         if not report.passed:
-            data = serialize_report(report)
+            data = self.config.hook.pytest_report_to_serializable(
+                config=self.config, report=report
+            )
             self.sendevent("collectreport", data=data)
 
-    # the pytest_logwarning hook was deprecated since pytest 4.0
-    if hasattr(
-        _pytest.hookspec, "pytest_logwarning"
-    ) and not _pytest.hookspec.pytest_logwarning.pytest_spec.get("warn_on_impl"):
-
-        def pytest_logwarning(self, message, code, nodeid, fslocation):
-            self.sendevent(
-                "logwarning",
-                message=message,
-                code=code,
-                nodeid=nodeid,
-                fslocation=str(fslocation),
-            )
-
-    # the pytest_warning_captured hook was introduced in pytest 3.8
-    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
-
-        def pytest_warning_captured(self, warning_message, when, item):
-            self.sendevent(
-                "warning_captured",
-                warning_message_data=serialize_warning_message(warning_message),
-                when=when,
-                # item cannot be serialized and will always be None when used with xdist
-                item=None,
-            )
-
-
-def serialize_report(rep):
-    def disassembled_report(rep):
-        reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
-        reprcrash = rep.longrepr.reprcrash.__dict__.copy()
-
-        new_entries = []
-        for entry in reprtraceback["reprentries"]:
-            entry_data = {"type": type(entry).__name__, "data": entry.__dict__.copy()}
-            for key, value in entry_data["data"].items():
-                if hasattr(value, "__dict__"):
-                    entry_data["data"][key] = value.__dict__.copy()
-            new_entries.append(entry_data)
-
-        reprtraceback["reprentries"] = new_entries
-
-        return {
-            "reprcrash": reprcrash,
-            "reprtraceback": reprtraceback,
-            "sections": rep.longrepr.sections,
-        }
-
-    import py
-
-    d = rep.__dict__.copy()
-    if hasattr(rep.longrepr, "toterminal"):
-        if hasattr(rep.longrepr, "reprtraceback") and hasattr(
-            rep.longrepr, "reprcrash"
-        ):
-            d["longrepr"] = disassembled_report(rep)
-        else:
-            d["longrepr"] = str(rep.longrepr)
-    else:
-        d["longrepr"] = rep.longrepr
-    for name in d:
-        if isinstance(d[name], py.path.local):
-            d[name] = str(d[name])
-        elif name == "result":
-            d[name] = None  # for now
-    return d
+    @pytest.hookimpl
+    def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+        self.sendevent(
+            "warning_recorded",
+            warning_message_data=serialize_warning_message(warning_message),
+            when=when,
+            nodeid=nodeid,
+            location=location,
+        )
 
 
 def serialize_warning_message(warning_message):
@@ -189,7 +188,7 @@ def serialize_warning_message(warning_message):
         message_class_name = type(warning_message.message).__name__
         message_str = str(warning_message.message)
         # check now if we can serialize the warning arguments (#349)
-        # if not, we will just use the exception message on the master node
+        # if not, we will just use the exception message on the controller node
         try:
             dumps(warning_message.message.args)
         except DumpError:
@@ -246,42 +245,47 @@ def getinfodict():
 
 
 def remote_initconfig(option_dict, args):
-    from _pytest.config import Config
-
     option_dict["plugins"].append("no:terminal")
-    config = Config.fromdictargs(option_dict, args)
+    return Config.fromdictargs(option_dict, args)
+
+
+def setup_config(config, basetemp):
+    config.option.loadgroup = config.getvalue("dist") == "loadgroup"
     config.option.looponfail = False
     config.option.usepdb = False
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
     config.option.maxprocesses = None
-    config.args = args
-    return config
+    config.option.basetemp = basetemp
 
 
 if __name__ == "__channelexec__":
-    import py
+    channel = channel  # type: ignore[name-defined] # noqa: F821
+    workerinput, args, option_dict, change_sys_path = channel.receive()  # type: ignore[name-defined]
 
-    channel = channel  # noqa
-    workerinput, args, option_dict, change_sys_path = channel.receive()
-
-    if change_sys_path:
+    if change_sys_path is None:
         importpath = os.getcwd()
         sys.path.insert(0, importpath)
         os.environ["PYTHONPATH"] = (
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
+    else:
+        sys.path = change_sys_path
 
+    os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
     os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
 
-    config = remote_initconfig(option_dict, args)
+    if hasattr(Config, "InvocationParams"):
+        config = _prepareconfig(args, None)
+    else:
+        config = remote_initconfig(option_dict, args)
+        config.args = args
+
+    setup_config(config, option_dict.get("basetemp"))
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
-    config.workerinput = workerinput
-    config.workeroutput = {}
-    # TODO: deprecated name, backward compatibility only. Remove it in future
-    config.slaveinput = config.workerinput
-    config.slaveoutput = config.workeroutput
-    interactor = WorkerInteractor(config, channel)
+    config.workerinput = workerinput  # type: ignore[attr-defined]
+    config.workeroutput = {}  # type: ignore[attr-defined]
+    interactor = WorkerInteractor(config, channel)  # type: ignore[name-defined]
     config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 from difflib import unified_diff
 
 
@@ -13,9 +12,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
 
     diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
     error_message = (
-        u"Different tests were collected between {from_id} and {to_id}. "
-        u"The difference is:\n"
-        u"{diff}"
+        "Different tests were collected between {from_id} and {to_id}. "
+        "The difference is:\n"
+        "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/__init__.py

@@ -2,3 +2,4 @@ from xdist.scheduler.each import EachScheduling  # noqa
 from xdist.scheduler.load import LoadScheduling  # noqa
 from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
+from xdist.scheduler.loadgroup import LoadGroupScheduling  # noqa

src/xdist/scheduler/each.py

@@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class EachScheduling(object):
+class EachScheduling:
     """Implement scheduling of test items on all nodes
 
     If a node gets added after the test run is started then it is
@@ -101,6 +101,14 @@ class EachScheduling(object):
     def mark_test_complete(self, node, item_index, duration=0):
         self.node2pending[node].remove(item_index)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def remove_node(self, node):
         # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)

src/xdist/scheduler/load.py

@@ -7,7 +7,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class LoadScheduling(object):
+class LoadScheduling:
     """Implement load scheduling across nodes.
 
     This distributes the tests collected across all nodes so each test
@@ -151,6 +151,14 @@ class LoadScheduling(object):
         self.node2pending[node].remove(item_index)
         self.check_schedule(node, duration=duration)
 
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
     def check_schedule(self, node, duration=0):
         """Maybe schedule new items on the node
 

src/xdist/scheduler/loadfile.py

@@ -21,7 +21,7 @@ class LoadFileScheduling(LoadScopeScheduling):
     """
 
     def __init__(self, config, log=None):
-        super(LoadFileScheduling, self).__init__(config, log)
+        super().__init__(config, log)
         if log is None:
             self.log = Producer("loadfilesched")
         else:

src/xdist/scheduler/loadgroup.py

@@ -0,0 +1,54 @@
+from .loadscope import LoadScopeScheduling
+from py.log import Producer
+
+
+class LoadGroupScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test by xdist_group mark.
+
+    This class behaves very much like LoadScopeScheduling, but it groups tests by xdist_group mark
+    instead of the module or class to which they belong to.
+    """
+
+    def __init__(self, config, log=None):
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadgroupsched")
+        else:
+            self.log = log.loadgroupsched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        With loadgroup, two cases are added::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            example/loadsuite/test/test_gamma.py::test_beta0@gname
+            example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname
+
+        This function will group tests with the scope determined by splitting the first ``@``
+        from the right. That is, test will be grouped in a single work unit when they have
+        same group name. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            gname
+            gname
+        """
+        if nodeid.rfind("@") > nodeid.rfind("]"):
+            # check the index of ']' to avoid the case: parametrize mark value has '@'
+            return nodeid.split("@")[-1]
+        else:
+            return nodeid

src/xdist/scheduler/loadscope.py

@@ -6,7 +6,7 @@ from xdist.report import report_collection_diff
 from xdist.workermanage import parse_spec_config
 
 
-class LoadScopeScheduling(object):
+class LoadScopeScheduling:
     """Implement load scheduling across nodes, but grouping test by scope.
 
     This distributes the tests collected across all nodes so each test is run
@@ -243,6 +243,9 @@ class LoadScopeScheduling(object):
         self.assigned_work[node][scope][nodeid] = True
         self._reschedule(node)
 
+    def mark_test_pending(self, item):
+        raise NotImplementedError()
+
     def _assign_work_unit(self, node):
         """Assign a work unit to a node."""
         assert self.workqueue
@@ -358,12 +361,12 @@ class LoadScopeScheduling(object):
         extra_nodes = len(self.nodes) - len(self.workqueue)
 
         if extra_nodes > 0:
-            self.log("Shuting down {0} nodes".format(extra_nodes))
+            self.log("Shuting down {} nodes".format(extra_nodes))
 
             for _ in range(extra_nodes):
                 unused_node, assigned = self.assigned_work.popitem(last=True)
 
-                self.log("Shuting down unused node {0}".format(unused_node))
+                self.log("Shuting down unused node {}".format(unused_node))
                 unused_node.shutdown()
 
         # Assign initial workload

src/xdist/workermanage.py

@@ -1,17 +1,15 @@
-from __future__ import print_function
 import fnmatch
 import os
 import re
 import sys
-import threading
+import uuid
 
 import py
 import pytest
 import execnet
 
 import xdist.remote
-
-from _pytest import runner  # XXX load dynamically
+from xdist.plugin import _sys_path
 
 
 def parse_spec_config(config):
@@ -31,14 +29,16 @@ def parse_spec_config(config):
     return xspeclist
 
 
-class NodeManager(object):
+class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
-        self._nodesready = threading.Event()
         self.trace = self.config.trace.get("nodemanager")
+        self.testrunuid = self.config.getoption("testrunuid")
+        if self.testrunuid is None:
+            self.testrunuid = uuid.uuid4().hex
         self.group = execnet.Group()
         if specs is None:
             specs = self._getxspecs()
@@ -63,10 +63,7 @@ class NodeManager(object):
     def setup_nodes(self, putevent):
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        nodes = []
-        for spec in self.specs:
-            nodes.append(self.setup_node(spec, putevent))
-        return nodes
+        return [self.setup_node(spec, putevent) for spec in self.specs]
 
     def setup_node(self, spec, putevent):
         gw = self.group.makegateway(spec)
@@ -93,8 +90,16 @@ class NodeManager(object):
         import pytest
         import _pytest
 
-        pytestpath = pytest.__file__.rstrip("co")
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
+        def get_dir(p):
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
         config = self.config
         candidates = [py._pydir, pytestpath, pytestdir]
         candidates += config.option.rsyncdir
@@ -105,7 +110,7 @@ class NodeManager(object):
         for root in candidates:
             root = py.path.local(root).realpath()
             if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: %r" % (root,))
+                raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root))
             if root not in roots:
                 roots.append(root)
         return roots
@@ -113,10 +118,13 @@ class NodeManager(object):
     def _getrsyncoptions(self):
         """Get options to be passed for rsync."""
         ignores = list(self.DEFAULT_IGNORES)
-        ignores += self.config.option.rsyncignore
-        ignores += self.config.getini("rsyncignore")
+        ignores += [str(path) for path in self.config.option.rsyncignore]
+        ignores += [str(path) for path in self.config.getini("rsyncignore")]
 
-        return {"ignores": ignores, "verbose": self.config.option.verbose}
+        return {
+            "ignores": ignores,
+            "verbose": getattr(self.config.option, "verbose", 0),
+        }
 
     def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
         """Perform rsync to remote hosts for node."""
@@ -150,17 +158,15 @@ class NodeManager(object):
 
 
 class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
-    """
+    """RSyncer that filters out common files"""
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
-        self._ignores = []
         ignores = kwargs.pop("ignores", None) or []
-        for x in ignores:
-            x = getattr(x, "strpath", x)
-            self._ignores.append(re.compile(fnmatch.translate(x)))
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
+        self._ignores = [
+            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
+        ]
+        super().__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
         path = py.path.local(path)
@@ -172,15 +178,13 @@ class HostRSync(execnet.RSync):
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(
-            gateway, remotepath, finishedcallback=finished, delete=True
-        )
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
 
     def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
+        if self._verbose > 0:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
-            print("%s:%s <= %s" % (gateway.spec, remotepath, path))
+            print("{}:{} <= {}".format(gateway.spec, remotepath, path))
 
 
 def make_reltoroot(roots, args):
@@ -190,21 +194,30 @@ def make_reltoroot(roots, args):
     for arg in args:
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
+        if not fspath.exists():
+            result.append(arg)
+            continue
         for root in roots:
             x = fspath.relto(root)
             if x or fspath == root:
                 parts[0] = root.basename + "/" + x
                 break
         else:
-            raise ValueError("arg %s not relative to an rsync root" % (arg,))
+            raise ValueError("arg {} not relative to an rsync root".format(arg))
         result.append(splitcode.join(parts))
     return result
 
 
-class WorkerController(object):
+class WorkerController:
     ENDMARK = -1
 
+    class RemoteHook:
+        @pytest.hookimpl(trylast=True)
+        def pytest_xdist_getremotemodule(self):
+            return xdist.remote
+
     def __init__(self, nodemanager, gateway, config, putevent):
+        config.pluginmanager.register(self.RemoteHook())
         self.nodemanager = nodemanager
         self.putevent = putevent
         self.gateway = gateway
@@ -212,12 +225,9 @@ class WorkerController(object):
         self.workerinput = {
             "workerid": gateway.id,
             "workercount": len(nodemanager.specs),
-            "slaveid": gateway.id,
-            "slavecount": len(nodemanager.specs),
+            "testrunuid": nodemanager.testrunuid,
             "mainargv": sys.argv,
         }
-        # TODO: deprecated name, backward compatibility only. Remove it in future
-        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
         self.log = py.log.Producer("workerctl-%s" % gateway.id)
@@ -225,7 +235,7 @@ class WorkerController(object):
             py.log.setconsumer(self.log._keywords, None)
 
     def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__, self.gateway.id)
+        return "<{} {}>".format(self.__class__.__name__, self.gateway.id)
 
     @property
     def shutting_down(self):
@@ -234,20 +244,28 @@ class WorkerController(object):
     def setup(self):
         self.log("setting up worker session")
         spec = self.gateway.spec
-        args = self.config.args
+        if hasattr(self.config, "invocation_params"):
+            args = [str(x) for x in self.config.invocation_params.args or ()]
+            option_dict = {}
+        else:
+            args = self.config.args
+            option_dict = vars(self.config.option)
         if not spec.popen or spec.chdir:
             args = make_reltoroot(self.nodemanager.roots, args)
-        option_dict = vars(self.config.option)
         if spec.popen:
             name = "popen-%s" % self.gateway.id
-            if hasattr(self.config, "_tmpdirhandler"):
-                basetemp = self.config._tmpdirhandler.getbasetemp()
-                option_dict["basetemp"] = str(basetemp.join(name))
+            if hasattr(self.config, "_tmp_path_factory"):
+                basetemp = self.config._tmp_path_factory.getbasetemp()
+                option_dict["basetemp"] = str(basetemp / name)
         self.config.hook.pytest_configure_node(node=self)
-        self.channel = self.gateway.remote_exec(xdist.remote)
+
+        remote_module = self.config.hook.pytest_xdist_getremotemodule()
+        self.channel = self.gateway.remote_exec(remote_module)
         # change sys.path only for remote workers
-        change_sys_path = not self.gateway.spec.popen
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
+
         if self.putevent:
             self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
 
@@ -272,26 +290,26 @@ class WorkerController(object):
         if not self._down:
             try:
                 self.sendcommand("shutdown")
-            except (IOError, OSError):
+            except OSError:
                 pass
             self._shutdown_sent = True
 
     def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
-        self.log("sending command %s(**%s)" % (name, kwargs))
+        """send a named parametrized command to the other side."""
+        self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
     def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing %s(**%s)" % (eventname, kwargs))
+        self.log("queuing {}(**{})".format(eventname, kwargs))
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
-        """ this gets called for each object we receive from
-            the other side and if the channel closes.
+        """this gets called for each object we receive from
+        the other side and if the channel closes.
 
-            Note that channel callbacks run in the receiver
-            thread of execnet gateways - we need to
-            avoid raising exceptions or doing heavy work.
+        Note that channel callbacks run in the receiver
+        thread of execnet gateways - we need to
+        avoid raising exceptions or doing heavy work.
         """
         try:
             if eventcall == self.ENDMARK:
@@ -304,18 +322,22 @@ class WorkerController(object):
                 return
             eventname, kwargs = eventcall
             if eventname in ("collectionstart",):
-                self.log("ignoring %s(%s)" % (eventname, kwargs))
+                self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
-                self.workeroutput = self.slaveoutput = kwargs["workeroutput"]
+                self.workeroutput = kwargs["workeroutput"]
                 self.notify_inproc("workerfinished", node=self)
             elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in ("testreport", "collectreport", "teardownreport"):
                 item_index = kwargs.pop("item_index", None)
-                rep = unserialize_report(eventname, kwargs["data"])
+                rep = self.config.hook.pytest_report_from_serializable(
+                    config=self.config, data=kwargs["data"]
+                )
                 if item_index is not None:
                     rep.item_index = item_index
                 self.notify_inproc(eventname, node=self, rep=rep)
@@ -341,92 +363,32 @@ class WorkerController(object):
                     when=kwargs["when"],
                     item=kwargs["item"],
                 )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
             else:
-                raise ValueError("unknown event: %s" % (eventname,))
+                raise ValueError("unknown event: {}".format(eventname))
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise
         except:  # noqa
             from _pytest._code import ExceptionInfo
 
-            # ExceptionInfo API changed in pytest 4.1
-            if hasattr(ExceptionInfo, "from_current"):
-                excinfo = ExceptionInfo.from_current()
-            else:
-                excinfo = ExceptionInfo()
+            excinfo = ExceptionInfo.from_current()
             print("!" * 20, excinfo)
             self.config.notify_exception(excinfo)
             self.shutdown()
             self.notify_inproc("errordown", node=self, error=excinfo)
 
 
-def unserialize_report(name, reportdict):
-    def assembled_report(reportdict):
-        from _pytest._code.code import (
-            ReprEntry,
-            ReprEntryNative,
-            ReprExceptionInfo,
-            ReprFileLocation,
-            ReprFuncArgs,
-            ReprLocals,
-            ReprTraceback,
-        )
-
-        if reportdict["longrepr"]:
-            if (
-                "reprcrash" in reportdict["longrepr"]
-                and "reprtraceback" in reportdict["longrepr"]
-            ):
-
-                reprtraceback = reportdict["longrepr"]["reprtraceback"]
-                reprcrash = reportdict["longrepr"]["reprcrash"]
-
-                unserialized_entries = []
-                reprentry = None
-                for entry_data in reprtraceback["reprentries"]:
-                    data = entry_data["data"]
-                    entry_type = entry_data["type"]
-                    if entry_type == "ReprEntry":
-                        reprfuncargs = None
-                        reprfileloc = None
-                        reprlocals = None
-                        if data["reprfuncargs"]:
-                            reprfuncargs = ReprFuncArgs(**data["reprfuncargs"])
-                        if data["reprfileloc"]:
-                            reprfileloc = ReprFileLocation(**data["reprfileloc"])
-                        if data["reprlocals"]:
-                            reprlocals = ReprLocals(data["reprlocals"]["lines"])
-
-                        reprentry = ReprEntry(
-                            lines=data["lines"],
-                            reprfuncargs=reprfuncargs,
-                            reprlocals=reprlocals,
-                            filelocrepr=reprfileloc,
-                            style=data["style"],
-                        )
-                    elif entry_type == "ReprEntryNative":
-                        reprentry = ReprEntryNative(data["lines"])
-                    else:
-                        report_unserialization_failure(entry_type, name, reportdict)
-                    unserialized_entries.append(reprentry)
-                reprtraceback["reprentries"] = unserialized_entries
-
-                exception_info = ReprExceptionInfo(
-                    reprtraceback=ReprTraceback(**reprtraceback),
-                    reprcrash=ReprFileLocation(**reprcrash),
-                )
-
-                for section in reportdict["longrepr"]["sections"]:
-                    exception_info.addsection(*section)
-                reportdict["longrepr"] = exception_info
-        return reportdict
-
-    if name == "testreport":
-        return runner.TestReport(**assembled_report(reportdict))
-    elif name == "collectreport":
-        return runner.CollectReport(**assembled_report(reportdict))
-
-
 def unserialize_warning_message(data):
     import warnings
     import importlib
@@ -461,23 +423,9 @@ def unserialize_warning_message(data):
 
     kwargs = {"message": message, "category": category}
     # access private _WARNING_DETAILS because the attributes vary between Python versions
-    for attr_name in warnings.WarningMessage._WARNING_DETAILS:
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:  # type: ignore[attr-defined]
         if attr_name in ("message", "category"):
             continue
         kwargs[attr_name] = data[attr_name]
 
-    return warnings.WarningMessage(**kwargs)
-
-
-def report_unserialization_failure(type_name, report_name, reportdict):
-    from pprint import pprint
-
-    url = "https://github.com/pytest-dev/pytest-xdist/issues"
-    stream = py.io.TextIO()
-    pprint("-" * 100, stream=stream)
-    pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream)
-    pprint("report_name: %s" % report_name, stream=stream)
-    pprint(reportdict, stream=stream)
-    pprint("Please report this bug at %s" % url, stream=stream)
-    pprint("-" * 100, stream=stream)
-    assert 0, stream.getvalue()
+    return warnings.WarningMessage(**kwargs)  # type: ignore[arg-type]

testing/conftest.py

@@ -1,37 +1,30 @@
-import py
-import pytest
 import execnet
-
-
-@pytest.fixture(scope="session", autouse=True)
-def _ensure_imports():
-    # we import some modules because pytest-2.8's testdir fixture
-    # will unload all modules after each test and this cause
-    # (unknown) problems with execnet.Group()
-    execnet.Group
-    execnet.makegateway
-
+import pytest
+import shutil
+from typing import List
 
 pytest_plugins = "pytester"
 
-# rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
-
 
 @pytest.fixture(autouse=True)
-def _divert_atexit(request, monkeypatch):
+def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
     import atexit
 
     finalizers = []
 
-    def finish():
-        while finalizers:
-            finalizers.pop()()
+    def fake_register(func, *args, **kwargs):
+        finalizers.append((func, args, kwargs))
 
-    monkeypatch.setattr(atexit, "register", finalizers.append)
-    request.addfinalizer(finish)
+    monkeypatch.setattr(atexit, "register", fake_register)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser) -> None:
     parser.addoption(
         "--gx",
         action="append",
@@ -41,36 +34,28 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture
-def specssh(request):
+def specssh(request) -> str:
     return getspecssh(request.config)
 
 
-@pytest.fixture
-def testdir(testdir):
-    # pytest before 2.8 did not have a runpytest_subprocess
-    if not hasattr(testdir, "runpytest_subprocess"):
-        testdir.runpytest_subprocess = testdir.runpytest
-    return testdir
-
-
 # configuration information for tests
-def getgspecs(config):
+def getgspecs(config) -> List[execnet.XSpec]:
     return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
 
 
-def getspecssh(config):
+def getspecssh(config) -> str:  # type: ignore[return]
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.ssh:
-            if not py.path.local.sysfind("ssh"):
-                py.test.skip("command not found: ssh")
+            if not shutil.which("ssh"):
+                pytest.skip("command not found: ssh")
             return str(spec)
-    py.test.skip("need '--gx ssh=...'")
+    pytest.skip("need '--gx ssh=...'")
 
 
-def getsocketspec(config):
+def getsocketspec(config) -> execnet.XSpec:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.socket:
             return spec
-    py.test.skip("need '--gx socket=...'")
+    pytest.skip("need '--gx socket=...'")

testing/test_dsession.py

@@ -1,59 +1,44 @@
-from xdist.dsession import DSession
+from xdist.dsession import DSession, get_default_max_worker_restart
 from xdist.report import report_collection_diff
 from xdist.scheduler import EachScheduling, LoadScheduling
+from typing import Optional
 
-import py
 import pytest
 import execnet
 
-XSpec = execnet.XSpec
-
-
-def run(item, node, excinfo=None):
-    runner = item.config.pluginmanager.getplugin("runner")
-    rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call")
-    rep.node = node
-    return rep
-
 
 class MockGateway:
-    _count = 0
-
-    def __init__(self):
+    def __init__(self) -> None:
+        self._count = 0
         self.id = str(self._count)
         self._count += 1
 
 
 class MockNode:
-    def __init__(self):
-        self.sent = []
+    def __init__(self) -> None:
+        self.sent = []  # type: ignore[var-annotated]
         self.gateway = MockGateway()
         self._shutdown = False
 
-    def send_runtest_some(self, indices):
+    def send_runtest_some(self, indices) -> None:
         self.sent.extend(indices)
 
-    def send_runtest_all(self):
+    def send_runtest_all(self) -> None:
         self.sent.append("ALL")
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         self._shutdown = True
 
     @property
-    def shutting_down(self):
+    def shutting_down(self) -> bool:
         return self._shutdown
 
 
-def dumpqueue(queue):
-    while queue.qsize():
-        print(queue.get())
-
-
 class TestEachScheduling:
-    def test_schedule_load_simple(self, testdir):
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
         node2 = MockNode()
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         sched = EachScheduling(config)
         sched.add_node(node1)
         sched.add_node(node2)
@@ -74,9 +59,9 @@ class TestEachScheduling:
         sched.mark_test_complete(node2, 0)
         assert sched.tests_finished
 
-    def test_schedule_remove_node(self, testdir):
+    def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
-        config = testdir.parseconfig("--tx=popen")
+        config = pytester.parseconfig("--tx=popen")
         sched = EachScheduling(config)
         sched.add_node(node1)
         collection = ["a.py::test_1"]
@@ -93,8 +78,8 @@ class TestEachScheduling:
 
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self, testdir):
-        config = testdir.parseconfig("--tx=2*popen")
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -117,8 +102,8 @@ class TestLoadScheduling:
         sched.mark_test_complete(node1, node1.sent[0])
         assert sched.tests_finished
 
-    def test_schedule_batch_size(self, testdir):
-        config = testdir.parseconfig("--tx=2*popen")
+    def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -144,8 +129,8 @@ class TestLoadScheduling:
         assert node1.sent == [0, 2, 4, 5]
         assert not sched.pending
 
-    def test_schedule_fewer_tests_than_nodes(self, testdir):
-        config = testdir.parseconfig("--tx=2*popen")
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -164,8 +149,10 @@ class TestLoadScheduling:
         assert sent3 == []
         assert not sched.pending
 
-    def test_schedule_fewer_than_two_tests_per_node(self, testdir):
-        config = testdir.parseconfig("--tx=2*popen")
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
         sched.add_node(MockNode())
         sched.add_node(MockNode())
@@ -184,9 +171,9 @@ class TestLoadScheduling:
         assert sent3 == [2]
         assert not sched.pending
 
-    def test_add_remove_node(self, testdir):
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
         node = MockNode()
-        config = testdir.parseconfig("--tx=popen")
+        config = pytester.parseconfig("--tx=popen")
         sched = LoadScheduling(config)
         sched.add_node(node)
         collection = ["test_file.py::test_func"]
@@ -197,13 +184,13 @@ class TestLoadScheduling:
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
 
-    def test_different_tests_collected(self, testdir):
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
         """
         Test that LoadScheduling is reporting collection errors when
         different test ids are collected by workers.
         """
 
-        class CollectHook(object):
+        class CollectHook:
             """
             Dummy hook that stores collection reports.
             """
@@ -215,7 +202,7 @@ class TestLoadScheduling:
                 self.reports.append(report)
 
         collect_hook = CollectHook()
-        config = testdir.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=2*popen")
         config.pluginmanager.register(collect_hook, "collect_hook")
         node1 = MockNode()
         node2 = MockNode()
@@ -231,9 +218,9 @@ class TestLoadScheduling:
 
 
 class TestDistReporter:
-    @py.test.mark.xfail
-    def test_rsync_printing(self, testdir, linecomp):
-        config = testdir.parseconfig()
+    @pytest.mark.xfail
+    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
+        config = pytester.parseconfig()
         from _pytest.pytest_terminal import TerminalReporter
 
         rep = TerminalReporter(config, file=linecomp.stringio)
@@ -258,17 +245,35 @@ class TestDistReporter:
         # linecomp.assert_contains_lines([
         #     "*X1*popen*xyz*2.5*"
         # ])
-        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
+        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])  # type: ignore[attr-defined]
         linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
 
 
-def test_report_collection_diff_equal():
+def test_report_collection_diff_equal() -> None:
     """Test reporting of equal collections."""
     from_collection = to_collection = ["aaa", "bbb", "ccc"]
     assert report_collection_diff(from_collection, to_collection, 1, 2) is None
 
 
-def test_report_collection_diff_different():
+def test_default_max_worker_restart() -> None:
+    class config:
+        class option:
+            maxworkerrestart: Optional[str] = None
+            numprocesses: int = 0
+
+    assert get_default_max_worker_restart(config) is None
+
+    config.option.numprocesses = 2
+    assert get_default_max_worker_restart(config) == 8
+
+    config.option.maxworkerrestart = "1"
+    assert get_default_max_worker_restart(config) == 1
+
+    config.option.maxworkerrestart = "0"
+    assert get_default_max_worker_restart(config) == 0
+
+
+def test_report_collection_diff_different() -> None:
     """Test reporting of different collections."""
     from_collection = ["aaa", "bbb", "ccc", "YYY"]
     to_collection = ["aZa", "bbb", "XXX", "ccc"]
@@ -293,8 +298,8 @@ def test_report_collection_diff_different():
 
 
 @pytest.mark.xfail(reason="duplicate test ids not supported yet")
-def test_pytest_issue419(testdir):
-    testdir.makepyfile(
+def test_pytest_issue419(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
         """
         import pytest
 
@@ -303,6 +308,6 @@ def test_pytest_issue419(testdir):
             pass
     """
     )
-    reprec = testdir.inline_run("-n1")
+    reprec = pytester.inline_run("-n1")
     reprec.assertoutcome(passed=2)
     assert 0

testing/test_looponfail.py

@@ -1,90 +1,106 @@
 import py
 import pytest
-from pkg_resources import parse_version
+import shutil
+import textwrap
+from pathlib import Path
 
 from xdist.looponfail import RemoteControl
 from xdist.looponfail import StatRecorder
 
 
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
+
+
 class TestStatRecorder:
-    def test_filechange(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
-        sd = StatRecorder([tmp])
+    def test_filechange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
+        sd = StatRecorder([py.path.local(tmp)])
         changed = sd.check()
         assert not changed
 
-        hello.write("world")
+        hello.write_text("world")
         changed = sd.check()
         assert changed
 
-        (hello + "c").write("hello")
+        hello.with_suffix(".pyc").write_text("hello")
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp / "new.py"
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "b", "c.py").ensure()
+        tmp.joinpath("a", "b").mkdir(parents=True)
+        tmp.joinpath("a", "b", "c.py").touch()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "c.txt").ensure()
+        tmp.joinpath("a", "c.txt").touch()
         changed = sd.check()
         assert changed
         changed = sd.check()
         assert not changed
 
-        tmp.join("a").remove()
+        shutil.rmtree(str(tmp.joinpath("a")))
         changed = sd.check()
         assert changed
 
-    def test_dirchange(self, tmpdir):
-        tmp = tmpdir
-        tmp.ensure("dir", "hello.py")
-        sd = StatRecorder([tmp])
-        assert not sd.fil(tmp.join("dir"))
+    def test_dirchange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        tmp.joinpath("dir").mkdir()
+        tmp.joinpath("dir", "hello.py").touch()
+        sd = StatRecorder([py.path.local(tmp)])
+        assert not sd.fil(py.path.local(tmp / "dir"))
 
-    def test_filechange_deletion_race(self, tmpdir, monkeypatch):
-        tmp = tmpdir
-        sd = StatRecorder([tmp])
+    def test_filechange_deletion_race(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        tmp = tmp_path
+        pytmp = py.path.local(tmp)
+        sd = StatRecorder([pytmp])
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp.joinpath("new.py")
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(tmp, "visit", lambda *args: [p])
+        monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
 
         changed = sd.check()
         assert changed
 
-    def test_pycremoval(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
-        sd = StatRecorder([tmp])
+    def test_pycremoval(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
+        sd = StatRecorder([py.path.local(tmp)])
         changed = sd.check()
         assert not changed
 
-        pycfile = hello + "c"
-        pycfile.ensure()
-        hello.write("world")
+        pycfile = hello.with_suffix(".pyc")
+        pycfile.touch()
+        hello.write_text("world")
         changed = sd.check()
         assert changed
-        assert not pycfile.check()
+        assert not pycfile.exists()
 
-    def test_waitonchange(self, tmpdir, monkeypatch):
-        tmp = tmpdir
-        sd = StatRecorder([tmp])
+    def test_waitonchange(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        tmp = tmp_path
+        sd = StatRecorder([py.path.local(tmp)])
 
         ret_values = [True, False]
         monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -93,63 +109,74 @@ class TestStatRecorder:
 
 
 class TestRemoteControl:
-    def test_nofailures(self, testdir):
-        item = testdir.getitem("def test_func(): pass\n")
+    def test_nofailures(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func(): pass\n")
         control = RemoteControl(item.config)
         control.setup()
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failures_somewhere(self, testdir):
-        item = testdir.getitem("def test_func():\n assert 0\n")
+    def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
         failures = control.runsession()
         assert failures
         control.setup()
-        item.fspath.write("def test_func():\n assert 1\n")
-        removepyc(item.fspath)
+        item_path = item.path if PYTEST_GTE_7 else Path(item.fspath)  # type: ignore[attr-defined]
+        item_path.write_text("def test_func():\n assert 1\n")
+        removepyc(item_path)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failure_change(self, testdir):
-        modcol = testdir.getitem(
-            """
-            def test_func():
-                assert 0
-        """
+    def test_failure_change(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
         )
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol.fspath.write(
-            py.code.Source(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)  # type: ignore[attr-defined]
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 1
+                def test_new():
+                    assert 0
                 """
-            def test_func():
-                assert 1
-            def test_new():
-                assert 0
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         control.loop_once()
         assert not control.failures
         control.loop_once()
         assert control.failures
         assert str(control.failures).find("test_new") != -1
 
-    def test_failure_subdir_no_init(self, testdir):
-        modcol = testdir.getitem(
-            """
-            def test_func():
-                assert 0
-        """
+    def test_failure_subdir_no_init(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
         )
-        parent = modcol.fspath.dirpath().dirpath()
-        parent.chdir()
+        if PYTEST_GTE_7:
+            parent = modcol.path.parent.parent  # type: ignore[attr-defined]
+        else:
+            parent = Path(modcol.fspath.dirpath().dirpath())
+        monkeypatch.chdir(parent)
         modcol.config.args = [
-            py.path.local(x).relto(parent) for x in modcol.config.args
+            str(Path(x).relative_to(parent)) for x in modcol.config.args
         ]
         control = RemoteControl(modcol.config)
         control.loop_once()
@@ -159,57 +186,63 @@ class TestRemoteControl:
 
 
 class TestLooponFailing:
-    def test_looponfail_from_fail_to_ok(self, testdir):
-        modcol = testdir.getmodulecol(
-            """
-            def test_one():
-                x = 0
-                assert x == 1
-            def test_two():
-                assert 1
-        """
+    def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                def test_two():
+                    assert 1
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-        modcol.fspath.write(
-            py.code.Source(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1
+                def test_two():
+                    assert 1
                 """
-            def test_one():
-                assert 1
-            def test_two():
-                assert 1
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert not remotecontrol.failures
 
-    def test_looponfail_from_one_to_two_tests(self, testdir):
-        modcol = testdir.getmodulecol(
-            """
-            def test_one():
-                assert 0
-        """
+    def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
         assert "test_one" in remotecontrol.failures[0]
 
-        modcol.fspath.write(
-            py.code.Source(
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1 # passes now
+                def test_two():
+                    assert 0 # new and fails
                 """
-            def test_one():
-                assert 1 # passes now
-            def test_two():
-                assert 0 # new and fails
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
         remotecontrol.loop_once()
@@ -217,47 +250,49 @@ class TestLooponFailing:
         assert "test_one" not in remotecontrol.failures[0]
         assert "test_two" in remotecontrol.failures[0]
 
-    @pytest.mark.xfail(
-        parse_version(pytest.__version__) >= parse_version("3.1"),
-        reason="broken by pytest 3.1+",
-        strict=True,
-    )
-    def test_looponfail_removed_test(self, testdir):
-        modcol = testdir.getmodulecol(
-            """
-            def test_one():
-                assert 0
-            def test_two():
-                assert 0
-        """
+    @pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
+    def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                def test_two():
+                    assert 0
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 2
 
-        modcol.fspath.write(
-            py.code.Source(
+        modcol.path.write_text(
+            textwrap.dedent(
+                """
+                def test_xxx(): # renamed test
+                    assert 0
+                def test_two():
+                    assert 1 # pass now
                 """
-            def test_xxx(): # renamed test
-                assert 0
-            def test_two():
-                assert 1 # pass now
-        """
             )
         )
-        removepyc(modcol.fspath)
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
 
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-    def test_looponfail_multiple_errors(self, testdir, monkeypatch):
-        modcol = testdir.getmodulecol(
-            """
-            def test_one():
-                assert 0
-        """
+    def test_looponfail_multiple_errors(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
         )
         remotecontrol = RemoteControl(modcol.config)
         orig_runsession = remotecontrol.runsession
@@ -274,55 +309,59 @@ class TestLooponFailing:
 
 
 class TestFunctional:
-    def test_fail_to_ok(self, testdir):
-        p = testdir.makepyfile(
-            """
-            def test_one():
-                x = 0
-                assert x == 1
-        """
+    def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                """
+            )
         )
-        # p = testdir.mkdir("sub").join(p1.basename)
+        # p = pytester.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+        child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
         child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
-        p.write(
-            py.code.Source(
+        p.write_text(
+            textwrap.dedent(
                 """
-            def test_one():
-                x = 1
-                assert x == 1
-        """
-            )
+                def test_one():
+                    x = 1
+                    assert x == 1
+                """
+            ),
         )
         child.expect(".*1 passed.*")
         child.kill(15)
 
-    def test_xfail_passes(self, testdir):
-        p = testdir.makepyfile(
-            """
-            import py
-            @py.test.mark.xfail
-            def test_one():
-                pass
-        """
+    def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                import pytest
+                @pytest.mark.xfail
+                def test_one():
+                    pass
+                """
+            )
         )
-        child = testdir.spawn_pytest("-f %s" % p)
+        child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
         child.kill(15)
 
 
-def removepyc(path):
+def removepyc(path: Path) -> None:
     # XXX damn those pyc files
-    pyc = path + "c"
-    if pyc.check():
-        pyc.remove()
-    c = path.dirpath("__pycache__")
-    if c.check():
-        c.remove()
+    pyc = path.with_suffix(".pyc")
+    if pyc.exists():
+        pyc.unlink()
+    c = path.parent / "__pycache__"
+    if c.exists():
+        shutil.rmtree(c)

testing/test_newhooks.py

@@ -3,8 +3,8 @@ import pytest
 
 class TestHooks:
     @pytest.fixture(autouse=True)
-    def create_test_file(self, testdir):
-        testdir.makepyfile(
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
             """
             import os
             def test_a(): pass
@@ -13,38 +13,41 @@ class TestHooks:
         """
         )
 
-    def test_runtest_logreport(self, testdir):
+    def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
         """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain "node", "nodeid" and "worker_id" attributes. (#8)
+        with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
         """
-        testdir.makeconftest(
+        pytester.makeconftest(
             """
             def pytest_runtest_logreport(report):
                 if hasattr(report, 'node'):
                     if report.when == "call":
                         workerid = report.node.workerinput['workerid']
+                        testrunuid = report.node.workerinput['testrunuid']
                         if workerid != report.worker_id:
                             print("HOOK: Worker id mismatch: %s %s"
                                    % (workerid, report.worker_id))
+                        elif testrunuid != report.testrun_uid:
+                            print("HOOK: Testrun uid mismatch: %s %s"
+                                   % (testrunuid, report.testrun_uid))
                         else:
-                            print("HOOK: %s %s"
-                                   % (report.nodeid, report.worker_id))
+                            print("HOOK: %s %s %s"
+                                   % (report.nodeid, report.worker_id, report.testrun_uid))
         """
         )
-        res = testdir.runpytest("-n1", "-s")
+        res = pytester.runpytest("-n1", "-s")
         res.stdout.fnmatch_lines(
             [
-                "*HOOK: test_runtest_logreport.py::test_a gw0*",
-                "*HOOK: test_runtest_logreport.py::test_b gw0*",
-                "*HOOK: test_runtest_logreport.py::test_c gw0*",
+                "*HOOK: test_runtest_logreport.py::test_a gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_b gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_c gw0 *",
                 "*3 passed*",
             ]
         )
 
-    def test_node_collection_finished(self, testdir):
-        """Test pytest_xdist_node_collection_finished hook (#8).
-        """
-        testdir.makeconftest(
+    def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
+        pytester.makeconftest(
             """
             def pytest_xdist_node_collection_finished(node, ids):
                 workerid = node.workerinput['workerid']
@@ -52,8 +55,42 @@ class TestHooks:
                 print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
         """
         )
-        res = testdir.runpytest("-n2", "-s")
+        res = pytester.runpytest("-n2", "-s")
         res.stdout.fnmatch_lines_random(
             ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
         )
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])

testing/test_plugin.py

@@ -1,41 +1,53 @@
-import py
+from contextlib import suppress
+from pathlib import Path
+
 import execnet
 from xdist.workermanage import NodeManager
 
+import pytest
 
-def test_dist_incompatibility_messages(testdir):
-    result = testdir.runpytest("--pdb", "--looponfail")
+
+def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
+    result = pytester.runpytest("--pdb", "--looponfail")
     assert result.ret != 0
-    result = testdir.runpytest("--pdb", "-n", "3")
+    result = pytester.runpytest("--pdb", "-n", "3")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
-    result = testdir.runpytest("--pdb", "-d", "--tx", "popen")
+    result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
 
 
-def test_dist_options(testdir):
+def test_dist_options(pytester: pytest.Pytester) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = testdir.parseconfigure("-n 2")
+    config = pytester.parseconfigure("-n 2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("--numprocesses", "2")
+    config = pytester.parseconfigure("--numprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
-    config = testdir.parseconfigure("-d")
+    config = pytester.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"
 
 
-def test_auto_detect_cpus(testdir, monkeypatch):
+def test_auto_detect_cpus(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
     import os
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
 
     if hasattr(os, "sched_getaffinity"):
         monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
@@ -46,52 +58,116 @@ def test_auto_detect_cpus(testdir, monkeypatch):
 
         monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
 
-    config = testdir.parseconfigure("-n2")
+    config = pytester.parseconfigure("-n2")
     assert config.getoption("numprocesses") == 2
 
-    config = testdir.parseconfigure("-nauto")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 99
 
+    config = pytester.parseconfigure("-nauto", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
+    config = pytester.parseconfigure("-nlogical", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
-    config = testdir.parseconfigure("-nauto")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 2
 
 
-def test_boxed_with_collect_only(testdir):
+def test_auto_detect_cpus_psutil(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = testdir.parseconfigure("-n1", "--boxed")
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_hook_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
+def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    config = pytester.parseconfigure("-n1", "--boxed")
     check_options(config)
     assert config.option.forked
 
-    config = testdir.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--collect-only")
     check_options(config)
     assert not config.option.forked
 
-    config = testdir.parseconfigure("-n1", "--boxed", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
     check_options(config)
     assert config.option.forked
 
 
-def test_dsession_with_collect_only(testdir):
+def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
     from xdist.plugin import pytest_cmdline_main as check_options
     from xdist.plugin import pytest_configure as configure
 
-    config = testdir.parseconfigure("-n1")
+    config = pytester.parseconfigure("-n1")
     check_options(config)
     configure(config)
     assert config.pluginmanager.hasplugin("dsession")
 
-    config = testdir.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-n1", "--collect-only")
     check_options(config)
     configure(config)
     assert not config.pluginmanager.hasplugin("dsession")
 
 
+def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    nm = NodeManager(config)
+    assert nm.testrunuid == "test123"
+
+
+def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--tx=popen")
+    nm = NodeManager(config)
+    assert len(nm.testrunuid) == 32
+
+
 class TestDistOptions:
-    def test_getxspecs(self, testdir):
-        config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
+    def test_getxspecs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
         nodemanager = NodeManager(config)
         xspecs = nodemanager._getxspecs()
         assert len(xspecs) == 2
@@ -99,39 +175,39 @@ class TestDistOptions:
         assert xspecs[0].popen
         assert xspecs[1].ssh == "xyz"
 
-    def test_xspecs_multiplied(self, testdir):
-        config = testdir.parseconfigure("--tx=3*popen")
+    def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=3*popen")
         xspecs = NodeManager(config)._getxspecs()
         assert len(xspecs) == 3
         assert xspecs[1].popen
 
-    def test_getrsyncdirs(self, testdir):
-        config = testdir.parseconfigure("--rsyncdir=" + str(testdir.tmpdir))
+    def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
         nm = NodeManager(config, specs=[execnet.XSpec("popen")])
         assert not nm._getrsyncdirs()
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert nm.roots
-        assert testdir.tmpdir in nm.roots
+        assert pytester.path in nm.roots
 
-    def test_getrsyncignore(self, testdir):
-        config = testdir.parseconfigure("--rsyncignore=fo*")
+    def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncignore=fo*")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert "fo*" in nm.rsyncoptions["ignores"]
 
-    def test_getrsyncdirs_with_conftest(self, testdir):
-        p = py.path.local()
-        for bn in "x y z".split():
-            p.mkdir(bn)
-        testdir.makeini(
+    def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
+        p = Path.cwd()
+        for bn in ("x", "y", "z"):
+            p.joinpath(bn).mkdir()
+        pytester.makeini(
             """
             [pytest]
             rsyncdirs= x
         """
         )
-        config = testdir.parseconfigure(testdir.tmpdir, "--rsyncdir=y", "--rsyncdir=z")
+        config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
         roots = nm._getrsyncdirs()
         # assert len(roots) == 3 + 1 # pylib
-        assert py.path.local("y") in roots
-        assert py.path.local("z") in roots
-        assert testdir.tmpdir.join("x") in roots
+        assert Path("y").resolve() in roots
+        assert Path("z").resolve() in roots
+        assert pytester.path.joinpath("x") in roots

testing/test_remote.py

@@ -1,14 +1,14 @@
-import py
 import pprint
+import py
 import pytest
 import sys
+import uuid
 
-from xdist.workermanage import WorkerController, unserialize_report
-from xdist.remote import serialize_report
+from xdist.workermanage import WorkerController
 import execnet
 import marshal
 
-from six.moves.queue import Queue
+from queue import Queue
 
 WAIT_TIMEOUT = 10.0
 
@@ -26,25 +26,26 @@ class EventCall:
         self.name, self.kwargs = eventcall
 
     def __str__(self):
-        return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
+        return "<EventCall {}(**{})>".format(self.name, self.kwargs)
 
 
 class WorkerSetup:
     use_callback = False
 
-    def __init__(self, request, testdir):
+    def __init__(self, request, pytester: pytest.Pytester) -> None:
         self.request = request
-        self.testdir = testdir
-        self.events = Queue()
+        self.pytester = pytester
+        self.events = Queue()  # type: ignore[var-annotated]
 
-    def setup(self,):
-        self.testdir.chdir()
+    def setup(self) -> None:
+        self.pytester.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
         self.gateway = execnet.makegateway()
-        self.config = config = self.testdir.parseconfigure()
+        self.config = config = self.pytester.parseconfigure()
         putevent = self.use_callback and self.events.put or None
 
         class DummyMananger:
+            testrunuid = uuid.uuid4().hex
             specs = [0, 1]
 
         self.slp = WorkerController(DummyMananger, self.gateway, config, putevent)
@@ -60,209 +61,41 @@ class WorkerSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping %s" % (ev,))
+            print("skipping {}".format(ev))
 
     def sendcommand(self, name, **kwargs):
         self.slp.sendcommand(name, **kwargs)
 
 
 @pytest.fixture
-def worker(request, testdir):
-    return WorkerSetup(request, testdir)
+def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
+    return WorkerSetup(request, pytester)
 
 
 @pytest.mark.xfail(reason="#59")
-def test_remoteinitconfig(testdir):
+def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
     from xdist.remote import remote_initconfig
 
-    config1 = testdir.parseconfig()
+    config1 = pytester.parseconfig()
     config2 = remote_initconfig(config1.option.__dict__, config1.args)
     assert config2.option.__dict__ == config1.option.__dict__
     assert config2.pluginmanager.getplugin("terminal") in (-1, None)
 
 
-class TestReportSerialization:
-    def test_xdist_longrepr_to_str_issue_241(self, testdir):
-        testdir.makepyfile(
-            """
-            import os
-            def test_a(): assert False
-            def test_b(): pass
-        """
-        )
-        testdir.makeconftest(
-            """
-            def pytest_runtest_logreport(report):
-                print(report.longrepr)
-        """
-        )
-        res = testdir.runpytest("-n1", "-s")
-        res.stdout.fnmatch_lines(["*1 failed, 1 passed *"])
-
-    def test_xdist_report_longrepr_reprcrash_130(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-                    import py
-                    def test_fail(): assert False, 'Expected Message'
-                """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        added_section = ("Failure Metadata", str("metadata metadata"), "*")
-        rep.longrepr.sections.append(added_section)
-        d = serialize_report(rep)
-        check_marshallable(d)
-        a = unserialize_report("testreport", d)
-        # Check assembled == rep
-        assert a.__dict__.keys() == rep.__dict__.keys()
-        for key in rep.__dict__.keys():
-            if key != "longrepr":
-                assert getattr(a, key) == getattr(rep, key)
-        assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno
-        assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message
-        assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path
-        assert rep.longrepr.reprtraceback.entrysep == a.longrepr.reprtraceback.entrysep
-        assert (
-            rep.longrepr.reprtraceback.extraline == a.longrepr.reprtraceback.extraline
-        )
-        assert rep.longrepr.reprtraceback.style == a.longrepr.reprtraceback.style
-        assert rep.longrepr.sections == a.longrepr.sections
-        # Missing section attribute PR171
-        assert added_section in a.longrepr.sections
-
-    def test_reprentries_serialization_170(self, testdir):
-        from _pytest._code.code import ReprEntry
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry():
-                                x = 0
-                                assert x
-                        """,
-            "--showlocals",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntry)
-            assert rep_entries[i].lines == a_entries[i].lines
-            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
-            assert (
-                rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
-            )
-            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
-            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
-            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
-            assert rep_entries[i].style == a_entries[i].style
-
-    def test_reprentries_serialization_196(self, testdir):
-        from _pytest._code.code import ReprEntryNative
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry_native():
-                                x = 0
-                                assert x
-                        """,
-            "--tb=native",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntryNative)
-            assert rep_entries[i].lines == a_entries[i].lines
-
-    def test_itemreport_outcomes(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-            import py
-            def test_pass(): pass
-            def test_fail(): 0/0
-            @py.test.mark.skipif("True")
-            def test_skip(): pass
-            def test_skip_imperative():
-                py.test.skip("hello")
-            @py.test.mark.xfail("True")
-            def test_xfail(): 0/0
-            def test_xfail_imperative():
-                py.test.xfail("hello")
-        """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 17  # with setup/teardown "passed" reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("testreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if newrep.skipped and not hasattr(newrep, "wasxfail"):
-                assert len(newrep.longrepr) == 3
-            assert newrep.outcome == rep.outcome
-            assert newrep.when == rep.when
-            assert newrep.keywords == rep.keywords
-            if rep.failed:
-                assert newrep.longreprtext == rep.longreprtext
-
-    def test_collectreport_passed(self, testdir):
-        reprec = testdir.inline_runsource("def test_func(): pass")
-        reports = reprec.getreports("pytest_collectreport")
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-
-    def test_collectreport_fail(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-    def test_extended_report_deserialization(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            rep.extra = True
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.extra
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-
 class TestWorkerInteractor:
-    def test_basic_collect_and_runtests(self, worker):
-        worker.testdir.makepyfile(
+    @pytest.fixture
+    def unserialize_report(self, pytestconfig):
+        def unserialize(data):
+            return pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
+
+        return unserialize
+
+    def test_basic_collect_and_runtests(
+        self, worker: WorkerSetup, unserialize_report
+    ) -> None:
+        worker.pytester.makepyfile(
             """
             def test_func():
                 pass
@@ -275,7 +108,7 @@ class TestWorkerInteractor:
         assert ev.name == "collectionstart"
         assert not ev.kwargs
         ev = worker.popevent("collectionfinish")
-        assert ev.kwargs["topdir"] == worker.testdir.tmpdir
+        assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
         ids = ev.kwargs["ids"]
         assert len(ids) == 1
         worker.sendcommand("runtests", indices=list(range(len(ids))))
@@ -286,15 +119,15 @@ class TestWorkerInteractor:
         ev = worker.popevent("testreport")  # setup
         ev = worker.popevent("testreport")
         assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, worker):
-        worker.testdir.makepyfile(
+    def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
             """
             import pytest
             pytest.skip("hello", allow_module_level=True)
@@ -305,26 +138,26 @@ class TestWorkerInteractor:
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
         assert rep.longrepr[2] == "Skipped: hello"
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, worker):
-        worker.testdir.makepyfile("""aasd qwe""")
+    def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile("""aasd qwe""")
         worker.setup()
         ev = worker.popevent("collectionstart")
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.failed
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, worker):
-        worker.testdir.makepyfile(
+    def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
             """
             def test_func(): pass
             def test_func2(): pass
@@ -345,22 +178,24 @@ class TestWorkerInteractor:
             for i in range(3):  # setup/call/teardown
                 ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
-                rep = unserialize_report(ev.name, ev.kwargs["data"])
+                rep = unserialize_report(ev.kwargs["data"])
                 assert rep.nodeid.endswith(func)
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, worker):
-        py.test.xfail("implement a simple test for event production")
-        assert not worker.use_callback
-        worker.testdir.makepyfile(
+    def test_happy_run_events_converted(
+        self, pytester: pytest.Pytester, worker: WorkerSetup
+    ) -> None:
+        pytest.xfail("implement a simple test for event production")
+        assert not worker.use_callback  # type: ignore[unreachable]
+        worker.pytester.makepyfile(
             """
             def test_func():
                 pass
         """
         )
         worker.setup()
-        hookrec = testdir.getreportrecorder(worker.config)
+        hookrec = pytester.getreportrecorder(worker.config)
         for data in worker.slp.channel:
             worker.slp.process_from_remote(data)
         worker.slp.process_from_remote(worker.slp.ENDMARK)
@@ -376,7 +211,9 @@ class TestWorkerInteractor:
             ]
         )
 
-    def test_process_from_remote_error_handling(self, worker, capsys):
+    def test_process_from_remote_error_handling(
+        self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
+    ) -> None:
         worker.use_callback = True
         worker.setup()
         worker.slp.process_from_remote(("<nonono>", ()))
@@ -386,22 +223,23 @@ class TestWorkerInteractor:
         assert ev.name == "errordown"
 
 
-def test_remote_env_vars(testdir):
-    testdir.makepyfile(
+def test_remote_env_vars(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
         """
         import os
         def test():
+            assert len(os.environ['PYTEST_XDIST_TESTRUNUID']) == 32
             assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     """
     )
-    result = testdir.runpytest("-n2", "--max-worker-restart=0")
+    result = pytester.runpytest("-n2", "--max-worker-restart=0")
     assert result.ret == 0
 
 
-def test_remote_inner_argv(testdir):
+def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
     """Test/document the behavior due to execnet using `python -c`."""
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import sys
 
@@ -409,14 +247,14 @@ def test_remote_inner_argv(testdir):
             assert sys.argv == ["-c"]
         """
     )
-    result = testdir.runpytest("-n1")
+    result = pytester.runpytest("-n1")
     assert result.ret == 0
 
 
-def test_remote_mainargv(testdir):
+def test_remote_mainargv(pytester: pytest.Pytester) -> None:
     outer_argv = sys.argv
 
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         def test_mainargv(request):
             assert request.config.workerinput["mainargv"] == {!r}
@@ -424,14 +262,14 @@ def test_remote_mainargv(testdir):
             outer_argv
         )
     )
-    result = testdir.runpytest("-n1")
+    result = pytester.runpytest("-n1")
     assert result.ret == 0
 
 
-def test_remote_usage_prog(testdir, request):
+def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
     if not hasattr(request.config._parser, "prog"):
         pytest.skip("prog not available in config parser")
-    testdir.makeconftest(
+    pytester.makeconftest(
         """
         import pytest
 
@@ -446,7 +284,7 @@ def test_remote_usage_prog(testdir, request):
             config_parser = config._parser
     """
     )
-    testdir.makepyfile(
+    pytester.makepyfile(
         """
         import sys
 
@@ -455,8 +293,20 @@ def test_remote_usage_prog(testdir, request):
     """
     )
 
-    result = testdir.runpytest_subprocess("-n1")
+    result = pytester.runpytest_subprocess("-n1")
     assert result.ret == 1
-    result.stdout.fnmatch_lines(
-        ["usage: pytest.py *", "pytest.py: error: my_usage_error"]
+    result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(pytester: pytest.Pytester) -> None:
+    """Work around sys.path differences due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
     )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0

testing/test_slavemanage.py

@@ -1,273 +0,0 @@
-from __future__ import print_function
-import py
-import pytest
-import textwrap
-import execnet
-from _pytest.pytester import HookRecorder
-from xdist import workermanage, newhooks
-from xdist.workermanage import HostRSync, NodeManager
-
-pytest_plugins = "pytester"
-
-
-@pytest.fixture
-def hookrecorder(request, config):
-    hookrecorder = HookRecorder(config.pluginmanager)
-    if hasattr(hookrecorder, "start_recording"):
-        hookrecorder.start_recording(newhooks)
-    request.addfinalizer(hookrecorder.finish_recording)
-    return hookrecorder
-
-
-@pytest.fixture
-def config(testdir):
-    return testdir.parseconfig()
-
-
-@pytest.fixture
-def mysetup(tmpdir):
-    class mysetup:
-        source = tmpdir.mkdir("source")
-        dest = tmpdir.mkdir("dest")
-
-    return mysetup()
-
-
-@pytest.fixture
-def workercontroller(monkeypatch):
-    class MockController(object):
-        def __init__(self, *args):
-            pass
-
-        def setup(self):
-            pass
-
-    monkeypatch.setattr(workermanage, "WorkerController", MockController)
-    return MockController
-
-
-class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, config):
-        gm = NodeManager(config, ["popen"])
-        assert gm.specs[0].chdir is None
-
-    def test_default_chdir(self, config):
-        specs = ["ssh=noco", "socket=xyz"]
-        for spec in NodeManager(config, specs).specs:
-            assert spec.chdir == "pyexecnetcache"
-        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
-            assert spec.chdir == "abc"
-
-    def test_popen_makegateway_events(self, config, hookrecorder, workercontroller):
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.setup_nodes(None)
-        call = hookrecorder.popcall("pytest_xdist_setupnodes")
-        assert len(call.specs) == 2
-
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.spec == execnet.XSpec("popen")
-        assert call.gateway.id == "gw0"
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.id == "gw1"
-        assert len(hm.group) == 2
-        hm.teardown_nodes()
-        assert not len(hm.group)
-
-    def test_popens_rsync(self, config, mysetup, workercontroller):
-        source = mysetup.source
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.setup_nodes(None)
-        assert len(hm.group) == 2
-        for gw in hm.group:
-
-            class pseudoexec:
-                args = []
-
-                def __init__(self, *args):
-                    self.args.extend(args)
-
-                def waitclose(self):
-                    pass
-
-            gw.remote_exec = pseudoexec
-        notifications = []
-        for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
-        assert not notifications
-        hm.teardown_nodes()
-        assert not len(hm.group)
-        assert "sys.path.insert" in gw.remote_exec.args[0]
-
-    def test_rsync_popen_with_path(self, config, mysetup, workercontroller):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
-        hm.setup_nodes(None)
-        source.ensure("dir1", "dir2", "hello")
-        notifications = []
-        for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
-        assert len(notifications) == 1
-        assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
-        hm.teardown_nodes()
-        dest = dest.join(source.basename)
-        assert dest.join("dir1").check()
-        assert dest.join("dir1", "dir2").check()
-        assert dest.join("dir1", "dir2", "hello").check()
-
-    def test_rsync_same_popen_twice(
-        self, config, mysetup, hookrecorder, workercontroller
-    ):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
-        hm.roots = []
-        hm.setup_nodes(None)
-        source.ensure("dir1", "dir2", "hello")
-        gw = hm.group[0]
-        hm.rsync(gw, source)
-        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
-        assert call.source == source
-        assert len(call.gateways) == 1
-        assert call.gateways[0] in hm.group
-        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
-
-
-class TestHRSync:
-    def test_hrsync_filter(self, mysetup):
-        source, _ = mysetup.source, mysetup.dest  # noqa
-        source.ensure("dir", "file.txt")
-        source.ensure(".svn", "entries")
-        source.ensure(".somedotfile", "moreentries")
-        source.ensure("somedir", "editfile~")
-        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        files = list(source.visit(rec=syncer.filter, fil=syncer.filter))
-        assert len(files) == 3
-        basenames = [x.basename for x in files]
-        assert "dir" in basenames
-        assert "file.txt" in basenames
-        assert "somedir" in basenames
-
-    def test_hrsync_one_host(self, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        gw = execnet.makegateway("popen//chdir=%s" % dest)
-        finished = []
-        rsync = HostRSync(source)
-        rsync.add_target_host(gw, finished=lambda: finished.append(1))
-        source.join("hello.py").write("world")
-        rsync.send()
-        gw.exit()
-        assert dest.join(source.basename, "hello.py").check()
-        assert len(finished) == 1
-
-
-class TestNodeManager:
-    @py.test.mark.xfail(run=False)
-    def test_rsync_roots_no_roots(self, testdir, mysetup):
-        mysetup.source.ensure("dir1", "file1").write("hello")
-        config = testdir.parseconfig(mysetup.source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
-        # assert nodemanager.config.topdir == source == config.topdir
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        p, = nodemanager.gwmanager.multi_exec(
-            "import os ; channel.send(os.getcwd())"
-        ).receive_each()
-        p = py.path.local(p)
-        print("remote curdir", p)
-        assert p == mysetup.dest.join(config.topdir.basename)
-        assert p.join("dir1").check()
-        assert p.join("dir1", "file1").check()
-
-    def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller):
-        source, dest = mysetup.source, mysetup.dest
-        dir1 = mysetup.source.mkdir("dir1")
-        dir2 = dir1.mkdir("dir2")
-        dir2.ensure("hello")
-        for rsyncroot in (dir1, source):
-            dest.remove()
-            nodemanager = NodeManager(
-                testdir.parseconfig(
-                    "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
-                )
-            )
-            nodemanager.setup_nodes(None)  # calls .rsync_roots()
-            if rsyncroot == source:
-                dest = dest.join("source")
-            assert dest.join("dir1").check()
-            assert dest.join("dir1", "dir2").check()
-            assert dest.join("dir1", "dir2", "hello").check()
-            nodemanager.teardown_nodes()
-
-    def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        source.ensure("dir1", "somefile", dir=1)
-        dir2.ensure("hello")
-        source.ensure("bogusdir", "file")
-        source.join("tox.ini").write(
-            textwrap.dedent(
-                """
-            [pytest]
-            rsyncdirs=dir1/dir2
-        """
-            )
-        )
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.setup_nodes(None)  # calls .rsync_roots()
-        assert dest.join("dir2").check()
-        assert not dest.join("dir1").check()
-        assert not dest.join("bogus").check()
-
-    def test_rsyncignore(self, testdir, mysetup, workercontroller):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        source.ensure("dir5", "dir6", "bogus")
-        source.ensure("dir5", "file")
-        dir2.ensure("hello")
-        source.ensure("foo", "bar")
-        source.ensure("bar", "foo")
-        source.join("tox.ini").write(
-            textwrap.dedent(
-                """
-            [pytest]
-            rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6 foo*
-        """
-            )
-        )
-        config = testdir.parseconfig(source)
-        config.option.rsyncignore = ["bar"]
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.setup_nodes(None)  # calls .rsync_roots()
-        assert dest.join("dir1").check()
-        assert not dest.join("dir1", "dir2").check()
-        assert dest.join("dir5", "file").check()
-        assert not dest.join("dir6").check()
-        assert not dest.join("foo").check()
-        assert not dest.join("bar").check()
-
-    def test_optimise_popen(self, testdir, mysetup, workercontroller):
-        source = mysetup.source
-        specs = ["popen"] * 3
-        source.join("conftest.py").write("rsyncdirs = ['a']")
-        source.ensure("a", dir=1)
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, specs)
-        nodemanager.setup_nodes(None)  # calls .rysnc_roots()
-        for gwspec in nodemanager.specs:
-            assert gwspec._samefilesystem()
-            assert not gwspec.chdir
-
-    def test_ssh_setup_nodes(self, specssh, testdir):
-        testdir.makepyfile(
-            __init__="",
-            test_x="""
-            def test_one():
-                pass
-        """,
-        )
-        reprec = testdir.inline_run(
-            "-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir
-        )
-        rep, = reprec.getreports("pytest_runtest_logreport")
-        assert rep.passed

testing/test_workermanage.py

@@ -0,0 +1,328 @@
+import execnet
+import py
+import pytest
+import shutil
+import textwrap
+from pathlib import Path
+from xdist import workermanage
+from xdist.workermanage import HostRSync, NodeManager
+
+pytest_plugins = "pytester"
+
+
+@pytest.fixture
+def hookrecorder(request, config, pytester: pytest.Pytester):
+    hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
+    return hookrecorder
+
+
+@pytest.fixture
+def config(pytester: pytest.Pytester):
+    return pytester.parseconfig()
+
+
+@pytest.fixture
+def source(tmp_path: Path) -> Path:
+    source = tmp_path / "source"
+    source.mkdir()
+    return source
+
+
+@pytest.fixture
+def dest(tmp_path: Path) -> Path:
+    dest = tmp_path / "dest"
+    dest.mkdir()
+    return dest
+
+
+@pytest.fixture
+def workercontroller(monkeypatch: pytest.MonkeyPatch):
+    class MockController:
+        def __init__(self, *args):
+            pass
+
+        def setup(self):
+            pass
+
+    monkeypatch.setattr(workermanage, "WorkerController", MockController)
+    return MockController
+
+
+class TestNodeManagerPopen:
+    def test_popen_no_default_chdir(self, config) -> None:
+        gm = NodeManager(config, ["popen"])
+        assert gm.specs[0].chdir is None
+
+    def test_default_chdir(self, config) -> None:
+        specs = ["ssh=noco", "socket=xyz"]
+        for spec in NodeManager(config, specs).specs:
+            assert spec.chdir == "pyexecnetcache"
+        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
+            assert spec.chdir == "abc"
+
+    def test_popen_makegateway_events(
+        self, config, hookrecorder, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)
+        call = hookrecorder.popcall("pytest_xdist_setupnodes")
+        assert len(call.specs) == 2
+
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
+        assert call.gateway.spec == execnet.XSpec("popen")
+        assert call.gateway.id == "gw0"
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
+        assert call.gateway.id == "gw1"
+        assert len(hm.group) == 2
+        hm.teardown_nodes()
+        assert not len(hm.group)
+
+    def test_popens_rsync(
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)
+        assert len(hm.group) == 2
+        for gw in hm.group:
+
+            class pseudoexec:
+                args = []  # type: ignore[var-annotated]
+
+                def __init__(self, *args):
+                    self.args.extend(args)
+
+                def waitclose(self):
+                    pass
+
+            gw.remote_exec = pseudoexec
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert not notifications
+        hm.teardown_nodes()
+        assert not len(hm.group)
+        assert "sys.path.insert" in gw.remote_exec.args[0]
+
+    def test_rsync_popen_with_path(
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
+        hm.setup_nodes(None)
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert len(notifications) == 1
+        assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
+        hm.teardown_nodes()
+        dest = dest.joinpath(source.name)
+        assert dest.joinpath("dir1").exists()
+        assert dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir1", "dir2", "hello").exists()
+
+    def test_rsync_same_popen_twice(
+        self,
+        config,
+        source: Path,
+        dest: Path,
+        hookrecorder,
+        workercontroller,
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
+        hm.roots = []
+        hm.setup_nodes(None)
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        gw = hm.group[0]
+        hm.rsync(gw, source)
+        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
+        assert call.source == source
+        assert len(call.gateways) == 1
+        assert call.gateways[0] in hm.group
+        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
+
+
+class TestHRSync:
+    def test_hrsync_filter(self, source: Path, dest: Path) -> None:
+        source.joinpath("dir").mkdir()
+        source.joinpath("dir", "file.txt").touch()
+        source.joinpath(".svn").mkdir()
+        source.joinpath(".svn", "entries").touch()
+        source.joinpath(".somedotfile").mkdir()
+        source.joinpath(".somedotfile", "moreentries").touch()
+        source.joinpath("somedir").mkdir()
+        source.joinpath("somedir", "editfile~").touch()
+        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
+        files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
+        assert len(files) == 3
+        basenames = [x.basename for x in files]
+        assert "dir" in basenames
+        assert "file.txt" in basenames
+        assert "somedir" in basenames
+
+    def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
+        gw = execnet.makegateway("popen//chdir=%s" % dest)
+        finished = []
+        rsync = HostRSync(source)
+        rsync.add_target_host(gw, finished=lambda: finished.append(1))
+        source.joinpath("hello.py").write_text("world")
+        rsync.send()
+        gw.exit()
+        assert dest.joinpath(source.name, "hello.py").exists()
+        assert len(finished) == 1
+
+
+class TestNodeManager:
+    @pytest.mark.xfail(run=False)
+    def test_rsync_roots_no_roots(
+        self, pytester: pytest.Pytester, source: Path, dest: Path
+    ) -> None:
+        source.joinpath("dir1").mkdir()
+        source.joinpath("dir1", "file1").write_text("hello")
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        # assert nodemanager.config.topdir == source == config.topdir
+        nodemanager.makegateways()  # type: ignore[attr-defined]
+        nodemanager.rsync_roots()  # type: ignore[call-arg]
+        (p,) = nodemanager.gwmanager.multi_exec(  # type: ignore[attr-defined]
+            "import os ; channel.send(os.getcwd())"
+        ).receive_each()
+        p = Path(p)
+        print("remote curdir", p)
+        assert p == dest.joinpath(config.rootpath.name)
+        assert p.joinpath("dir1").check()
+        assert p.joinpath("dir1", "file1").check()
+
+    def test_popen_rsync_subdir(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        dir2 = dir1 / "dir2"
+        dir2.mkdir()
+        dir2.joinpath("hello").touch()
+        for rsyncroot in (dir1, source):
+            shutil.rmtree(str(dest), ignore_errors=True)
+            nodemanager = NodeManager(
+                pytester.parseconfig(
+                    "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
+                )
+            )
+            nodemanager.setup_nodes(None)  # calls .rsync_roots()
+            if rsyncroot == source:
+                dest = dest.joinpath("source")
+            assert dest.joinpath("dir1").exists()
+            assert dest.joinpath("dir1", "dir2").exists()
+            assert dest.joinpath("dir1", "dir2", "hello").exists()
+            nodemanager.teardown_nodes()
+
+    @pytest.mark.parametrize(
+        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+    )
+    def test_rsync_report(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller,
+        capsys: pytest.CaptureFixture[str],
+        flag: str,
+        expects_report: bool,
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
+        if flag:
+            args.append(flag)
+        nodemanager = NodeManager(pytester.parseconfig(*args))
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
+    def test_init_rsync_roots(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir1", "somefile").mkdir()
+        dir2.joinpath("hello").touch()
+        source.joinpath("bogusdir").mkdir()
+        source.joinpath("bogusdir", "file").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs=dir1/dir2
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        assert dest.joinpath("dir2").exists()
+        assert not dest.joinpath("dir1").exists()
+        assert not dest.joinpath("bogus").exists()
+
+    def test_rsyncignore(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir5", "dir6").mkdir(parents=True)
+        source.joinpath("dir5", "dir6", "bogus").touch()
+        source.joinpath("dir5", "file").touch()
+        dir2.joinpath("hello").touch()
+        source.joinpath("foo").mkdir()
+        source.joinpath("foo", "bar").touch()
+        source.joinpath("bar").mkdir()
+        source.joinpath("bar", "foo").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs = dir1 dir5
+                rsyncignore = dir1/dir2 dir5/dir6 foo*
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        config.option.rsyncignore = ["bar"]
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        assert dest.joinpath("dir1").exists()
+        assert not dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir5", "file").exists()
+        assert not dest.joinpath("dir6").exists()
+        assert not dest.joinpath("foo").exists()
+        assert not dest.joinpath("bar").exists()
+
+    def test_optimise_popen(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        specs = ["popen"] * 3
+        source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
+        source.joinpath("a").mkdir()
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, specs)
+        nodemanager.setup_nodes(None)  # calls .rysnc_roots()
+        for gwspec in nodemanager.specs:
+            assert gwspec._samefilesystem()
+            assert not gwspec.chdir
+
+    def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            __init__="",
+            test_x="""
+            def test_one():
+                pass
+        """,
+        )
+        reprec = pytester.inline_run(
+            "-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
+        )
+        (rep,) = reprec.getreports("pytest_runtest_logreport")
+        assert rep.passed

tox.ini

@@ -1,23 +1,38 @@
 [tox]
 envlist=
   linting
-  py{27,34,35,36,37}-pytestlatest
-  py{27,36}-pytest{master,features}
+  py{36,37,38,39,310}-pytestlatest
+  py38-pytestmain
+  py38-psutil
+  py38-setproctitle
 
 [testenv]
-changedir=testing
-passenv = USER USERNAME
+extras = testing
 deps =
   pytestlatest: pytest
-  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
-  pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
-  filelock
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
 commands=
   pytest {posargs}
 
+[testenv:py38-psutil]
+extras =
+  testing
+  psutil
+commands =
+  pytest {posargs:-k psutil}
+
+[testenv:py38-setproctitle]
+extras =
+  testing
+  setproctitle
+deps = pytest
+commands =
+  pytest {posargs}
+
 [testenv:linting]
-skipsdist = True
+skip_install = True
 usedevelop = True
+passenv = PRE_COMMIT_HOME
 deps =
   pre-commit
 commands = pre-commit run --all-files --show-diff-on-failure
@@ -25,17 +40,20 @@ commands = pre-commit run --all-files --show-diff-on-failure
 [testenv:release]
 changedir=
 decription = do a release, required posarg of the version number
-basepython = python3.6
+basepython = python3.7
 skipsdist = True
 usedevelop = True
 passenv = *
 deps =
-    towncrier
+  towncrier
 commands =
-    towncrier --version {posargs} --yes
+  towncrier --version {posargs} --yes
 
 [pytest]
-addopts = -ra
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
+testpaths = testing
 
 [flake8]
 max-line-length = 120

xdist/__init__.py

@@ -1,3 +0,0 @@
-from xdist._version import version as __version__
-
-__all__ = ["__version__"]

xdist/newhooks.py

@@ -1,52 +0,0 @@
-"""
-xdist hooks.
-
-Additionally, pytest-xdist will also decorate a few other hooks
-with the worker instance that executed the hook originally:
-
-``pytest_runtest_logreport``: ``rep`` parameter has a ``node`` attribute.
-
-You can use this hooks just as you would use normal pytest hooks, but some care
-must be taken in plugins in case ``xdist`` is not installed. Please see:
-
-    http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
-"""
-import pytest
-
-
-def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
-
-
-def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
-
-
-def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
-
-
-def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
-
-
-def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
-
-
-def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
-
-
-def pytest_testnodedown(node, error):
-    """ Test Node is down. """
-
-
-def pytest_xdist_node_collection_finished(node, ids):
-    """called by the master node when a node finishes collecting.
-    """
-
-
-@pytest.mark.firstresult
-def pytest_xdist_make_scheduler(config, log):
-    """ return a node scheduler implementation """