Release 3.2.1

Fix hang caused by `steal` command with empty test queue

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,15 @@
+Thanks for submitting a PR, your contribution is really appreciated!
+
+Here's a quick checklist that should be present in PRs:
+
+- [ ] Make sure to include reasonable tests for your change if necessary
+
+- [ ] We use [towncrier](https://pypi.python.org/pypi/towncrier) for changelog management, so please add a *news* file into the `changelog` folder following these guidelines:
+  * Name it `$issue_id.$type` for example `588.bugfix`;
+  * If you don't have an issue_id change it to the PR id after creating it
+  * Ensure type is one of `removal`, `feature`, `bugfix`, `vendor`, `doc` or `trivial`
+  * Make sure to use full sentences with correct case and punctuation, for example:
+
+    ```
+    Fix issue with non-ascii contents in doctest text files.
+    ```

.github/workflows/deploy.yml

@@ -0,0 +1,33 @@
+name: deploy
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+    - name: Install build
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: |
+        python -m build
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.github/workflows/test.yml

@@ -0,0 +1,67 @@
+name: test
+
+on:
+  push:
+    branches:
+      - "*"
+
+  pull_request:
+    branches:
+      - "*"
+
+jobs:
+  test:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py310-pytestmain"
+          - "py311-pytestlatest"
+          - "py311-pytestmain"
+          - "py310-psutil"
+          - "py310-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10"
+          - tox_env: "py310-pytestmain"
+            python: "3.10"
+          - tox_env: "py311-pytestlatest"
+            python: "3.11"
+          - tox_env: "py311-pytestmain"
+            python: "3.11"
+          - tox_env: "py310-psutil"
+            python: "3.10"
+          - tox_env: "py310-setproctitle"
+            python: "3.10"
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}

.gitignore

@@ -22,7 +22,14 @@ dist/
 include/
 lib/
 bin/
+env/
+src/xdist/_version.py*
 pytest_xdist.egg-info
 issue/
 3rdparty/
-.tox
+pytestdebug.log
+.tox/
+.cache/
+.pytest_cache/
+.eggs/
+.idea/

.hgtags

@@ -1,16 +0,0 @@
-42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0
-eca7ce17eabf296983c36812c8b8be901e7055a3 1.1
-56d8e5280be224a0ad3220a9deed55334710bd23 1.2
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-eaf8b1cb7c312883598677231be5bbeea3b5c127 1.3
-a423748bf17ee778a37853225210257699cad9c1 1.4
-cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
-4815040bdad8f182a5487f57a9da385483836e75 1.6
-20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
-20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
-29c38e195526f5f0fdd651fb51f59d6efaaafbb0 1.7
-0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
-0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
-1d27987c267577899350a25ba5828d55d87083ad 1.8
-5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9

.pre-commit-config.yaml

@@ -0,0 +1,49 @@
+repos:
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v2.0.1
+    hooks:
+    -   id: autoflake
+        args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"]
+-   repo: https://github.com/psf/black
+    rev: 23.1.0
+    hooks:
+    -   id: black
+        args: [--safe, --quiet, --target-version, py35]
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.13.0
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black==23.1.0]
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: debug-statements
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+    -   id: flake8
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.3.1
+    hooks:
+    -   id: pyupgrade
+        args: [--py3-plus]
+-   repo: local
+    hooks:
+    -   id: rst
+        name: rst
+        entry: rst-lint --encoding utf-8
+        files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
+        language: python
+        additional_dependencies: [pygments, restructuredtext_lint]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.0.1
+    hooks:
+    -   id: mypy
+        files: ^(src/|testing/)
+        args: []
+        additional_dependencies:
+          - pytest>=6.2.0
+          - py>=1.10.0

CHANGELOG

@@ -1,123 +0,0 @@
-1.10
--------------------------
-
-- add glob support for rsyncignores, add command line option to pass
-  additional rsyncignores. Thanks Anatoly Bubenkov.
-
-- fix pytest issue382 - produce "pytest_runtest_logstart" event again
-  in master. Thanks Aron Curzon.
-
-- fix pytest issue419 by sending/receiving indices into the test 
-  collection instead of node ids (which are not neccessarily unique
-  for functions parametrized with duplicate values)
-
-- send multiple "to test" indices in one network message to a slave
-  and improve heuristics for sending chunks where the chunksize
-  depends on the number of remaining tests rather than fixed numbers.
-  This reduces the number of master -> node messages (but not the
-  reverse direction)
-
-
-1.9
--------------------------
-
-- changed LICENSE to MIT
-
-- fix duplicate reported test ids with --looponfailing
-  (thanks Jeremy Thurgood)
-
-- fix pytest issue41: re-run tests on all file changes, not just
-  randomly select ones like .py/.c.
-
-- fix pytest issue347: slaves running on top of Python3.2
-  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
-  bugs.
-
-1.8
--------------------------
-
-- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
-  interface to perform eager teardowns for test items.
-
-1.7
--------------------------
-
-- fix incompatibilities with pytest-2.2.0 (allow multiple
-  pytest_runtest_logreport reports for a test item)
-
-1.6
--------------------------
-
-- terser collection reporting
-
-- fix issue34 - distributed testing with -p plugin now works correctly
-
-- fix race condition in looponfail mode where a concurrent file removal
-  could cause a crash
-
-1.5
--------------------------
-
-- adapt to and require pytest-2.0 changes, rsyncdirs and rsyncignore can now
-  only be specified in [pytest] sections of ini files, see "py.test -h"
-  for details.
-- major internal refactoring to match the pytest-2.0 event refactoring
-  - perform test collection always at slave side instead of at the master
-  - make python2/python3 bridging work, remove usage of pickling
-- improve initial reporting by using line-rewriting
-- remove all trailing whitespace from source
-
-1.4
--------------------------
-
-- perform distributed testing related reporting in the plugin
-  rather than having dist-related code in the generic py.test
-  distribution
-
-- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to
-  gateway specification strings.
-
-- show detailed gateway setup and platform information only when
-  "-v" or "--verbose" is specified.
-
-1.3
--------------------------
-
-- fix --looponfailing - it would not actually run against the fully changed
-  source tree when initial conftest files load application state.
-
-- adapt for py-1.3.1's new --maxfailure option
-
-1.2
--------------------------
-
-- fix issue79: sessionfinish/teardown hooks are now called systematically
-  on the slave side
-- introduce a new data input/output mechanism to allow the master side
-  to send and receive data from a slave.
-- fix race condition in underlying pickling/unpickling handling
-- use and require new register hooks facility of py.test>=1.3.0
-- require improved execnet>=1.0.6 because of various race conditions
-  that can arise in xdist testing modes.
-- fix some python3 related pickling related race conditions
-- fix PyPI description
-
-1.1
--------------------------
-
-- fix an indefinite hang which would wait for events although no events
-  are pending - this happened if items arrive very quickly while
-  the "reschedule-event" tried unconditionally avoiding a busy-loop
-  and not schedule new work.
-
-1.0
--------------------------
-
-- moved code out of py-1.1.1 into its own plugin
-- use a new, faster and more sensible model to do load-balancing
-  of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing
-  works effectively even with very few tests.
-- cleaned up termination handling
-- make -x cause hard killing of test nodes to decrease wait time
-  until the traceback shows up on first failure
-

CHANGELOG.rst

@@ -0,0 +1,994 @@
+pytest-xdist 3.2.1 (2023-03-12)
+===============================
+
+Bug Fixes
+---------
+
+- `#884 <https://github.com/pytest-dev/pytest-xdist/issues/884>`_: Fixed hang in ``worksteal`` scheduler.
+
+
+pytest-xdist 3.2.0 (2023-02-07)
+===============================
+
+Improved Documentation
+----------------------
+
+- `#863 <https://github.com/pytest-dev/pytest-xdist/issues/863>`_: Document limitations for debugging due to standard I/O of workers not being forwarded. Also, mention remote debugging as a possible workaround.
+
+
+Features
+--------
+
+- `#855 <https://github.com/pytest-dev/pytest-xdist/issues/855>`_: Users can now configure ``load`` scheduling precision using ``--maxschedchunk`` command
+  line option.
+
+- `#858 <https://github.com/pytest-dev/pytest-xdist/issues/858>`_: New ``worksteal`` scheduler, based on the idea of `work stealing <https://en.wikipedia.org/wiki/Work_stealing>`_. It's similar to ``load`` scheduler, but it should handle tests with significantly differing duration better, and, at the same time, it should provide similar or better reuse of fixtures.
+
+
+Trivial Changes
+---------------
+
+- `#870 <https://github.com/pytest-dev/pytest-xdist/issues/870>`_: Make the tests pass even when ``$PYTEST_XDIST_AUTO_NUM_WORKERS`` is set.
+
+
+pytest-xdist 3.1.0 (2022-12-01)
+===============================
+
+Features
+--------
+
+- `#789 <https://github.com/pytest-dev/pytest-xdist/issues/789>`_: Users can now set a default distribution mode in their configuration file:
+
+  .. code-block:: ini
+
+      [pytest]
+      addopts = --dist loadscope
+
+- `#842 <https://github.com/pytest-dev/pytest-xdist/issues/842>`_: Python 3.11 is now officially supported.
+
+
+Removals
+--------
+
+- `#842 <https://github.com/pytest-dev/pytest-xdist/issues/842>`_: Python 3.6 is no longer supported.
+
+
+pytest-xdist 3.0.2 (2022-10-25)
+===============================
+
+Bug Fixes
+---------
+
+- `#813 <https://github.com/pytest-dev/pytest-xdist/issues/813>`_: Cancel shutdown when a crashed worker is restarted.
+
+
+Deprecations
+------------
+
+- `#825 <https://github.com/pytest-dev/pytest-xdist/issues/825>`_: The ``--rsyncdir`` command line argument and ``rsyncdirs`` config variable are deprecated.
+
+  The rsync feature will be removed in pytest-xdist 4.0.
+
+- `#826 <https://github.com/pytest-dev/pytest-xdist/issues/826>`_: The ``--looponfail`` command line argument and ``looponfailroots`` config variable are deprecated.
+
+  The loop-on-fail feature will be removed in pytest-xdist 4.0.
+
+
+Improved Documentation
+----------------------
+
+- `#791 <https://github.com/pytest-dev/pytest-xdist/issues/791>`_: Document the ``pytest_xdist_auto_num_workers`` hook.
+
+- `#796 <https://github.com/pytest-dev/pytest-xdist/issues/796>`_: Added known limitations section to documentation.
+
+- `#829 <https://github.com/pytest-dev/pytest-xdist/issues/829>`_: Document the ``-n logical`` option.
+
+
+Features
+--------
+
+- `#792 <https://github.com/pytest-dev/pytest-xdist/issues/792>`_: The environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` can now be used to
+  specify the default for ``-n auto`` and ``-n logical``.
+
+- `#812 <https://github.com/pytest-dev/pytest-xdist/issues/812>`_: Partially restore old initial batch distribution algorithm in ``LoadScheduling``.
+
+  pytest orders tests for optimal sequential execution - i. e. avoiding
+  unnecessary setup and teardown of fixtures. So executing tests in consecutive
+  chunks is important for optimal performance.
+
+  In v1.14, initial test distribution in ``LoadScheduling`` was changed to
+  round-robin, optimized for the corner case, when the number of tests is less
+  than ``2 * number of nodes``. At the same time, it became worse for all other
+  cases.
+
+  For example: if some tests use some "heavy" fixture, and these tests fit into
+  the initial batch, with round-robin distribution the fixture will be created
+  ``min(n_tests, n_workers)`` times, no matter how many other tests there are.
+
+  With the old algorithm (before v1.14), if there are enough tests not using
+  the fixture, the fixture was created only once.
+
+  So restore the old behavior for typical cases where the number of tests is
+  much greater than the number of workers (or, strictly speaking, when there
+  are at least 2 tests for every node).
+
+
+Removals
+--------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command-line option has been removed. If you still need this functionality, install `pytest-forked <https://pypi.org/project/pytest-forked>`__ separately.
+
+
+Trivial Changes
+---------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``py`` dependency has been dropped.
+
+- `#822 <https://github.com/pytest-dev/pytest-xdist/issues/822>`_: Replace internal usage of ``py.log`` with a custom solution (but with the same interface).
+
+- `#823 <https://github.com/pytest-dev/pytest-xdist/issues/823>`_: Remove usage of ``py._pydir`` as an rsync candidate.
+
+- `#824 <https://github.com/pytest-dev/pytest-xdist/issues/824>`_: Replace internal usages of ``py.path.local`` by ``pathlib.Path``.
+
+
+pytest-xdist 2.5.0 (2021-12-10)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command line argument is deprecated. Install `pytest-forked <https://pypi.org/project/pytest-forked>`__ and use ``--forked`` instead. pytest-xdist 3.0.0 will remove the ``--boxed`` argument and ``pytest-forked`` dependency.
+
+
+Features
+--------
+
+- `#722 <https://github.com/pytest-dev/pytest-xdist/issues/722>`_: Full compatibility with pytest 7 - no deprecation warnings or use of legacy features.
+
+- `#733 <https://github.com/pytest-dev/pytest-xdist/issues/733>`_: New ``--dist=loadgroup`` option, which ensures all tests marked with ``@pytest.mark.xdist_group`` run in the same session/worker. Other tests run distributed as in ``--dist=load``.
+
+
+Trivial Changes
+---------------
+
+- `#708 <https://github.com/pytest-dev/pytest-xdist/issues/708>`_: Use ``@pytest.hookspec`` decorator to declare hook options in ``newhooks.py`` to avoid warnings in ``pytest 7.0``.
+
+- `#719 <https://github.com/pytest-dev/pytest-xdist/issues/719>`_: Use up-to-date ``setup.cfg``/``pyproject.toml`` packaging setup.
+
+- `#720 <https://github.com/pytest-dev/pytest-xdist/issues/720>`_: Require pytest>=6.2.0.
+
+- `#721 <https://github.com/pytest-dev/pytest-xdist/issues/721>`_: Started using type annotations and mypy checking internally. The types are incomplete and not published.
+
+
+pytest-xdist 2.4.0 (2021-09-20)
+===============================
+
+Features
+--------
+
+- `#696 <https://github.com/pytest-dev/pytest-xdist/issues/696>`_: On Linux, the process title now changes to indicate the current worker state (running/idle).
+
+  Depends on the `setproctitle <https://pypi.org/project/setproctitle/>`__ package, which can be installed with ``pip install pytest-xdist[setproctitle]``.
+
+- `#704 <https://github.com/pytest-dev/pytest-xdist/issues/704>`_: Add support for Python 3.10.
+
+
+pytest-xdist 2.3.0 (2021-06-16)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#654 <https://github.com/pytest-dev/pytest-xdist/issues/654>`_: Python 3.5 is no longer supported.
+
+
+Features
+--------
+
+- `#646 <https://github.com/pytest-dev/pytest-xdist/issues/646>`_: Add ``--numprocesses=logical`` flag, which automatically uses the number of logical CPUs available, instead of physical CPUs with ``auto``.
+
+  This is very useful for test suites which are not CPU-bound.
+
+- `#650 <https://github.com/pytest-dev/pytest-xdist/issues/650>`_: Added new ``pytest_handlecrashitem`` hook to allow handling and rescheduling crashed items.
+
+
+Bug Fixes
+---------
+
+- `#421 <https://github.com/pytest-dev/pytest-xdist/issues/421>`_: Copy the parent process sys.path into local workers, to work around execnet's python -c adding the current directory to sys.path.
+
+- `#638 <https://github.com/pytest-dev/pytest-xdist/issues/638>`_: Fix issue caused by changing the branch name of the pytest repository.
+
+
+Trivial Changes
+---------------
+
+- `#592 <https://github.com/pytest-dev/pytest-xdist/issues/592>`_: Replace master with controller where ever possible.
+
+- `#643 <https://github.com/pytest-dev/pytest-xdist/issues/643>`_: Use 'main' to refer to pytest default branch in tox env names.
+
+
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
+pytest-xdist 2.0.0 (2020-08-12)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: Drop backward-compatibility "slave" aliases related to worker nodes.  We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.
+
+- `#569 <https://github.com/pytest-dev/pytest-xdist/issues/569>`_: ``pytest-xdist`` no longer supports Python 2.7.
+
+
+Features
+--------
+
+- `#504 <https://github.com/pytest-dev/pytest-xdist/issues/504>`_: New functions ``xdist.is_xdist_worker``, ``xdist.is_xdist_master``, ``xdist.get_xdist_worker_id``, to easily identify the current node.
+
+
+Bug Fixes
+---------
+
+- `#471 <https://github.com/pytest-dev/pytest-xdist/issues/471>`_: Fix issue with Rsync reporting in quiet mode.
+
+- `#553 <https://github.com/pytest-dev/pytest-xdist/issues/553>`_: When using ``-n auto``, count the number of physical CPU cores instead of logical ones.
+
+
+Trivial Changes
+---------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: ``pytest-xdist`` now requires ``pytest>=6.0``.
+
+
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
+pytest-xdist 1.32.0 (2020-05-03)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- `#475 <https://github.com/pytest-dev/pytest-xdist/issues/475>`_: Drop support for EOL Python 3.4.
+
+
+Features
+--------
+
+- `#524 <https://github.com/pytest-dev/pytest-xdist/issues/524>`_: Add `testrun_uid` fixture. This is a shared value that uniquely identifies a test run among all workers.
+  This also adds a `PYTEST_XDIST_TESTRUNUID` environment variable that is accessible within a test as well as a command line option `--testrunuid` to manually set the value from outside.
+
+
+pytest-xdist 1.31.0 (2019-12-19)
+================================
+
+Features
+--------
+
+- `#486 <https://github.com/pytest-dev/pytest-xdist/issues/486>`_: Add support for Python 3.8.
+
+
+Bug Fixes
+---------
+
+- `#491 <https://github.com/pytest-dev/pytest-xdist/issues/491>`_: Fix regression that caused custom plugin command-line arguments to be discarded when using ``--tx`` mode.
+
+
+
+pytest-xdist 1.30.0 (2019-10-01)
+================================
+
+Features
+--------
+
+- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
+  long-standing issues related to startup with the ``-c`` option.
+
+  Issues:
+
+  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
+  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
+
+  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
+
+
+Bug Fixes
+---------
+
+- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
+
+
+pytest-xdist 1.29.0 (2019-06-14)
+================================
+
+Features
+--------
+
+- `#226 <https://github.com/pytest-dev/pytest-xdist/issues/226>`_: ``--max-worker-restart`` now assumes a more reasonable value (4 times the number of
+  nodes) when not given explicitly. This prevents test suites from running forever when the suite crashes during collection.
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: When the test session is interrupted due to running out of workers, the reason is shown in the test summary
+  for easier viewing.
+
+- `#442 <https://github.com/pytest-dev/pytest-xdist/issues/442>`_: Compatibility fix for upcoming pytest 5.0: ``session.exitstatus`` is now an ``IntEnum`` object.
+
+
+Bug Fixes
+---------
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: No longer show an internal error when we run out of workers due to crashes.
+
+
+pytest-xdist 1.28.0 (2019-04-02)
+================================
+
+Features
+--------
+
+- `#426 <https://github.com/pytest-dev/pytest-xdist/issues/426>`_: ``pytest-xdist`` now uses the new ``pytest_report_to_serializable`` and ``pytest_report_from_serializable``
+  hooks from ``pytest 4.4`` (still experimental). This will make report serialization more reliable and
+  extensible.
+
+  This also means that ``pytest-xdist`` now requires ``pytest>=4.4``.
+
+
+pytest-xdist 1.27.0 (2019-02-15)
+================================
+
+Features
+--------
+
+- `#374 <https://github.com/pytest-dev/pytest-xdist/issues/374>`_: The new ``pytest_xdist_getremotemodule`` hook allows overriding the module run on remote nodes.
+
+- `#415 <https://github.com/pytest-dev/pytest-xdist/issues/415>`_: Improve behavior of ``--numprocesses=auto`` to work well with ``--pdb`` option.
+
+
+pytest-xdist 1.26.1 (2019-01-28)
+================================
+
+Bug Fixes
+---------
+
+- `#406 <https://github.com/pytest-dev/pytest-xdist/issues/406>`_: Do not implement deprecated ``pytest_logwarning`` hook in pytest versions where it is deprecated.
+
+
+pytest-xdist 1.26.0 (2019-01-11)
+================================
+
+Features
+--------
+
+- `#376 <https://github.com/pytest-dev/pytest-xdist/issues/376>`_: The current directory is no longer added ``sys.path`` for local workers, only for remote connections.
+
+  This behavior is surprising because it makes xdist runs and non-xdist runs to potentially behave differently.
+
+
+Bug Fixes
+---------
+
+- `#379 <https://github.com/pytest-dev/pytest-xdist/issues/379>`_: Warning attributes are checked to make sure they can be dumped prior to
+  serializing the warning for submission to the master node.
+
+
+pytest-xdist 1.25.0 (2018-12-12)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- `#372 <https://github.com/pytest-dev/pytest-xdist/issues/372>`_: Pytest versions older than 3.6 are no longer supported.
+
+
+Features
+--------
+
+- `#373 <https://github.com/pytest-dev/pytest-xdist/issues/373>`_: Node setup information is hidden when pytest is run in quiet mode to reduce noise on many-core machines.
+
+- `#388 <https://github.com/pytest-dev/pytest-xdist/issues/388>`_: ``mainargv`` is made available in ``workerinput`` from the host's ``sys.argv``.
+
+  This can be used via ``request.config.workerinput["mainargv"]``.
+
+
+Bug Fixes
+---------
+
+- `#332 <https://github.com/pytest-dev/pytest-xdist/issues/332>`_: Fix report of module-level skips (``pytest.skip(reason, allow_module_level=True)``).
+
+- `#378 <https://github.com/pytest-dev/pytest-xdist/issues/378>`_: Fix support for gevent monkeypatching
+
+- `#384 <https://github.com/pytest-dev/pytest-xdist/issues/384>`_: pytest 4.1 support: ``ExceptionInfo`` API changes.
+
+- `#390 <https://github.com/pytest-dev/pytest-xdist/issues/390>`_: pytest 4.1 support: ``pytest_logwarning`` hook removed.
+
+
+pytest-xdist 1.24.1 (2018-11-09)
+================================
+
+Bug Fixes
+---------
+
+- `#349 <https://github.com/pytest-dev/pytest-xdist/issues/349>`_: Correctly handle warnings created with arguments that can't be serialized during the transfer from workers to master node.
+
+
+pytest-xdist 1.24.0 (2018-10-18)
+================================
+
+Features
+--------
+
+- `#337 <https://github.com/pytest-dev/pytest-xdist/issues/337>`_: New ``--maxprocesses`` command-line option that limits the maximum number of workers when using ``--numprocesses=auto``.
+
+
+Bug Fixes
+---------
+
+- `#351 <https://github.com/pytest-dev/pytest-xdist/issues/351>`_: Fix scheduling deadlock in case of inter-test locking.
+
+
+pytest-xdist 1.23.2 (2018-09-28)
+================================
+
+Bug Fixes
+---------
+
+- `#344 <https://github.com/pytest-dev/pytest-xdist/issues/344>`_: Fix issue where Warnings could cause pytest to fail if they do not set the args attribute correctly.
+
+
+pytest-xdist 1.23.1 (2018-09-25)
+================================
+
+Bug Fixes
+---------
+
+- `#341 <https://github.com/pytest-dev/pytest-xdist/issues/341>`_: Fix warnings transfer between workers and master node with pytest >= 3.8.
+
+
+pytest-xdist 1.23.0 (2018-08-23)
+================================
+
+Features
+--------
+
+- `#330 <https://github.com/pytest-dev/pytest-xdist/issues/330>`_: Improve collection performance by reducing the number of events sent to ``master`` node.
+
+
+pytest-xdist 1.22.5 (2018-07-27)
+================================
+
+Bug Fixes
+---------
+
+- `#321 <https://github.com/pytest-dev/pytest-xdist/issues/321>`_: Revert change that dropped support for ``pytest<3.4`` and require ``six``.
+
+  This change caused problems in some installations, and was a mistaken
+  in the first place as we should not change version requirements
+  in bug-fix releases unless they fix an actual bug.
+
+
+pytest-xdist 1.22.4 (2018-07-27)
+================================
+
+Bug Fixes
+---------
+
+- `#305 <https://github.com/pytest-dev/pytest-xdist/issues/305>`_: Remove last references to obsolete ``py.code``.
+
+  Remove some unnecessary references to ``py.builtin``.
+
+- `#316 <https://github.com/pytest-dev/pytest-xdist/issues/316>`_: Workaround cpu detection on Travis CI.
+
+
+pytest-xdist 1.22.3 (2018-07-23)
+================================
+
+Bug Fixes
+---------
+
+- Fix issue of virtualized or containerized environments not reporting the number of CPUs correctly. (`#9 <https://github.com/pytest-dev/pytest-xdist/issues/9>`_)
+
+
+Trivial Changes
+---------------
+
+- Make all classes subclass from ``object`` and fix ``super()`` call in ``LoadFileScheduling``; (`#297 <https://github.com/pytest-dev/pytest-xdist/issues/297>`_)
+
+
+pytest-xdist 1.22.2 (2018-02-26)
+================================
+
+Bug Fixes
+---------
+
+- Add backward compatibility for ``slaveoutput`` attribute to
+  ``WorkerController`` instances. (`#285
+  <https://github.com/pytest-dev/pytest-xdist/issues/285>`_)
+
+
+pytest-xdist 1.22.1 (2018-02-19)
+================================
+
+Bug Fixes
+---------
+
+- Fix issue when using ``loadscope`` or ``loadfile`` where tests would fail to
+  start if the first scope had only one test. (`#257
+  <https://github.com/pytest-dev/pytest-xdist/issues/257>`_)
+
+
+Trivial Changes
+---------------
+
+- Change terminology used by ``pytest-xdist`` to *master* and *worker* in
+  arguments and messages (for example ``--max-worker-reset``). (`#234
+  <https://github.com/pytest-dev/pytest-xdist/issues/234>`_)
+
+
+pytest-xdist 1.22.0 (2018-01-11)
+================================
+
+Features
+--------
+
+- Add support for the ``pytest_runtest_logfinish`` hook which will be released
+  in pytest 3.4. (`#266
+  <https://github.com/pytest-dev/pytest-xdist/issues/266>`_)
+
+
+pytest-xdist 1.21.0 (2017-12-22)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- Drop support for EOL Python 2.6. (`#259
+  <https://github.com/pytest-dev/pytest-xdist/issues/259>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadfile`` option which load-distributes test to workers grouped
+  by the file the tests live in. (`#242
+  <https://github.com/pytest-dev/pytest-xdist/issues/242>`_)
+
+
+Bug Fixes
+---------
+
+- Fix accidental mutation of test report during serialization causing longrepr
+  string-ification to break. (`#241
+  <https://github.com/pytest-dev/pytest-xdist/issues/241>`_)
+
+
+pytest-xdist 1.20.1 (2017-10-05)
+================================
+
+Bug Fixes
+---------
+
+- Fix hang when all worker nodes crash and restart limit is reached (`#45
+  <https://github.com/pytest-dev/pytest-xdist/issues/45>`_)
+
+- Fix issue where the -n option would still run distributed tests when pytest
+  was run with the --collect-only option (`#5
+  <https://github.com/pytest-dev/pytest-xdist/issues/5>`_)
+
+
+pytest-xdist 1.20.0 (2017-08-17)
+================================
+
+Features
+--------
+
+- ``xdist`` now supports tests to log results multiple times, improving
+  integration with plugins which require it like `pytest-rerunfailures
+  <https://github.com/gocept/pytest-rerunfailures>`_ and `flaky
+  <https://pypi.python.org/pypi/flaky>`_. (`#206 <https://github.com/pytest-
+  dev/pytest-xdist/issues/206>`_)
+
+
+Bug Fixes
+---------
+
+- Fix issue where tests were being incorrectly identified if a worker crashed
+  during the ``teardown`` stage of the test. (`#124 <https://github.com/pytest-
+  dev/pytest-xdist/issues/124>`_)
+
+
+pytest-xdist 1.19.1 (2017-08-10)
+================================
+
+Bug Fixes
+---------
+
+- Fix crash when transferring internal pytest warnings from workers to the
+  master node. (`#214 <https://github.com/pytest-dev/pytest-
+  xdist/issues/214>`_)
+
+
+pytest-xdist 1.19.0 (2017-08-09)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- ``--boxed`` functionality has been moved to a separate plugin, `pytest-forked
+  <https://github.com/pytest-dev/pytest-forked>`_. This release now depends on
+  `` pytest-forked`` and provides ``--boxed`` as a backward compatibility
+  option. (`#1 <https://github.com/pytest-dev/pytest-xdist/issues/1>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadscope`` option: sends group of related tests to the same
+  worker. Tests are grouped by module for test functions and by class for test
+  methods. See ``README.rst`` for more information. (`#191 <https://github.com
+  /pytest-dev/pytest-xdist/issues/191>`_)
+
+- Warnings are now properly transferred from workers to the master node. (`#92
+  <https://github.com/pytest-dev/pytest-xdist/issues/92>`_)
+
+
+Bug Fixes
+---------
+
+- Fix serialization of native tracebacks (``--tb=native``). (`#196
+  <https://github.com/pytest-dev/pytest-xdist/issues/196>`_)
+
+
+pytest-xdist 1.18.2 (2017-07-28)
+================================
+
+Bug Fixes
+---------
+
+- Removal of unnecessary dependency on incorrect version of py. (`#105
+  <https://github.com/pytest-dev/pytest-xdist/issues/105>`_)
+
+- Fix bug in internal event-loop error handler in the master node. This bug
+  would shadow the original errors making extremely hard/impossible for users
+  to diagnose the problem properly. (`#175 <https://github.com/pytest-
+  dev/pytest-xdist/issues/175>`_)
+
+
+pytest-xdist 1.18.1 (2017-07-05)
+================================
+
+Bug Fixes
+---------
+
+- Fixed serialization of ``longrepr.sections`` during error reporting from
+  workers. (`#171 <https://github.com/pytest-dev/pytest-xdist/issues/171>`_)
+
+- Fix ``ReprLocal`` not being unserialized breaking --showlocals usages. (`#176
+  <https://github.com/pytest-dev/pytest-xdist/issues/176>`_)
+
+
+pytest-xdist 1.18.0 (2017-06-26)
+================================
+
+- ``pytest-xdist`` now requires ``pytest>=3.0.0``.
+
+Features
+--------
+
+- Add long option `--numprocesses` as alternative for `-n`. (#168)
+
+
+Bug Fixes
+---------
+
+- Fix serialization and deserialization dropping longrepr details. (#133)
+
+
+pytest-xdist 1.17.1 (2017-06-10)
+================================
+
+Bug Fixes
+---------
+
+- Hot fix release reverting the change introduced by #124, unfortunately it
+  broke a number of test suites so we are reversing this change while we
+  investigate the problem. (#157)
+
+
+Improved Documentation
+----------------------
+
+- Introduced ``towncrier`` for ``CHANGELOG`` management. (#154)
+
+- Added ``HOWTORELEASE`` documentation. (#155)
+
+
+1.17.0
+------
+
+- fix #124: xdist would mark test as complete after 'call' step. As a result,
+  xdist could identify the wrong test as failing when test crashes at teardown.
+  To address this issue, xdist now marks test as complete at teardown.
+
+1.16.0
+------
+
+- ``pytest-xdist`` now requires pytest 2.7 or later.
+
+- Add ``worker_id`` attribute in the TestReport
+
+- new hook: ``pytest_xdist_make_scheduler(config, log)``, can return custom tests items
+  distribution logic implementation. You can take a look at built-in ``LoadScheduling``
+  and ``EachScheduling`` implementations. Note that required scheduler class public
+  API may change in next ``pytest-xdist`` versions.
+
+1.15.0
+------
+
+- new ``worker_id`` fixture, returns the id of the worker in a test or fixture.
+  Thanks Jared Hellman for the PR.
+
+- display progress during collection only when in a terminal, similar to pytest #1397 issue.
+  Thanks Bruno Oliveira for the PR.
+
+- fix internal error message when ``--maxfail`` is used (#62, #65).
+  Thanks Collin RM Stocks and Bryan A. Jones for reports and Bruno Oliveira for the PR.
+
+
+1.14
+----
+
+- new hook: ``pytest_xdist_node_collection_finished(node, ids)``, called when
+  a worker has finished collection. Thanks Omer Katz for the request and
+  Bruno Oliveira for the PR.
+
+- fix README display on pypi
+
+- fix #22: xdist now works if the internal tmpdir plugin is disabled.
+  Thanks Bruno Oliveira for the PR.
+
+- fix #32: xdist now works if looponfail or boxed are disabled.
+  Thanks Bruno Oliveira for the PR.
+
+
+1.13.1
+-------
+
+- fix a regression -n 0 now disables xdist again
+
+
+1.13
+-------------------------
+
+- extended the tox matrix with the supported py.test versions
+
+- split up the plugin into 3 plugin's
+  to prepare the departure of boxed and looponfail.
+
+  looponfail will be a part of core
+  and forked boxed will be replaced
+  with a more reliable primitive based on xdist
+
+- conforming with new pytest-2.8 behavior of returning non-zero when all
+  tests were skipped or deselected.
+
+- new "--max-slave-restart" option that can be used to control maximum
+  number of times pytest-xdist can restart slaves due to crashes. Thanks to
+  Anatoly Bubenkov for the report and Bruno Oliveira for the PR.
+
+- release as wheel
+
+- "-n" option now can be set to "auto" for automatic detection of number
+  of cpus in the host system. Thanks Suloev Dmitry for the PR.
+
+1.12
+-------------------------
+
+- fix issue594: properly report errors when the test collection
+  is random.  Thanks Bruno Oliveira.
+
+- some internal test suite adaptation (to become forward
+  compatible with the upcoming pytest-2.8)
+
+
+1.11
+-------------------------
+
+- fix pytest/xdist issue485 (also depends on py-1.4.22):
+  attach stdout/stderr on --boxed processes that die.
+
+- fix pytest/xdist issue503: make sure that a node has usually
+  two items to execute to avoid scoped fixtures to be torn down
+  pre-maturely (fixture teardown/setup is "nextitem" sensitive).
+  Thanks to Andreas Pelme for bug analysis and failing test.
+
+- restart crashed nodes by internally refactoring setup handling
+  of nodes.  Also includes better code documentation.
+  Many thanks to Floris Bruynooghe for the complete PR.
+
+
+1.10
+-------------------------
+
+- add glob support for rsyncignores, add command line option to pass
+  additional rsyncignores. Thanks Anatoly Bubenkov.
+
+- fix pytest issue382 - produce "pytest_runtest_logstart" event again
+  in master. Thanks Aron Curzon.
+
+- fix pytest issue419 by sending/receiving indices into the test
+  collection instead of node ids (which are not necessarily unique
+  for functions parametrized with duplicate values)
+
+- send multiple "to test" indices in one network message to a slave
+  and improve heuristics for sending chunks where the chunksize
+  depends on the number of remaining tests rather than fixed numbers.
+  This reduces the number of master -> node messages (but not the
+  reverse direction)
+
+
+1.9
+-------------------------
+
+- changed LICENSE to MIT
+
+- fix duplicate reported test ids with --looponfailing
+  (thanks Jeremy Thurgood)
+
+- fix pytest issue41: re-run tests on all file changes, not just
+  randomly select ones like .py/.c.
+
+- fix pytest issue347: slaves running on top of Python3.2
+  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
+  bugs.
+
+1.8
+-------------------------
+
+- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
+  interface to perform eager teardowns for test items.
+
+1.7
+-------------------------
+
+- fix incompatibilities with pytest-2.2.0 (allow multiple
+  pytest_runtest_logreport reports for a test item)
+
+1.6
+-------------------------
+
+- terser collection reporting
+
+- fix issue34 - distributed testing with -p plugin now works correctly
+
+- fix race condition in looponfail mode where a concurrent file removal
+  could cause a crash
+
+1.5
+-------------------------
+
+- adapt to and require pytest-2.0 changes, rsyncdirs and rsyncignore can now
+  only be specified in [pytest] sections of ini files, see "py.test -h"
+  for details.
+- major internal refactoring to match the pytest-2.0 event refactoring
+  - perform test collection always at slave side instead of at the master
+  - make python2/python3 bridging work, remove usage of pickling
+- improve initial reporting by using line-rewriting
+- remove all trailing whitespace from source
+
+1.4
+-------------------------
+
+- perform distributed testing related reporting in the plugin
+  rather than having dist-related code in the generic py.test
+  distribution
+
+- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to
+  gateway specification strings.
+
+- show detailed gateway setup and platform information only when
+  "-v" or "--verbose" is specified.
+
+1.3
+-------------------------
+
+- fix --looponfailing - it would not actually run against the fully changed
+  source tree when initial conftest files load application state.
+
+- adapt for py-1.3.1's new --maxfailure option
+
+1.2
+-------------------------
+
+- fix issue79: sessionfinish/teardown hooks are now called systematically
+  on the slave side
+- introduce a new data input/output mechanism to allow the master side
+  to send and receive data from a slave.
+- fix race condition in underlying pickling/unpickling handling
+- use and require new register hooks facility of py.test>=1.3.0
+- require improved execnet>=1.0.6 because of various race conditions
+  that can arise in xdist testing modes.
+- fix some python3 related pickling related race conditions
+- fix PyPI description
+
+1.1
+-------------------------
+
+- fix an indefinite hang which would wait for events although no events
+  are pending - this happened if items arrive very quickly while
+  the "reschedule-event" tried unconditionally avoiding a busy-loop
+  and not schedule new work.
+
+1.0
+-------------------------
+
+- moved code out of py-1.1.1 into its own plugin
+- use a new, faster and more sensible model to do load-balancing
+  of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing
+  works effectively even with very few tests.
+- cleaned up termination handling
+- make -x cause hard killing of test nodes to decrease wait time
+  until the traceback shows up on first failure

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no py.test plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

LICENSE

@@ -5,10 +5,10 @@
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:
-     
+
   The above copyright notice and this permission notice shall be included in all
   copies or substantial portions of the Software.
- 
+
   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -16,4 +16,3 @@
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
   SOFTWARE.
-

MANIFEST.in

@@ -1,7 +1,5 @@
-include CHANGELOG
-include LICENSE 
-include README.txt
-include setup.py
-include tox.ini
-graft testing
-prune .hg 
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

README.rst

@@ -0,0 +1,33 @@
+============
+pytest-xdist
+============
+
+.. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
+    :alt: PyPI version
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://img.shields.io/conda/vn/conda-forge/pytest-xdist.svg
+    :target: https://anaconda.org/conda-forge/pytest-xdist
+
+.. image:: https://img.shields.io/pypi/pyversions/pytest-xdist.svg
+    :alt: Python versions
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/test/badge.svg
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/ambv/black
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+Documentation
+=============
+
+Documentation is available at `Read The Docs <https://pytest-xdist.readthedocs.io>`__.

README.txt

@@ -1,212 +0,0 @@
-xdist: pytest distributed testing plugin
-===============================================================
-
-The `pytest-xdist`_ plugin extends py.test with some unique
-test execution modes:
-
-* test run parallelization_: if you have multiple CPUs or hosts you can use
-  those for a combined test run.  This allows to speed up
-  development or to use special resources of `remote machines`_.
-
-* ``--boxed``: (not available on Windows) run each test in a boxed_
-  subprocess to survive ``SEGFAULTS`` or otherwise dying processes
-
-* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
-  py.test waits until a file in your project changes and then re-runs
-  the previously failing tests.  This is repeated until all tests pass
-  after which again a full run is performed.
-
-* `Multi-Platform`_ coverage: you can specify different Python interpreters
-  or different platforms and run tests in parallel on all of them.
-
-Before running tests remotely, ``py.test`` efficiently "rsyncs" your
-program source code to the remote place.  All test results
-are reported back and displayed to your local terminal.
-You may specify different Python versions and interpreters.
-
-
-Installation
------------------------
-
-Install the plugin with::
-
-    easy_install pytest-xdist
-
-    # or
-
-    pip install pytest-xdist
-
-or use the package in develope/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
-
-    python setup.py develop
-
-Usage examples
----------------------
-
-.. _parallelization:
-
-Speed up test runs by sending tests to multiple CPUs
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-To send tests to multiple CPUs, type::
-
-    py.test -n NUM
-
-Especially for longer running tests or tests requiring
-a lot of IO this can lead to considerable speed ups.
-
-
-Running tests in a Python subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-To instantiate a python2.5 sub process and send tests to it, you may type::
-
-    py.test -d --tx popen//python=python2.5
-
-This will start a subprocess which is run with the "python2.5"
-Python interpreter, found in your system binary lookup path.
-
-If you prefix the --tx option value like this::
-
-    --tx 3*popen//python=python2.5
-
-then three subprocesses would be created and tests
-will be load-balanced across these three processes.
-
-.. _boxed:
-
-Running tests in a boxed subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-If you have tests involving C or C++ libraries you might have to deal
-with tests crashing the process.  For this case you may use the boxing
-options::
-
-    py.test --boxed
-
-which will run each test in a subprocess and will report if a test
-crashed the process.  You can also combine this option with
-running multiple processes to speed up the test run and use your CPU cores::
-
-    py.test -n3 --boxed
-
-this would run 3 testing subprocesses in parallel which each
-create new boxed subprocesses for each test.
-
-
-.. _`remote machines`:
-
-Sending tests to remote SSH accounts
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Suppose you have a package ``mypkg`` which contains some
-tests that you can successfully run locally. And you
-have a ssh-reachable machine ``myhost``.  Then
-you can ad-hoc distribute your tests by typing::
-
-    py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
-
-This will synchronize your ``mypkg`` package directory
-to an remote ssh account and then locally collect tests
-and send them to remote places for execution.
-
-You can specify multiple ``--rsyncdir`` directories
-to be sent to the remote side.
-
-**NOTE:** For py.test to collect and send tests correctly
-you not only need to make sure all code and tests
-directories are rsynced, but that any test (sub) directory
-also has an ``__init__.py`` file because internally
-py.test references tests as a fully qualified python
-module path.  **You will otherwise get strange errors**
-during setup of the remote side.
-
-You can specify multiple ``--rsyncignore`` glob-patterns
-to be ignored when file are sent to the remote side.
-There are also internal ignores: .*, *.pyc, *.pyo, *~
-Those you cannot override using rsyncignore command-line or
-ini-file option(s).
-
-
-Sending tests to remote Socket Servers
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Download the single-module `socketserver.py`_ Python program
-and run it like this::
-
-    python socketserver.py
-
-It will tell you that it starts listening on the default
-port.  You can now on your home machine specify this
-new socket host with something like this::
-
-    py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
-
-
-.. _`atonce`:
-.. _`Multi-Platform`:
-
-
-Running tests on many platforms at once
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-The basic command to run tests on multiple platforms is::
-
-    py.test --dist=each --tx=spec1 --tx=spec2
-
-If you specify a windows host, an OSX host and a Linux
-environment this command will send each tests to all
-platforms - and report back failures from all platforms
-at once.   The specifications strings use the `xspec syntax`_.
-
-.. _`xspec syntax`: http://codespeak.net/execnet/trunk/basics.html#xspec
-
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
-
-.. _`execnet`: http://codespeak.net/execnet
-
-Specifying test exec environments in an ini file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-pytest (since version 2.0) supports ini-style cofiguration.
-You can for example make running with three subprocesses
-your default like this::
-
-    [pytest]
-    addopts = -n3
-
-You can also add default environments like this::
-
-    [pytest]
-    addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6
-
-and then just type::
-
-    py.test --dist=each
-
-to run tests in each of the environments.
-
-Specifying "rsync" dirs in an ini-file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
-you may specify directories to include or to exclude in synchronisation::
-
-    [pytest]
-    rsyncdirs = . mypkg helperpkg
-    rsyncignore = .hg
-
-These directory specifications are relative to the directory
-where the configuration file was found.
-
-.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
-.. _`pytest-xdist repository`: http://bitbucket.org/hpk42/pytest-xdist
-.. _`pytest`: http://pytest.org
-
-Issue and Bug Tracker
-------------------------
-
-Please use the pytest issue tracker for bugs in this plugin, see https://bitbucket.org/hpk42/pytest/issues .
-
-

RELEASING.rst

@@ -0,0 +1,45 @@
+======================
+Releasing pytest-xdist
+======================
+
+This document describes the steps to make a new ``pytest-xdist`` release.
+
+Version
+-------
+
+``master`` should always be green and a potential release candidate. ``pytest-xdist`` follows
+semantic versioning, so given that the current version is ``X.Y.Z``, to find the next version number
+one needs to look at the ``changelog`` folder:
+
+- If there is any file named ``*.feature``, then we must make a new **minor** release: next
+  release will be ``X.Y+1.0``.
+
+- Otherwise it is just a **bug fix** release: ``X.Y.Z+1``.
+
+
+Steps
+-----
+
+To publish a new release ``X.Y.Z``, the steps are as follows:
+
+#. Create a new branch named ``release-X.Y.Z`` from the latest ``master``.
+
+#. Install ``tox`` in a virtualenv::
+
+    $ pip install tox
+
+#. Update the necessary files with::
+
+    $ tox -e release -- X.Y.Z
+
+#. Commit and push the branch for review.
+
+#. Once PR is **green** and **approved**, create and push a tag::
+
+    $ export VERSION=X.Y.Z
+    $ git tag v$VERSION release-$VERSION
+    $ git push git@github.com:pytest-dev/pytest-xdist.git v$VERSION
+
+That will build the package and publish it on ``PyPI`` automatically.
+
+#. Merge the release PR to `master`.

changelog/_template.rst

@@ -0,0 +1,38 @@
+{% for section in sections %}
+{% set underline = "-" %}
+{% if section %}
+{{section}}
+{{ underline * section|length }}{% set underline = "~" %}
+
+{% endif %}
+{% if sections[section] %}
+{% for category, val in definitions.items() if category in sections[section] %}
+
+{{ definitions[category]['name'] }}
+{{ underline * definitions[category]['name']|length }}
+
+{% if definitions[category]['showcontent'] %}
+{% for text, values in sections[section][category]|dictsort(by='value') %}
+- `{{ values[0] }} <https://github.com/pytest-dev/pytest-xdist/issues/{{ values[0][1:] }}>`_: {{ text }}
+
+{% endfor %}
+{% else %}
+- {{ sections[section][category]['']|sort|join(', ') }}
+
+
+{% endif %}
+{% if sections[section][category]|length == 0 %}
+
+No significant changes.
+
+
+{% else %}
+{% endif %}
+{% endfor %}
+{% else %}
+
+No significant changes.
+
+
+{% endif %}
+{% endfor %}

docs/.gitignore

@@ -0,0 +1 @@
+_build/

docs/changelog.rst

@@ -0,0 +1,5 @@
+=========
+Changelog
+=========
+
+.. include:: ../CHANGELOG.rst

docs/conf.py

@@ -0,0 +1,55 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "pytest-xdist"
+copyright = "2022, holger krekel and contributors"
+author = "holger krekel and contributors"
+
+master_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx_rtd_theme",
+    "sphinx.ext.autodoc",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']

docs/crash.rst

@@ -0,0 +1,7 @@
+When tests crash
+================
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.

docs/distribution.rst

@@ -0,0 +1,94 @@
+.. _parallelization:
+
+Running tests across multiple CPUs
+==================================
+
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
+
+    pytest -n auto
+
+This can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
+
+With ``-n auto``, pytest-xdist will use as many processes as your computer
+has CPU cores.
+
+Use ``-n logical`` to use the number of *logical* CPU cores rather than
+physical ones. This currently requires the ``psutil`` package to be installed;
+if it is not, pytest-xdist will fall back to ``-n auto`` behavior.
+
+Pass a number, e.g. ``-n 8``, to specify the number of processes explicitly.
+
+To specify a different meaning for ``-n auto`` and ``-n logical`` for your
+tests, you can:
+
+* Set the environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` to the
+  desired number of processes.
+
+* Implement the ``pytest_xdist_auto_num_workers``
+  `pytest hook <https://docs.pytest.org/en/latest/how-to/writing_plugins.html>`__
+  (a ``pytest_xdist_auto_num_workers(config)`` function in e.g. ``conftest.py``)
+  that returns the number of processes to use.
+  The hook can use ``config.option.numprocesses`` to determine if the user
+  asked for ``"auto"`` or ``"logical"``, and it can return ``None`` to fall
+  back to the default.
+
+If both the hook and environment variable are specified, the hook takes
+priority.
+
+
+Parallelization can be configured further with these options:
+
+* ``--maxprocesses=maxprocesses``: limit the maximum number of workers to
+  process the tests.
+
+* ``--max-worker-restart``: maximum number of workers that can be restarted
+  when crashed (set to zero to disable this feature).
+
+The test distribution algorithm is configured with the ``--dist`` command-line option:
+
+.. _distribution modes:
+
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order.
+
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
+
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker.
+
+  .. code-block:: python
+
+      @pytest.mark.xdist_group(name="group1")
+      def test1():
+          pass
+
+
+      class TestA:
+          @pytest.mark.xdist_group("group1")
+          def test2():
+              pass
+
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
+
+* ``--dist worksteal``: Initially, tests are distributed evenly among all
+  available workers. When a worker completes most of its assigned tests and
+  doesn't have enough tests to continue (currently, every worker needs at least
+  two tests in its queue), an attempt is made to reassign ("steal") a portion
+  of tests from some other worker's queue. The results should be similar to
+  the ``load`` method, but ``worksteal`` should handle tests with significantly
+  differing duration better, and, at the same time, it should provide similar
+  or better reuse of fixtures.
+
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).

docs/how-it-works.rst

@@ -0,0 +1,90 @@
+How it works?
+=============
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* index of passing the full test
+   id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

docs/how-to.rst

@@ -0,0 +1,232 @@
+How-tos
+-------
+
+This section show cases how to accomplish some specialized tasks with ``pytest-xdist``.
+
+Identifying the worker process during a test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.15.*
+
+If you need to determine the identity of a worker process in
+a test or fixture, you may use the ``worker_id`` fixture to do so:
+
+.. code-block:: python
+
+    @pytest.fixture()
+    def user_account(worker_id):
+        """use a different account in each xdist worker"""
+        return "account_%s" % worker_id
+
+When ``xdist`` is disabled (running with ``-n0`` for example), then
+``worker_id`` will return ``"master"``.
+
+Worker processes also have the following environment variables
+defined:
+
+.. envvar:: PYTEST_XDIST_WORKER
+
+The name of the worker, e.g., ``"gw2"``.
+
+.. envvar:: PYTEST_XDIST_WORKER_COUNT
+
+The total number of workers in this session, e.g., ``"4"`` when ``-n 4`` is given in the command-line.
+
+The information about the worker_id in a test is stored in the ``TestReport`` as
+well, under the ``worker_id`` attribute.
+
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+
+.. autofunction:: xdist.is_xdist_worker
+.. autofunction:: xdist.is_xdist_controller
+.. autofunction:: xdist.is_xdist_master
+.. autofunction:: xdist.get_xdist_worker_id
+
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
+Uniquely identifying the current test run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """create a unique database for this particular test run"""
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """retrieve unique database"""
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+.. envvar:: PYTEST_XDIST_TESTRUNUID
+
+The unique id of the test run.
+
+Accessing ``sys.argv`` from the controller node in workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the ``sys.argv`` passed to the command-line of the controller node, use
+``request.config.workerinput["mainargv"]``.
+
+
+Specifying test exec environments in an ini file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use pytest's ini file configuration to avoid typing common options.
+You can for example make running with three subprocesses your default like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = -n3
+
+You can also add default environments like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
+
+and then just type::
+
+    pytest --dist=each
+
+to run tests in each of the environments.
+
+
+Specifying "rsync" dirs in an ini-file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
+you may specify directories to include or to exclude in synchronisation:
+
+.. code-block:: ini
+
+    [pytest]
+    rsyncdirs = . mypkg helperpkg
+    rsyncignore = .hg
+
+These directory specifications are relative to the directory
+where the configuration file was found.
+
+.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
+.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
+.. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+Creating one log file for each worker
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create one log file for each worker with ``pytest-xdist``, you can leverage :envvar:`PYTEST_XDIST_WORKER`
+to generate a unique filename for each worker.
+
+Example:
+
+.. code-block:: python
+
+    # content of conftest.py
+    def pytest_configure(config):
+        worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+        if worker_id is not None:
+            logging.basicConfig(
+                format=config.getini("log_file_format"),
+                filename=f"tests_{worker_id}.log",
+                level=config.getini("log_file_level"),
+            )
+
+
+When running the tests with ``-n3``, for example, three files will be created in the current directory:
+``tests_gw0.log``, ``tests_gw1.log`` and ``tests_gw2.log``.

docs/index.rst

@@ -0,0 +1,63 @@
+pytest-xdist
+============
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+.. note::
+    Due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+Features
+--------
+
+* Test run :ref:`parallelization`: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of :ref:`remote machines`.
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
+  pytest waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed (DEPRECATED).
+
+* :ref:`Multi-Platform` coverage: you can specify different Python interpreters
+  or different platforms and run tests in parallel on all of them.
+
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  program source code to the remote place.
+  You may specify different Python versions and interpreters. It does not
+  installs/synchronize dependencies however.
+
+  **Note**: this mode exists mostly for backward compatibility, as modern development
+  relies on continuous integration for multi-platform testing.
+
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Contents:
+
+    distribution
+    subprocess
+    remote
+    crash
+    how-to
+    how-it-works
+    known-limitations
+    changelog

docs/known-limitations.rst

@@ -0,0 +1,71 @@
+Known limitations
+=================
+
+pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist.
+
+Order and amount of test must be consistent
+-------------------------------------------
+
+It is not possible to have tests that differ in order or their amount across workers.
+
+This is especially true with ``pytest.mark.parametrize``, when values are produced with sets or other unordered iterables/generators.
+
+
+Example:
+
+.. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", {"a", "b"})
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+In the example above, the fact that ``set`` are not necessarily ordered can cause different workers
+to collect tests in different order, which will throw an error.
+
+Workarounds
+~~~~~~~~~~~
+
+A solution to this is to guarantee that the parametrized values have the same order.
+
+Some solutions:
+
+* Convert your sequence to a ``list``.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", ["a", "b"])
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+* Sort your sequence, guaranteeing order.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", sorted({"a", "b"}))
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+Output (stdout and stderr) from workers
+---------------------------------------
+
+The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time.
+
+However, this option does not work with ``pytest-xdist`` because `execnet <https://github.com/pytest-dev/execnet>`__ the underlying library used for communication between master and workers, does not support transferring stdout/stderr from workers.
+
+Currently, there are no plans to support this in ``pytest-xdist``.
+
+Debugging
+~~~~~~~~~
+
+This also means that debugging using PDB (or any other debugger that wants to use standard I/O) will not work. The ``--pdb`` option is disabled when distributing tests with ``pytest-xdist`` for this reason.
+
+It is generally likely best to use ``pytest-xdist`` to find failing tests and then debug them without distribution; however, if you need to debug from within a worker process (for example, to address failures that only happen when running tests concurrently), remote debuggers (for example, `python-remote-pdb <https://github.com/ionelmc/python-remote-pdb>`__ or `python-web-pdb <https://github.com/romanvm/python-web-pdb>`__) have been reported to work for this purpose.

docs/remote.rst

@@ -0,0 +1,83 @@
+
+.. _`Multi-Platform`:
+.. _`remote machines`:
+
+Sending tests to remote SSH accounts
+====================================
+
+.. deprecated:: 3.0
+
+.. warning::
+
+    This feature is deprecated because the support for ``rsync`` is faulty
+    in terms of reproducing the development environment in the remote
+    worker, and there is no clear solution moving forward.
+
+    This feature is scheduled to be removed in release 4.0, to let the team
+    focus on a smaller set of features.
+
+Suppose you have a package ``mypkg`` which contains some
+tests that you can successfully run locally. And you
+have a ssh-reachable machine ``myhost``.  Then
+you can ad-hoc distribute your tests by typing::
+
+    pytest -d  --rsyncdir mypkg --tx ssh=myhostpopen mypkg/tests/unit/test_something.py
+
+This will synchronize your :code:`mypkg` package directory
+to a remote ssh account and then locally collect tests
+and send them to remote places for execution.
+
+You can specify multiple :code:`--rsyncdir` directories
+to be sent to the remote side.
+
+.. note::
+
+  For pytest to collect and send tests correctly
+  you not only need to make sure all code and tests
+  directories are rsynced, but that any test (sub) directory
+  also has an :code:`__init__.py` file because internally
+  pytest references tests as a fully qualified python
+  module path.  **You will otherwise get strange errors**
+  during setup of the remote side.
+
+
+You can specify multiple :code:`--rsyncignore` glob patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
+Sending tests to remote Socket Servers
+--------------------------------------
+
+Download the single-module `socketserver.py`_ Python program
+and run it like this::
+
+    python socketserver.py
+
+It will tell you that it starts listening on the default
+port.  You can now on your home machine specify this
+new socket host with something like this::
+
+    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg
+
+
+
+Running tests on many platforms at once
+---------------------------------------
+
+The basic command to run tests on multiple platforms is::
+
+    pytest --dist=each --tx=spec1 --tx=spec2
+
+If you specify a windows host, an OSX host and a Linux
+environment this command will send each tests to all
+platforms - and report back failures from all platforms
+at once. The specifications strings use the `xspec syntax`_.
+
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
+
+.. _`execnet`: https://codespeak.net/execnet
+
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py

docs/subprocess.rst

@@ -0,0 +1,16 @@
+Running tests in a Python subprocess
+====================================
+
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
+
+    pytest -d --tx popen//python=python3.9
+
+This will start a subprocess which is run with the ``python3.9``
+Python interpreter, found in your system binary lookup path.
+
+If you prefix the --tx option value like this::
+
+    --tx 3*popen//python=python3.9
+
+then three subprocesses would be created and tests
+will be load-balanced across these three processes.

example/boxed.txt

@@ -1,8 +1,14 @@
+.. warning::
+
+  Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a
+  separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_
+  which can be installed independently. The ``--boxed`` command-line option is deprecated
+  and will be removed in pytest-xdist 3.0.0; use ``--forked`` from pytest-forked instead.
 
 
 If your testing involves C or C++ libraries you might have to deal
 with crashing processes.  The xdist-plugin provides the ``--boxed`` option
-to run each test in a controled subprocess.  Here is a basic example::
+to run each test in a controlled subprocess.  Here is a basic example::
 
     # content of test_module.py
 
@@ -14,19 +20,19 @@ to run each test in a controled subprocess.  Here is a basic example::
     @pytest.mark.parametrize("arg", range(50))
     def test_func(arg):
         time.sleep(0.05) # each tests takes a while
-        if arg % 19 == 0: 
+        if arg % 19 == 0:
             os.kill(os.getpid(), 15)
 
 If you run this with::
 
-    $ py.test --boxed
+    $ pytest -n1
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     collecting ... collected 50 items
-    
+
     test_module.py f..................f..................f...........
-    
+
     ================================= FAILURES =================================
     _______________________________ test_func[0] _______________________________
     /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
@@ -40,13 +46,13 @@ You'll see that a couple of tests are reported as crashing, indicated
 by lower-case ``f`` and the respective failure summary. You can also use
 the xdist-provided parallelization feature to speed up your testing::
 
-    $ py.test --boxed -n3
+    $ pytest -n3
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
     gw0 I / gw1 I / gw2 I
     gw0 [50] / gw1 [50] / gw2 [50]
-    
+
     scheduling tests via LoadScheduling
     ..f...............f..................f............
     ================================= FAILURES =================================

example/loadscope/epsilon/__init__.py

@@ -0,0 +1,37 @@
+def epsilon1(arg1, arg2=1000):
+    """Do epsilon1
+
+    Usage:
+
+    >>> epsilon1(10, 20)
+    40
+    >>> epsilon1(30)
+    1040
+    """
+    return arg1 + arg2 + 10
+
+
+def epsilon2(arg1, arg2=1000):
+    """Do epsilon2
+
+    Usage:
+
+    >>> epsilon2(10, 20)
+    -20
+    >>> epsilon2(30)
+    -980
+    """
+    return arg1 - arg2 - 10
+
+
+def epsilon3(arg1, arg2=1000):
+    """Do epsilon3
+
+    Usage:
+
+    >>> epsilon3(10, 20)
+    200
+    >>> epsilon3(30)
+    30000
+    """
+    return arg1 * arg2

example/loadscope/requirements.txt

@@ -0,0 +1,3 @@
+ipdb
+pytest
+../../

example/loadscope/test/test_alpha.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_alpha0():
+    sleep(5)
+    assert True
+
+
+def test_alpha1():
+    sleep(5)
+    assert True
+
+
+def test_alpha2():
+    sleep(5)
+    assert True
+
+
+def test_alpha3():
+    sleep(5)
+    assert True
+
+
+def test_alpha4():
+    sleep(5)
+    assert True
+
+
+def test_alpha5():
+    sleep(5)
+    assert True
+
+
+def test_alpha6():
+    sleep(5)
+    assert True
+
+
+def test_alpha7():
+    sleep(5)
+    assert True
+
+
+def test_alpha8():
+    sleep(5)
+    assert True
+
+
+def test_alpha9():
+    sleep(5)
+    assert True

example/loadscope/test/test_beta.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_beta0():
+    sleep(5)
+    assert True
+
+
+def test_beta1():
+    sleep(5)
+    assert True
+
+
+def test_beta2():
+    sleep(5)
+    assert True
+
+
+def test_beta3():
+    sleep(5)
+    assert True
+
+
+def test_beta4():
+    sleep(5)
+    assert True
+
+
+def test_beta5():
+    sleep(5)
+    assert True
+
+
+def test_beta6():
+    sleep(5)
+    assert True
+
+
+def test_beta7():
+    sleep(5)
+    assert True
+
+
+def test_beta8():
+    sleep(5)
+    assert True
+
+
+def test_beta9():
+    sleep(5)
+    assert True

example/loadscope/test/test_delta.py

@@ -0,0 +1,86 @@
+from time import sleep
+from unittest import TestCase
+
+
+class Delta1(TestCase):
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True
+
+
+class Delta2(TestCase):
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True

example/loadscope/test/test_gamma.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_gamma0():
+    sleep(5)
+    assert True
+
+
+def test_gamma1():
+    sleep(5)
+    assert True
+
+
+def test_gamma2():
+    sleep(5)
+    assert True
+
+
+def test_gamma3():
+    sleep(5)
+    assert True
+
+
+def test_gamma4():
+    sleep(5)
+    assert True
+
+
+def test_gamma5():
+    sleep(5)
+    assert True
+
+
+def test_gamma6():
+    sleep(5)
+    assert True
+
+
+def test_gamma7():
+    sleep(5)
+    assert True
+
+
+def test_gamma8():
+    sleep(5)
+    assert True
+
+
+def test_gamma9():
+    sleep(5)
+    assert True

example/loadscope/tox.ini

@@ -0,0 +1,17 @@
+[tox]
+envlist = test
+setupdir = {toxinidir}/../../
+
+[testenv:test]
+basepython = python3
+passenv = http_proxy https_proxy
+deps = -rrequirements.txt
+changedir = {envtmpdir}
+commands =
+    pytest -s -v \
+        --doctest-modules \
+        --junitxml=tests.xml \
+        --dist=loadscope \
+        --tx=8*popen \
+        {toxinidir}/test \
+        {toxinidir}/epsilon

pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = [
+  "setuptools>=45.0",
+  "setuptools-scm[toml]>=6.2.3",
+  "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+write_to = "src/xdist/_version.py"
+
+[tool.towncrier]
+package = "xdist"
+filename = "CHANGELOG.rst"
+directory = "changelog/"
+title_format = "pytest-xdist {version} ({project_date})"
+template = "changelog/_template.rst"
+
+[tool.towncrier.fragment.removal]
+name = "Removals"
+
+[tool.towncrier.fragment.deprecation]
+name = "Deprecations"
+
+[tool.towncrier.fragment.feature]
+name = "Features"
+
+[tool.towncrier.fragment.bugfix]
+name = "Bug Fixes"
+
+[tool.towncrier.fragment.vendor]
+name = "Vendored Libraries"
+
+[tool.towncrier.fragment.doc]
+name = "Improved Documentation"
+
+[tool.towncrier.fragment.trivial]
+name = "Trivial Changes"

setup.cfg

@@ -0,0 +1,81 @@
+[metadata]
+name = pytest-xdist
+description = pytest xdist plugin for distributed testing, most importantly across multiple CPUs
+long_description = file: README.rst
+license = MIT
+author = holger krekel and contributors
+author_email = pytest-dev@python.org,holger@merlinux.eu
+url = https://github.com/pytest-dev/pytest-xdist
+platforms =
+    linux
+    osx
+    win32
+classifiers =
+    Development Status :: 5 - Production/Stable
+    Framework :: Pytest
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Operating System :: POSIX
+    Operating System :: Microsoft :: Windows
+    Operating System :: MacOS :: MacOS X
+    Topic :: Software Development :: Testing
+    Topic :: Software Development :: Quality Assurance
+    Topic :: Utilities
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+license_file = LICENSE
+project_urls =
+    Documentation=https://pytest-xdist.readthedocs.io/en/latest
+    Changelog=https://pytest-xdist.readthedocs.io/en/latest/changelog.html
+    Source=https://github.com/pytest-dev/pytest-xdist
+    Tracker=https://github.com/pytest-dev/pytest-xdist/issues
+
+[options]
+packages = find:
+package_dir = =src
+zip_safe = False
+python_requires = >=3.7
+install_requires =
+    execnet>=1.1
+    pytest>=6.2.0
+
+[options.packages.find]
+where = src
+
+[options.entry_points]
+pytest11 =
+    xdist = xdist.plugin
+    xdist.looponfail = xdist.looponfail
+
+[options.extras_require]
+testing =
+    filelock
+psutil = psutil>=3.0
+setproctitle = setproctitle
+
+[flake8]
+# Ignore any errors related to formatting, let black worry/fix them.
+ignore = E501, W503, E203
+max-line-length = 100
+
+[mypy]
+mypy_path = src
+# TODO: Enable this & fix errors.
+# check_untyped_defs = True
+disallow_any_generics = True
+ignore_missing_imports = True
+no_implicit_optional = True
+show_error_codes = True
+strict_equality = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+warn_unused_configs = True
+# TODO: Enable this & fix errors.
+# no_implicit_reexport = True

setup.py

@@ -1,30 +0,0 @@
-from setuptools import setup
-
-setup(
-    name="pytest-xdist",
-    version='1.10',
-    description='py.test xdist plugin for distributed testing and loop-on-failing modes',
-    long_description=open('README.txt').read(),
-    license='MIT',
-    author='holger krekel and contributors',
-    author_email='pytest-dev@python.org,holger@merlinux.eu',
-    url='http://bitbucket.org/hpk42/pytest-xdist',
-    platforms=['linux', 'osx', 'win32'],
-    packages = ['xdist'],
-    entry_points = {'pytest11': ['xdist = xdist.plugin'],},
-    zip_safe=False,
-    install_requires = ['execnet>=1.1', 'pytest>=2.4.2'],
-    classifiers=[
-    'Development Status :: 5 - Production/Stable',
-    'Intended Audience :: Developers',
-    'License :: OSI Approved :: MIT License',
-    'Operating System :: POSIX',
-    'Operating System :: Microsoft :: Windows',
-    'Operating System :: MacOS :: MacOS X',
-    'Topic :: Software Development :: Testing',
-    'Topic :: Software Development :: Quality Assurance',
-    'Topic :: Utilities',
-    'Programming Language :: Python',
-    'Programming Language :: Python :: 3',
-    ],
-)

src/xdist/__init__.py

@@ -0,0 +1,15 @@
+from xdist.plugin import (
+    is_xdist_worker,
+    is_xdist_master,
+    get_xdist_worker_id,
+    is_xdist_controller,
+)
+from xdist._version import version as __version__
+
+__all__ = [
+    "__version__",
+    "is_xdist_worker",
+    "is_xdist_master",
+    "is_xdist_controller",
+    "get_xdist_worker_id",
+]

src/xdist/_path.py

@@ -0,0 +1,19 @@
+import os
+from itertools import chain
+from pathlib import Path
+from typing import Callable, Iterator
+
+
+def visit_path(
+    path: Path, *, filter: Callable[[Path], bool], recurse: Callable[[Path], bool]
+) -> Iterator[Path]:
+    """
+    Implements the interface of ``py.path.local.visit()`` for Path objects,
+    to simplify porting the code over from ``py.path.local``.
+    """
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames[:] = [x for x in dirnames if recurse(Path(dirpath, x))]
+        for name in chain(dirnames, filenames):
+            p = Path(dirpath, name)
+            if filter(p):
+                yield p

src/xdist/dsession.py

@@ -0,0 +1,459 @@
+import pytest
+
+from xdist.remote import Producer
+from xdist.workermanage import NodeManager
+from xdist.scheduler import (
+    EachScheduling,
+    LoadScheduling,
+    LoadScopeScheduling,
+    LoadFileScheduling,
+    LoadGroupScheduling,
+    WorkStealingScheduling,
+)
+
+
+from queue import Empty, Queue
+
+
+class Interrupted(KeyboardInterrupt):
+    """signals an immediate interruption."""
+
+
+class DSession:
+    """A pytest plugin which runs a distributed test session
+
+    At the beginning of the test session this creates a NodeManager
+    instance which creates and starts all nodes.  Nodes then emit
+    events processed in the pytest_runtestloop hook using the worker_*
+    methods.
+
+    Once a node is started it will automatically start running the
+    pytest mainloop with some custom hooks.  This means a node
+    automatically starts collecting tests.  Once tests are collected
+    it will wait for instructions.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.log = Producer("dsession", enabled=config.option.debug)
+        self.nodemanager = None
+        self.sched = None
+        self.shuttingdown = False
+        self.countfailures = 0
+        self.maxfail = config.getvalue("maxfail")
+        self.queue = Queue()
+        self._session = None
+        self._failed_collection_errors = {}
+        self._active_nodes = set()
+        self._failed_nodes_count = 0
+        self._max_worker_restart = get_default_max_worker_restart(self.config)
+        # summary message to print at the end of the session
+        self._summary_report = None
+        self.terminal = config.pluginmanager.getplugin("terminalreporter")
+        if self.terminal:
+            self.trdist = TerminalDistReporter(config)
+            config.pluginmanager.register(self.trdist, "terminaldistreporter")
+
+    @property
+    def session_finished(self):
+        """Return True if the distributed session has finished
+
+        This means all nodes have executed all test items.  This is
+        used by pytest_runtestloop to break out of its loop.
+        """
+        return bool(self.shuttingdown and not self._active_nodes)
+
+    def report_line(self, line):
+        if self.terminal and self.config.option.verbose >= 0:
+            self.terminal.write_line(line)
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_sessionstart(self, session):
+        """Creates and starts the nodes.
+
+        The nodes are setup to put their events onto self.queue.  As
+        soon as nodes start they will emit the worker_workerready event.
+        """
+        self.nodemanager = NodeManager(self.config)
+        nodes = self.nodemanager.setup_nodes(putevent=self.queue.put)
+        self._active_nodes.update(nodes)
+        self._session = session
+
+    @pytest.hookimpl
+    def pytest_sessionfinish(self, session):
+        """Shutdown all nodes."""
+        nm = getattr(self, "nodemanager", None)  # if not fully initialized
+        if nm is not None:
+            nm.teardown_nodes()
+        self._session = None
+
+    @pytest.hookimpl
+    def pytest_collection(self):
+        # prohibit collection of test items in controller process
+        return True
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_xdist_make_scheduler(self, config, log):
+        dist = config.getvalue("dist")
+        schedulers = {
+            "each": EachScheduling,
+            "load": LoadScheduling,
+            "loadscope": LoadScopeScheduling,
+            "loadfile": LoadFileScheduling,
+            "loadgroup": LoadGroupScheduling,
+            "worksteal": WorkStealingScheduling,
+        }
+        return schedulers[dist](config, log)
+
+    @pytest.hookimpl
+    def pytest_runtestloop(self):
+        self.sched = self.config.hook.pytest_xdist_make_scheduler(
+            config=self.config, log=self.log
+        )
+        assert self.sched is not None
+
+        self.shouldstop = False
+        while not self.session_finished:
+            self.loop_once()
+            if self.shouldstop:
+                self.triggershutdown()
+                raise Interrupted(str(self.shouldstop))
+        return True
+
+    def loop_once(self):
+        """Process one callback from one of the workers."""
+        while 1:
+            if not self._active_nodes:
+                # If everything has died stop looping
+                self.triggershutdown()
+                raise RuntimeError("Unexpectedly no active workers available")
+            try:
+                eventcall = self.queue.get(timeout=2.0)
+                break
+            except Empty:
+                continue
+        callname, kwargs = eventcall
+        assert callname, kwargs
+        method = "worker_" + callname
+        call = getattr(self, method)
+        self.log("calling method", method, kwargs)
+        call(**kwargs)
+        if self.sched.tests_finished:
+            self.triggershutdown()
+
+    #
+    # callbacks for processing events from workers
+    #
+
+    def worker_workerready(self, node, workerinfo):
+        """Emitted when a node first starts up.
+
+        This adds the node to the scheduler, nodes continue with
+        collection without any further input.
+        """
+        node.workerinfo = workerinfo
+        node.workerinfo["id"] = node.gateway.id
+        node.workerinfo["spec"] = node.gateway.spec
+
+        self.config.hook.pytest_testnodeready(node=node)
+        if self.shuttingdown:
+            node.shutdown()
+        else:
+            self.sched.add_node(node)
+
+    def worker_workerfinished(self, node):
+        """Emitted when node executes its pytest_sessionfinish hook.
+
+        Removes the node from the scheduler.
+
+        The node might not be in the scheduler if it had not emitted
+        workerready before shutdown was triggered.
+        """
+        self.config.hook.pytest_testnodedown(node=node, error=None)
+        if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
+            self.shouldstop = f"{node} received keyboard-interrupt"
+            self.worker_errordown(node, "keyboard-interrupt")
+            return
+        if node in self.sched.nodes:
+            crashitem = self.sched.remove_node(node)
+            assert not crashitem, (crashitem, node)
+        self._active_nodes.remove(node)
+
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
+    def worker_errordown(self, node, error):
+        """Emitted by the WorkerController when a node dies."""
+        self.config.hook.pytest_testnodedown(node=node, error=error)
+        try:
+            crashitem = self.sched.remove_node(node)
+        except KeyError:
+            pass
+        else:
+            if crashitem:
+                self.handle_crashitem(crashitem, node)
+
+        self._failed_nodes_count += 1
+        maximum_reached = (
+            self._max_worker_restart is not None
+            and self._failed_nodes_count > self._max_worker_restart
+        )
+        if maximum_reached:
+            if self._max_worker_restart == 0:
+                msg = "worker {} crashed and worker restarting disabled".format(
+                    node.gateway.id
+                )
+            else:
+                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
+        else:
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
+            self.shuttingdown = False
+            self._clone_node(node)
+        self._active_nodes.remove(node)
+
+    @pytest.hookimpl
+    def pytest_terminal_summary(self, terminalreporter):
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", f"xdist: {self._summary_report}")
+
+    def worker_collectionfinish(self, node, ids):
+        """worker has finished test collection.
+
+        This adds the collection for this node to the scheduler.  If
+        the scheduler indicates collection is finished (i.e. all
+        initial nodes have submitted their collections), then tells the
+        scheduler to schedule the collected items.  When initiating
+        scheduling the first time it logs which scheduler is in use.
+        """
+        if self.shuttingdown:
+            return
+        self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
+        # tell session which items were effectively collected otherwise
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
+        self._session.testscollected = len(ids)
+        self.sched.add_node_collection(node, ids)
+        if self.terminal:
+            self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids)))
+        if self.sched.collection_is_completed:
+            if self.terminal and not self.sched.has_pending:
+                self.trdist.ensure_show_status()
+                self.terminal.write_line("")
+                if self.config.option.verbose > 0:
+                    self.terminal.write_line(
+                        "scheduling tests via %s" % (self.sched.__class__.__name__)
+                    )
+            self.sched.schedule()
+
+    def worker_logstart(self, node, nodeid, location):
+        """Emitted when a node calls the pytest_runtest_logstart hook."""
+        self.config.hook.pytest_runtest_logstart(nodeid=nodeid, location=location)
+
+    def worker_logfinish(self, node, nodeid, location):
+        """Emitted when a node calls the pytest_runtest_logfinish hook."""
+        self.config.hook.pytest_runtest_logfinish(nodeid=nodeid, location=location)
+
+    def worker_testreport(self, node, rep):
+        """Emitted when a node calls the pytest_runtest_logreport hook."""
+        rep.node = node
+        self.config.hook.pytest_runtest_logreport(report=rep)
+        self._handlefailures(rep)
+
+    def worker_runtest_protocol_complete(self, node, item_index, duration):
+        """
+        Emitted when a node fires the 'runtest_protocol_complete' event,
+        signalling that a test has completed the runtestprotocol and should be
+        removed from the pending list in the scheduler.
+        """
+        self.sched.mark_test_complete(node, item_index, duration)
+
+    def worker_unscheduled(self, node, indices):
+        """
+        Emitted when a node fires the 'unscheduled' event, signalling that
+        some tests have been removed from the worker's queue and should be
+        sent to some worker again.
+
+        This should happen only in response to 'steal' command, so schedulers
+        not using 'steal' command don't have to implement it.
+        """
+        self.sched.remove_pending_tests_from_node(node, indices)
+
+    def worker_collectreport(self, node, rep):
+        """Emitted when a node calls the pytest_collectreport hook.
+
+        Because we only need the report when there's a failure/skip, as optimization
+        we only expect to receive failed/skipped reports from workers (#330).
+        """
+        assert not rep.passed
+        self._failed_worker_collectreport(node, rep)
+
+    def worker_warning_captured(self, warning_message, when, item):
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
+        # This hook as been removed in pytest 7.1, and we can remove support once we only
+        # support pytest >=7.1.
+        kwargs = dict(warning_message=warning_message, when=when, item=item)
+        self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
+
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
+    def _clone_node(self, node):
+        """Return new node based on an existing one.
+
+        This is normally for when a node dies, this will copy the spec
+        of the existing node and create a new one with a new id.  The
+        new node will have been setup so it will start calling the
+        "worker_*" hooks and do work soon.
+        """
+        spec = node.gateway.spec
+        spec.id = None
+        self.nodemanager.group.allocate_id(spec)
+        node = self.nodemanager.setup_node(spec, self.queue.put)
+        self._active_nodes.add(node)
+        return node
+
+    def _failed_worker_collectreport(self, node, rep):
+        # Check we haven't already seen this report (from
+        # another worker).
+        if rep.longrepr not in self._failed_collection_errors:
+            self._failed_collection_errors[rep.longrepr] = True
+            self.config.hook.pytest_collectreport(report=rep)
+            self._handlefailures(rep)
+
+    def _handlefailures(self, rep):
+        if rep.failed:
+            self.countfailures += 1
+            if self.maxfail and self.countfailures >= self.maxfail:
+                self.shouldstop = "stopping after %d failures" % (self.countfailures)
+
+    def triggershutdown(self):
+        self.log("triggering shutdown")
+        self.shuttingdown = True
+        for node in self.sched.nodes:
+            node.shutdown()
+
+    def handle_crashitem(self, nodeid, worker):
+        # XXX get more reporting info by recording pytest_runtest_logstart?
+        # XXX count no of failures and retry N times
+        runner = self.config.pluginmanager.getplugin("runner")
+        fspath = nodeid.split("::")[0]
+        msg = f"worker {worker.gateway.id!r} crashed while running {nodeid!r}"
+        rep = runner.TestReport(
+            nodeid, (fspath, None, fspath), (), "failed", msg, "???"
+        )
+        rep.node = worker
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
+        self.config.hook.pytest_runtest_logreport(report=rep)
+
+
+class TerminalDistReporter:
+    def __init__(self, config):
+        self.config = config
+        self.tr = config.pluginmanager.getplugin("terminalreporter")
+        self._status = {}
+        self._lastlen = 0
+        self._isatty = getattr(self.tr, "isatty", self.tr.hasmarkup)
+
+    def write_line(self, msg):
+        self.tr.write_line(msg)
+
+    def ensure_show_status(self):
+        if not self._isatty:
+            self.write_line(self.getstatus())
+
+    def setstatus(self, spec, status, show=True):
+        self._status[spec.id] = status
+        if show and self._isatty:
+            self.rewrite(self.getstatus())
+
+    def getstatus(self):
+        if self.config.option.verbose >= 0:
+            parts = [f"{spec.id} {self._status[spec.id]}" for spec in self._specs]
+            return " / ".join(parts)
+        else:
+            return "bringing up nodes..."
+
+    def rewrite(self, line, newline=False):
+        pline = line + " " * max(self._lastlen - len(line), 0)
+        if newline:
+            self._lastlen = 0
+            pline += "\n"
+        else:
+            self._lastlen = len(line)
+        self.tr.rewrite(pline, bold=True)
+
+    @pytest.hookimpl
+    def pytest_xdist_setupnodes(self, specs):
+        self._specs = specs
+        for spec in specs:
+            self.setstatus(spec, "I", show=False)
+        self.setstatus(spec, "I", show=True)
+        self.ensure_show_status()
+
+    @pytest.hookimpl
+    def pytest_xdist_newgateway(self, gateway):
+        if self.config.option.verbose > 0:
+            rinfo = gateway._rinfo()
+            version = "%s.%s.%s" % rinfo.version_info[:3]
+            self.rewrite(
+                "[%s] %s Python %s cwd: %s"
+                % (gateway.id, rinfo.platform, version, rinfo.cwd),
+                newline=True,
+            )
+        self.setstatus(gateway.spec, "C")
+
+    @pytest.hookimpl
+    def pytest_testnodeready(self, node):
+        if self.config.option.verbose > 0:
+            d = node.workerinfo
+            infoline = "[{}] Python {}".format(
+                d["id"], d["version"].replace("\n", " -- ")
+            )
+            self.rewrite(infoline, newline=True)
+        self.setstatus(node.gateway.spec, "ok")
+
+    @pytest.hookimpl
+    def pytest_testnodedown(self, node, error):
+        if not error:
+            return
+        self.write_line(f"[{node.gateway.id}] node down: {error}")
+
+
+def get_default_max_worker_restart(config):
+    """gets the default value of --max-worker-restart option if it is not provided.
+
+    Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
+    """
+    result = config.option.maxworkerrestart
+    if result is not None:
+        result = int(result)
+    elif config.option.numprocesses:
+        # if --max-worker-restart was not provided, use a reasonable default (#226)
+        result = config.option.numprocesses * 4
+    return result

src/xdist/looponfail.py

@@ -1,69 +1,109 @@
 """
-    Implement -f aka looponfailing for py.test.
+    Implement -f aka looponfailing for pytest.
 
     NOTE that we try to avoid loading and depending on application modules
     within the controlling process (the one that starts repeatedly test
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
+import os
+from pathlib import Path
+from typing import Dict, Sequence
 
-import py, pytest
+import pytest
 import sys
+import time
 import execnet
+from _pytest._io import TerminalWriter
 
-def looponfail_main(config):
+from xdist._path import visit_path
+
+
+@pytest.hookimpl
+def pytest_addoption(parser):
+    group = parser.getgroup("xdist", "distributed and subprocess testing")
+    group._addoption(
+        "-f",
+        "--looponfail",
+        action="store_true",
+        dest="looponfail",
+        default=False,
+        help="run tests in subprocess, wait for modified files "
+        "and re-run failing test set until all pass.",
+    )
+
+
+@pytest.hookimpl
+def pytest_cmdline_main(config):
+    if config.getoption("looponfail"):
+        usepdb = config.getoption("usepdb", False)  # a core option
+        if usepdb:
+            raise pytest.UsageError("--pdb is incompatible with --looponfail.")
+        looponfail_main(config)
+        return 2  # looponfail only can get stop with ctrl-C anyway
+
+
+def looponfail_main(config: "pytest.Config") -> None:
     remotecontrol = RemoteControl(config)
-    rootdirs = config.getini("looponfailroots")
+    config_roots = config.getini("looponfailroots")
+    if not config_roots:
+        config_roots = [Path.cwd()]
+    rootdirs = [Path(root) for root in config_roots]
     statrecorder = StatRecorder(rootdirs)
     try:
         while 1:
             remotecontrol.loop_once()
             if not remotecontrol.failures and remotecontrol.wasfailing:
-                continue # the last failures passed, let's immediately rerun all
+                # the last failures passed, let's immediately rerun all
+                continue
             repr_pytest_looponfailinfo(
-                failreports=remotecontrol.failures,
-                rootdirs=rootdirs)
+                failreports=remotecontrol.failures, rootdirs=rootdirs
+            )
             statrecorder.waitonchange(checkinterval=2.0)
     except KeyboardInterrupt:
         print()
 
-class RemoteControl(object):
+
+class RemoteControl:
     def __init__(self, config):
         self.config = config
         self.failures = []
 
     def trace(self, *args):
         if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
-            py.builtin.print_("RemoteControl:", msg)
+            msg = " ".join(str(x) for x in args)
+            print("RemoteControl:", msg)
 
     def initgateway(self):
         return execnet.makegateway("popen")
 
     def setup(self, out=None):
         if out is None:
-            out = py.io.TerminalWriter()
-        if hasattr(self, 'gateway'):
+            out = TerminalWriter()
+        if hasattr(self, "gateway"):
             raise ValueError("already have gateway %r" % self.gateway)
-        self.trace("setting up slave session")
+        self.trace("setting up worker session")
         self.gateway = self.initgateway()
-        self.channel = channel = self.gateway.remote_exec(init_slave_session,
+        self.channel = channel = self.gateway.remote_exec(
+            init_worker_session,
             args=self.config.args,
             option_dict=vars(self.config.option),
         )
         remote_outchannel = channel.receive()
+
         def write(s):
             out._file.write(s)
             out._file.flush()
+
         remote_outchannel.setcallback(write)
 
     def ensure_teardown(self):
-        if hasattr(self, 'channel'):
+        if hasattr(self, "channel"):
             if not self.channel.isclosed():
                 self.trace("closing", self.channel)
                 self.channel.close()
             del self.channel
-        if hasattr(self, 'gateway'):
+        if hasattr(self, "gateway"):
             self.trace("exiting", self.gateway)
             self.gateway.exit()
             del self.gateway
@@ -87,7 +127,7 @@ class RemoteControl(object):
         result = self.runsession()
         failures, reports, collection_failed = result
         if collection_failed:
-            reports = ["Collection failed, keeping previous failure set"]
+            pass  # "Collection failed, keeping previous failure set"
         else:
             uniq_failures = []
             for failure in failures:
@@ -95,8 +135,9 @@ class RemoteControl(object):
                     uniq_failures.append(failure)
             self.failures = uniq_failures
 
+
 def repr_pytest_looponfailinfo(failreports, rootdirs):
-    tr = py.io.TerminalWriter()
+    tr = TerminalWriter()
     if failreports:
         tr.sep("#", "LOOPONFAILING", bold=True)
         for report in failreports:
@@ -104,14 +145,15 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
                 tr.line(report, red=True)
     tr.sep("#", "waiting for changes", bold=True)
     for rootdir in rootdirs:
-        tr.line("### Watching:   %s" %(rootdir,), bold=True)
+        tr.line(f"### Watching:   {rootdir}", bold=True)
 
 
-def init_slave_session(channel, args, option_dict):
-    import os, sys
-    import py
+def init_worker_session(channel, args, option_dict):
+    import os
+    import sys
+
     outchannel = channel.gateway.newchannel()
-    sys.stdout = sys.stderr = outchannel.makefile('w')
+    sys.stdout = sys.stderr = outchannel.makefile("w")
     channel.send(outchannel)
     # prune sys.path to not contain relative paths
     newpaths = []
@@ -122,14 +164,17 @@ def init_slave_session(channel, args, option_dict):
             newpaths.append(p)
     sys.path[:] = newpaths
 
-    #fullwidth, hasmarkup = channel.receive()
+    # fullwidth, hasmarkup = channel.receive()
     from _pytest.config import Config
+
     config = Config.fromdictargs(option_dict, list(args))
     config.args = args
-    from xdist.looponfail import SlaveFailSession
-    SlaveFailSession(config, channel).main()
+    from xdist.looponfail import WorkerFailSession
 
-class SlaveFailSession:
+    WorkerFailSession(config, channel).main()
+
+
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -143,6 +188,7 @@ class SlaveFailSession:
         if self.config.option.debug:
             print(" ".join(map(str, args)))
 
+    @pytest.hookimpl
     def pytest_collection(self, session):
         self.session = session
         self.trails = self.current_command
@@ -151,25 +197,29 @@ class SlaveFailSession:
             items = session.perform_collect(self.trails or None)
         except pytest.UsageError:
             items = session.perform_collect(None)
-        hook.pytest_collection_modifyitems(session=session, config=session.config, items=items)
+        hook.pytest_collection_modifyitems(
+            session=session, config=session.config, items=items
+        )
         hook.pytest_collection_finish(session=session)
         return True
 
+    @pytest.hookimpl
     def pytest_runtest_logreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)
 
+    @pytest.hookimpl
     def pytest_collectreport(self, report):
         if report.failed:
             self.recorded_failures.append(report)
             self.collection_failed = True
 
     def main(self):
-        self.DEBUG("SLAVE: received configuration, waiting for command trails")
+        self.DEBUG("WORKER: received configuration, waiting for command trails")
         try:
             command = self.channel.receive()
         except KeyboardInterrupt:
-            return # in the slave we can't do much about this
+            return  # in the worker we can't do much about this
         self.DEBUG("received", command)
         self.current_command = command
         self.config.hook.pytest_cmdline_main(config=self.config)
@@ -177,55 +227,58 @@ class SlaveFailSession:
         for rep in self.recorded_failures:
             trails.append(rep.nodeid)
             loc = rep.longrepr
-            loc = str(getattr(loc, 'reprcrash', loc))
+            loc = str(getattr(loc, "reprcrash", loc))
             failreports.append(loc)
         self.channel.send((trails, failreports, self.collection_failed))
 
-class StatRecorder:
-    def __init__(self, rootdirlist):
-        self.rootdirlist = rootdirlist
-        self.statcache = {}
-        self.check() # snapshot state
 
-    def fil(self, p):
-        return p.check(file=1, dotfile=0) and p.ext != ".pyc"
-    def rec(self, p):
-        return p.check(dotfile=0)
+class StatRecorder:
+    def __init__(self, rootdirlist: Sequence[Path]) -> None:
+        self.rootdirlist = rootdirlist
+        self.statcache: Dict[Path, os.stat_result] = {}
+        self.check()  # snapshot state
+
+    def fil(self, p: Path) -> bool:
+        return p.is_file() and not p.name.startswith(".") and p.suffix != ".pyc"
+
+    def rec(self, p: Path) -> bool:
+        return not p.name.startswith(".") and p.exists()
 
     def waitonchange(self, checkinterval=1.0):
         while 1:
             changed = self.check()
             if changed:
                 return
-            py.std.time.sleep(checkinterval)
+            time.sleep(checkinterval)
 
-    def check(self, removepycfiles=True):
+    def check(self, removepycfiles: bool = True) -> bool:  # noqa, too complex
         changed = False
-        statcache = self.statcache
-        newstat = {}
+        newstat: Dict[Path, os.stat_result] = {}
         for rootdir in self.rootdirlist:
-            for path in rootdir.visit(self.fil, self.rec):
-                oldstat = statcache.pop(path, None)
+            for path in visit_path(rootdir, filter=self.fil, recurse=self.rec):
+                oldstat = self.statcache.pop(path, None)
                 try:
-                    newstat[path] = curstat = path.stat()
-                except py.error.ENOENT:
+                    curstat = path.stat()
+                except OSError:
                     if oldstat:
                         changed = True
                 else:
-                    if oldstat:
-                       if oldstat.mtime != curstat.mtime or \
-                          oldstat.size != curstat.size:
+                    newstat[path] = curstat
+                    if oldstat is not None:
+                        if (
+                            oldstat.st_mtime != curstat.st_mtime
+                            or oldstat.st_size != curstat.st_size
+                        ):
                             changed = True
-                            py.builtin.print_("# MODIFIED", path)
-                            if removepycfiles and path.ext == ".py":
-                                pycfile = path + "c"
-                                if pycfile.check():
-                                    pycfile.remove()
+                            print("# MODIFIED", path)
+                            if removepycfiles and path.suffix == ".py":
+                                pycfile = path.with_suffix(".pyc")
+                                if pycfile.is_file():
+                                    os.unlink(pycfile)
 
                     else:
                         changed = True
-        if statcache:
+        if self.statcache:
             changed = True
         self.statcache = newstat
         return changed
-

src/xdist/newhooks.py

@@ -0,0 +1,95 @@
+"""
+xdist hooks.
+
+Additionally, pytest-xdist will also decorate a few other hooks
+with the worker instance that executed the hook originally:
+
+``pytest_runtest_logreport``: ``rep`` parameter has a ``node`` attribute.
+
+You can use this hooks just as you would use normal pytest hooks, but some care
+must be taken in plugins in case ``xdist`` is not installed. Please see:
+
+    http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
+"""
+import pytest
+
+
+@pytest.hookspec()
+def pytest_xdist_setupnodes(config, specs):
+    """called before any remote node is set up."""
+
+
+@pytest.hookspec()
+def pytest_xdist_newgateway(gateway):
+    """called on new raw gateway creation."""
+
+
+@pytest.hookspec(
+    warn_on_impl="rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+)
+def pytest_xdist_rsyncstart(source, gateways):
+    """called before rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(
+    warn_on_impl="rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+)
+def pytest_xdist_rsyncfinish(source, gateways):
+    """called after rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_getremotemodule():
+    """called when creating remote node"""
+
+
+@pytest.hookspec()
+def pytest_configure_node(node):
+    """configure node information before it gets instantiated."""
+
+
+@pytest.hookspec()
+def pytest_testnodeready(node):
+    """Test Node is ready to operate."""
+
+
+@pytest.hookspec()
+def pytest_testnodedown(node, error):
+    """Test Node is down."""
+
+
+@pytest.hookspec()
+def pytest_xdist_node_collection_finished(node, ids):
+    """called by the controller node when a worker node finishes collecting."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_make_scheduler(config, log):
+    """return a node scheduler implementation"""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_handlecrashitem(crashitem, report, sched):
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -0,0 +1,342 @@
+import os
+import uuid
+import sys
+import warnings
+
+import pytest
+
+
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
+
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
+
+
+@pytest.hookimpl
+def pytest_xdist_auto_num_workers(config):
+    env_var = os.environ.get("PYTEST_XDIST_AUTO_NUM_WORKERS")
+    if env_var:
+        try:
+            return int(env_var)
+        except ValueError:
+            warnings.warn(
+                "PYTEST_XDIST_AUTO_NUM_WORKERS is not a number: {env_var!r}. Ignoring it."
+            )
+
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
+    try:
+        from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
+    except ImportError:
+        if os.environ.get("TRAVIS") == "true":
+            # workaround https://bitbucket.org/pypy/pypy/issues/2375
+            return 2
+        try:
+            from os import cpu_count
+        except ImportError:
+            from multiprocessing import cpu_count
+    try:
+        n = cpu_count()
+    except NotImplementedError:
+        return 1
+    return n if n else 1
+
+
+def parse_numprocesses(s):
+    if s in ("auto", "logical"):
+        return s
+    elif s is not None:
+        return int(s)
+
+
+@pytest.hookimpl
+def pytest_addoption(parser):
+    group = parser.getgroup("xdist", "distributed and subprocess testing")
+    group._addoption(
+        "-n",
+        "--numprocesses",
+        dest="numprocesses",
+        metavar="numprocesses",
+        action="store",
+        type=parse_numprocesses,
+        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
+        "to detect physical CPU count. With 'logical', detect logical CPU "
+        "count. If physical CPU count cannot be found, falls back to logical "
+        "count. This will be 0 when used with --pdb.",
+    )
+    group.addoption(
+        "--maxprocesses",
+        dest="maxprocesses",
+        metavar="maxprocesses",
+        action="store",
+        type=int,
+        help="limit the maximum number of workers to process the tests when using --numprocesses=auto",
+    )
+    group.addoption(
+        "--max-worker-restart",
+        action="store",
+        default=None,
+        dest="maxworkerrestart",
+        help="maximum number of workers that can be restarted "
+        "when crashed (set to zero to disable this feature)",
+    )
+    group.addoption(
+        "--dist",
+        metavar="distmode",
+        action="store",
+        choices=[
+            "each",
+            "load",
+            "loadscope",
+            "loadfile",
+            "loadgroup",
+            "worksteal",
+            "no",
+        ],
+        dest="dist",
+        default="no",
+        help=(
+            "set mode for distributing tests to exec environments.\n\n"
+            "each: send each test to all available environments.\n\n"
+            "load: load balance by sending any pending test to any"
+            " available environment.\n\n"
+            "loadscope: load balance by sending pending groups of tests in"
+            " the same scope to any available environment.\n\n"
+            "loadfile: load balance by sending test grouped by file"
+            " to any available environment.\n\n"
+            "loadgroup: like load, but sends tests marked with 'xdist_group' to the same worker.\n\n"
+            "worksteal: split the test suite between available environments,"
+            " then rebalance when any worker runs out of tests.\n\n"
+            "(default) no: run tests inprocess, don't distribute."
+        ),
+    )
+    group.addoption(
+        "--tx",
+        dest="tx",
+        action="append",
+        default=[],
+        metavar="xspec",
+        help=(
+            "add a test execution environment. some examples: "
+            "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
+            "--tx ssh=user@codespeak.net//chdir=testcache"
+        ),
+    )
+    group._addoption(
+        "-d",
+        action="store_true",
+        dest="distload",
+        default=False,
+        help="load-balance tests.  shortcut for '--dist=load'",
+    )
+    group.addoption(
+        "--rsyncdir",
+        action="append",
+        default=[],
+        metavar="DIR",
+        help="add directory for rsyncing to remote tx nodes.",
+    )
+    group.addoption(
+        "--rsyncignore",
+        action="append",
+        default=[],
+        metavar="GLOB",
+        help="add expression for ignores when rsyncing to remote tx nodes.",
+    )
+    group.addoption(
+        "--testrunuid",
+        action="store",
+        help=(
+            "provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture,\n\n,"
+            "if not provided, 'testrun_uid' is filled with a new unique string "
+            "on every test run."
+        ),
+    )
+    group.addoption(
+        "--maxschedchunk",
+        action="store",
+        type=int,
+        help=(
+            "Maximum number of tests scheduled in one step for --dist=load. "
+            "Setting it to 1 will force pytest to send tests to workers one by "
+            "one - might be useful for a small number of slow tests. "
+            "Larger numbers will allow the scheduler to submit consecutive "
+            "chunks of tests to workers - allows reusing fixtures. "
+            "Unlimited if not set."
+        ),
+    )
+
+    parser.addini(
+        "rsyncdirs",
+        "list of (relative) paths to be rsynced for remote distributed testing.",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
+    )
+    parser.addini(
+        "rsyncignore",
+        "list of (relative) glob-style paths to be ignored for rsyncing.",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
+    )
+    parser.addini(
+        "looponfailroots",
+        type="paths" if PYTEST_GTE_7 else "pathlist",
+        help="directories to check for changes. Default: current directory.",
+    )
+
+
+# -------------------------------------------------------------------------
+# distributed testing hooks
+# -------------------------------------------------------------------------
+
+
+@pytest.hookimpl
+def pytest_addhooks(pluginmanager):
+    from xdist import newhooks
+
+    pluginmanager.add_hookspecs(newhooks)
+
+
+# -------------------------------------------------------------------------
+# distributed testing initialization
+# -------------------------------------------------------------------------
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_configure(config):
+    config_line = (
+        "xdist_group: specify group for tests should run in same session."
+        "in relation to one another. Provided by pytest-xdist."
+    )
+    config.addinivalue_line("markers", config_line)
+
+    # Skip this plugin entirely when only doing collection.
+    if config.getvalue("collectonly"):
+        return
+
+    # Create the distributed session in case we have a valid distribution
+    # mode and test environments.
+    if config.getoption("dist") != "no" and config.getoption("tx"):
+        from xdist.dsession import DSession
+
+        session = DSession(config)
+        config.pluginmanager.register(session, "dsession")
+        tr = config.pluginmanager.getplugin("terminalreporter")
+        if tr:
+            tr.showfspath = False
+
+    # Deprecation warnings for deprecated command-line/configuration options.
+    if config.getoption("looponfail", None) or config.getini("looponfailroots"):
+        warning = DeprecationWarning(
+            "The --looponfail command line argument and looponfailroots config variable are deprecated.\n"
+            "The loop-on-fail feature will be removed in pytest-xdist 4.0."
+        )
+        config.issue_config_time_warning(warning, 2)
+
+    if config.getoption("rsyncdir", None) or config.getini("rsyncdirs"):
+        warning = DeprecationWarning(
+            "The --rsyncdir command line argument and rsyncdirs config variable are deprecated.\n"
+            "The rsync feature will be removed in pytest-xdist 4.0."
+        )
+        config.issue_config_time_warning(warning, 2)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_cmdline_main(config):
+    usepdb = config.getoption("usepdb", False)  # a core option
+    if config.option.numprocesses in ("auto", "logical"):
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
+
+    if config.option.numprocesses:
+        if config.option.dist == "no":
+            config.option.dist = "load"
+        numprocesses = config.option.numprocesses
+        if config.option.maxprocesses:
+            numprocesses = min(numprocesses, config.option.maxprocesses)
+        config.option.tx = ["popen"] * numprocesses
+    if config.option.distload:
+        config.option.dist = "load"
+    val = config.getvalue
+    if not val("collectonly") and val("dist") != "no" and usepdb:
+        raise pytest.UsageError(
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
+        )  # noqa: E501
+
+
+# -------------------------------------------------------------------------
+# fixtures and API to easily know the role of current node
+# -------------------------------------------------------------------------
+
+
+def is_xdist_worker(request_or_session) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_controller(request_or_session) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
+def get_xdist_worker_id(request_or_session):
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the controller node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        return request_or_session.config.workerinput["workerid"]
+    else:
+        # TODO: remove "master", ideally for a None
+        return "master"
+
+
+@pytest.fixture(scope="session")
+def worker_id(request):
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the master node.
+    """
+    # TODO: remove "master", ideally for a None
+    return get_xdist_worker_id(request)
+
+
+@pytest.fixture(scope="session")
+def testrun_uid(request):
+    """Return the unique id of the current test."""
+    if hasattr(request.config, "workerinput"):
+        return request.config.workerinput["testrunuid"]
+    else:
+        return uuid.uuid4().hex

src/xdist/remote.py

@@ -0,0 +1,355 @@
+"""
+    This module is executed in remote subprocesses and helps to
+    control a remote testing session and relay back information.
+    It assumes that 'py' is importable and does not have dependencies
+    on the rest of the xdist code.  This means that the xdist-plugin
+    needs not to be installed in remote environments.
+"""
+
+import contextlib
+import sys
+import os
+import time
+from typing import Any
+
+import pytest
+from execnet.gateway_base import dumps, DumpError
+
+from _pytest.config import _prepareconfig, Config
+
+try:
+    from setproctitle import setproctitle
+except ImportError:
+
+    def setproctitle(title):
+        pass
+
+
+class Producer:
+    """
+    Simplified implementation of the same interface as py.log, for backward compatibility
+    since we dropped the dependency on pylib.
+    Note: this is defined here because this module can't depend on xdist, so we need
+    to have the other way around.
+    """
+
+    def __init__(self, name: str, *, enabled: bool = True):
+        self.name = name
+        self.enabled = enabled
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r}, enabled={self.enabled})"
+
+    def __call__(self, *a: Any, **k: Any) -> None:
+        if self.enabled:
+            print(f"[{self.name}]", *a, **k, file=sys.stderr)
+
+    def __getattr__(self, name: str) -> "Producer":
+        return type(self)(name, enabled=self.enabled)
+
+
+def worker_title(title):
+    try:
+        setproctitle(title)
+    except Exception:
+        # changing the process name is very optional, no errors please
+        pass
+
+
+class WorkerInteractor:
+    SHUTDOWN_MARK = object()
+    QUEUE_REPLACED_MARK = object()
+
+    def __init__(self, config, channel):
+        self.config = config
+        self.workerid = config.workerinput.get("workerid", "?")
+        self.testrunuid = config.workerinput["testrunuid"]
+        self.log = Producer(f"worker-{self.workerid}", enabled=config.option.debug)
+        self.channel = channel
+        self.torun = self._make_queue()
+        self.nextitem_index = None
+        config.pluginmanager.register(self)
+
+    def _make_queue(self):
+        return self.channel.gateway.execmodel.queue.Queue()
+
+    def _get_next_item_index(self):
+        """Gets the next item from test queue. Handles the case when the queue
+        is replaced concurrently in another thread.
+        """
+        result = self.torun.get()
+        while result is self.QUEUE_REPLACED_MARK:
+            result = self.torun.get()
+        return result
+
+    def sendevent(self, name, **kwargs):
+        self.log("sending", name, kwargs)
+        self.channel.send((name, kwargs))
+
+    @pytest.hookimpl
+    def pytest_internalerror(self, excrepr):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
+            self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
+
+    @pytest.hookimpl
+    def pytest_sessionstart(self, session):
+        self.session = session
+        workerinfo = getinfodict()
+        self.sendevent("workerready", workerinfo=workerinfo)
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_sessionfinish(self, exitstatus):
+        # in pytest 5.0+, exitstatus is an IntEnum object
+        self.config.workeroutput["exitstatus"] = int(exitstatus)
+        yield
+        self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
+
+    @pytest.hookimpl
+    def pytest_collection(self, session):
+        self.sendevent("collectionstart")
+
+    def handle_command(self, command):
+        if command is self.SHUTDOWN_MARK:
+            self.torun.put(self.SHUTDOWN_MARK)
+            return
+
+        name, kwargs = command
+
+        self.log("received command", name, kwargs)
+        if name == "runtests":
+            for i in kwargs["indices"]:
+                self.torun.put(i)
+        elif name == "runtests_all":
+            for i in range(len(self.session.items)):
+                self.torun.put(i)
+        elif name == "shutdown":
+            self.torun.put(self.SHUTDOWN_MARK)
+        elif name == "steal":
+            self.steal(kwargs["indices"])
+
+    def steal(self, indices):
+        indices = set(indices)
+        stolen = []
+
+        old_queue, self.torun = self.torun, self._make_queue()
+
+        def old_queue_get_nowait_noraise():
+            with contextlib.suppress(self.channel.gateway.execmodel.queue.Empty):
+                return old_queue.get_nowait()
+
+        for i in iter(old_queue_get_nowait_noraise, None):
+            if i in indices:
+                stolen.append(i)
+            else:
+                self.torun.put(i)
+
+        self.sendevent("unscheduled", indices=stolen)
+        old_queue.put(self.QUEUE_REPLACED_MARK)
+
+    @pytest.hookimpl
+    def pytest_runtestloop(self, session):
+        self.log("entering main loop")
+        self.channel.setcallback(self.handle_command, endmarker=self.SHUTDOWN_MARK)
+        self.nextitem_index = self._get_next_item_index()
+        while self.nextitem_index is not self.SHUTDOWN_MARK:
+            self.run_one_test()
+        return True
+
+    def run_one_test(self):
+        self.item_index = self.nextitem_index
+        self.nextitem_index = self._get_next_item_index()
+
+        items = self.session.items
+        item = items[self.item_index]
+        if self.nextitem_index is self.SHUTDOWN_MARK:
+            nextitem = None
+        else:
+            nextitem = items[self.nextitem_index]
+
+        worker_title("[pytest-xdist running] %s" % item.nodeid)
+
+        start = time.time()
+        self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
+        duration = time.time() - start
+
+        worker_title("[pytest-xdist idle]")
+
+        self.sendevent(
+            "runtest_protocol_complete", item_index=self.item_index, duration=duration
+        )
+
+    def pytest_collection_modifyitems(self, session, config, items):
+        # add the group name to nodeid as suffix if --dist=loadgroup
+        if config.getvalue("loadgroup"):
+            for item in items:
+                mark = item.get_closest_marker("xdist_group")
+                if not mark:
+                    continue
+                gname = (
+                    mark.args[0]
+                    if len(mark.args) > 0
+                    else mark.kwargs.get("name", "default")
+                )
+                item._nodeid = f"{item.nodeid}@{gname}"
+
+    @pytest.hookimpl
+    def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
+        self.sendevent(
+            "collectionfinish",
+            topdir=topdir,
+            ids=[item.nodeid for item in session.items],
+        )
+
+    @pytest.hookimpl
+    def pytest_runtest_logstart(self, nodeid, location):
+        self.sendevent("logstart", nodeid=nodeid, location=location)
+
+    @pytest.hookimpl
+    def pytest_runtest_logfinish(self, nodeid, location):
+        self.sendevent("logfinish", nodeid=nodeid, location=location)
+
+    @pytest.hookimpl
+    def pytest_runtest_logreport(self, report):
+        data = self.config.hook.pytest_report_to_serializable(
+            config=self.config, report=report
+        )
+        data["item_index"] = self.item_index
+        data["worker_id"] = self.workerid
+        data["testrun_uid"] = self.testrunuid
+        assert self.session.items[self.item_index].nodeid == report.nodeid
+        self.sendevent("testreport", data=data)
+
+    @pytest.hookimpl
+    def pytest_collectreport(self, report):
+        # send only reports that have not passed to controller as optimization (#330)
+        if not report.passed:
+            data = self.config.hook.pytest_report_to_serializable(
+                config=self.config, report=report
+            )
+            self.sendevent("collectreport", data=data)
+
+    @pytest.hookimpl
+    def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+        self.sendevent(
+            "warning_recorded",
+            warning_message_data=serialize_warning_message(warning_message),
+            when=when,
+            nodeid=nodeid,
+            location=location,
+        )
+
+
+def serialize_warning_message(warning_message):
+    if isinstance(warning_message.message, Warning):
+        message_module = type(warning_message.message).__module__
+        message_class_name = type(warning_message.message).__name__
+        message_str = str(warning_message.message)
+        # check now if we can serialize the warning arguments (#349)
+        # if not, we will just use the exception message on the controller node
+        try:
+            dumps(warning_message.message.args)
+        except DumpError:
+            message_args = None
+        else:
+            message_args = warning_message.message.args
+    else:
+        message_str = warning_message.message
+        message_module = None
+        message_class_name = None
+        message_args = None
+    if warning_message.category:
+        category_module = warning_message.category.__module__
+        category_class_name = warning_message.category.__name__
+    else:
+        category_module = None
+        category_class_name = None
+
+    result = {
+        "message_str": message_str,
+        "message_module": message_module,
+        "message_class_name": message_class_name,
+        "message_args": message_args,
+        "category_module": category_module,
+        "category_class_name": category_class_name,
+    }
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warning_message._WARNING_DETAILS:
+        if attr_name in ("message", "category"):
+            continue
+        attr = getattr(warning_message, attr_name)
+        # Check if we can serialize the warning detail, marking `None` otherwise
+        # Note that we need to define the attr (even as `None`) to allow deserializing
+        try:
+            dumps(attr)
+        except DumpError:
+            result[attr_name] = repr(attr)
+        else:
+            result[attr_name] = attr
+    return result
+
+
+def getinfodict():
+    import platform
+
+    return dict(
+        version=sys.version,
+        version_info=tuple(sys.version_info),
+        sysplatform=sys.platform,
+        platform=platform.platform(),
+        executable=sys.executable,
+        cwd=os.getcwd(),
+    )
+
+
+def remote_initconfig(option_dict, args):
+    option_dict["plugins"].append("no:terminal")
+    return Config.fromdictargs(option_dict, args)
+
+
+def setup_config(config, basetemp):
+    config.option.loadgroup = config.getvalue("dist") == "loadgroup"
+    config.option.looponfail = False
+    config.option.usepdb = False
+    config.option.dist = "no"
+    config.option.distload = False
+    config.option.numprocesses = None
+    config.option.maxprocesses = None
+    config.option.basetemp = basetemp
+
+
+if __name__ == "__channelexec__":
+    channel = channel  # type: ignore[name-defined] # noqa: F821
+    workerinput, args, option_dict, change_sys_path = channel.receive()  # type: ignore[name-defined]
+
+    if change_sys_path is None:
+        importpath = os.getcwd()
+        sys.path.insert(0, importpath)
+        os.environ["PYTHONPATH"] = (
+            importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
+        )
+    else:
+        sys.path = change_sys_path
+
+    os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
+    os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
+    os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
+
+    if hasattr(Config, "InvocationParams"):
+        config = _prepareconfig(args, None)
+    else:
+        config = remote_initconfig(option_dict, args)
+        config.args = args
+
+    setup_config(config, option_dict.get("basetemp"))
+    config._parser.prog = os.path.basename(workerinput["mainargv"][0])
+    config.workerinput = workerinput  # type: ignore[attr-defined]
+    config.workeroutput = {}  # type: ignore[attr-defined]
+    interactor = WorkerInteractor(config, channel)  # type: ignore[name-defined]
+    config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -0,0 +1,21 @@
+from difflib import unified_diff
+
+
+def report_collection_diff(from_collection, to_collection, from_id, to_id):
+    """Report the collected test difference between two nodes.
+
+    :returns: detailed message describing the difference between the given
+    collections, or None if they are equal.
+    """
+    if from_collection == to_collection:
+        return None
+
+    diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
+    error_message = (
+        "Different tests were collected between {from_id} and {to_id}. "
+        "The difference is:\n"
+        "{diff}\n"
+        "To see why this happens see Known limitations in documentation"
+    ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
+    return msg

src/xdist/scheduler/__init__.py

@@ -0,0 +1,6 @@
+from xdist.scheduler.each import EachScheduling  # noqa
+from xdist.scheduler.load import LoadScheduling  # noqa
+from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
+from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
+from xdist.scheduler.loadgroup import LoadGroupScheduling  # noqa
+from xdist.scheduler.worksteal import WorkStealingScheduling  # noqa

src/xdist/scheduler/each.py

@@ -0,0 +1,139 @@
+from xdist.remote import Producer
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class EachScheduling:
+    """Implement scheduling of test items on all nodes
+
+    If a node gets added after the test run is started then it is
+    assumed to replace a node which got removed before it finished
+    its collection.  In this case it will only be used if a node
+    with the same spec got removed earlier.
+
+    Any nodes added after the run is started will only get items
+    assigned if a node with a matching spec was removed before it
+    finished all its pending items.  The new node will then be
+    assigned the remaining items from the removed node.
+    """
+
+    def __init__(self, config, log=None):
+        self.config = config
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self._started = []
+        self._removed2pending = {}
+        if log is None:
+            self.log = Producer("eachsched")
+        else:
+            self.log = log.eachsched
+        self.collection_is_completed = False
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def tests_finished(self):
+        if not self.collection_is_completed:
+            return False
+        if self._removed2pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        Collection is complete once all nodes have submitted their
+        collection.  In this case its pending list is set to an empty
+        list.  When the collection is already completed this
+        submission is from a node which was restarted to replace a
+        dead node.  In this case we already assign the pending items
+        here.  In either case ``.schedule()`` will instruct the
+        node to start running the required tests.
+        """
+        assert node in self.node2pending
+        if not self.collection_is_completed:
+            self.node2collection[node] = list(collection)
+            self.node2pending[node] = []
+            if len(self.node2collection) >= self.numnodes:
+                self.collection_is_completed = True
+        elif self._removed2pending:
+            for deadnode in self._removed2pending:
+                if deadnode.gateway.spec == node.gateway.spec:
+                    dead_collection = self.node2collection[deadnode]
+                    if collection != dead_collection:
+                        msg = report_collection_diff(
+                            dead_collection,
+                            collection,
+                            deadnode.gateway.id,
+                            node.gateway.id,
+                        )
+                        self.log(msg)
+                        return
+                    pending = self._removed2pending.pop(deadnode)
+                    self.node2pending[node] = pending
+                    break
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        self.node2pending[node].remove(item_index)
+
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
+    def remove_node(self, node):
+        # KeyError if we didn't get an add_node() yet
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+        crashitem = self.node2collection[node][pending.pop(0)]
+        if pending:
+            self._removed2pending[node] = pending
+        return crashitem
+
+    def schedule(self):
+        """Schedule the test items on the nodes
+
+        If the node's pending list is empty it is a new node which
+        needs to run all the tests.  If the pending list is already
+        populated (by ``.add_node_collection()``) then it replaces a
+        dead node and we only need to run those tests.
+        """
+        assert self.collection_is_completed
+        for node, pending in self.node2pending.items():
+            if node in self._started:
+                continue
+            if not pending:
+                pending[:] = range(len(self.node2collection[node]))
+                node.send_runtest_all()
+                node.shutdown()
+            else:
+                node.send_runtest_some(pending)
+            self._started.append(node)

src/xdist/scheduler/load.py

@@ -0,0 +1,315 @@
+from itertools import cycle
+
+from _pytest.runner import CollectReport
+
+from xdist.remote import Producer
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class LoadScheduling:
+    """Implement load scheduling across nodes.
+
+    This distributes the tests collected across all nodes so each test
+    is run just once.  All nodes collect and submit the test suite and
+    when all collections are received it is verified they are
+    identical collections.  Then the collection gets divided up in
+    chunks and chunks get submitted to nodes.  Whenever a node finishes
+    an item, it calls ``.mark_test_complete()`` which will trigger the
+    scheduler to assign more tests if the number of pending tests for
+    the node falls below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to
+    submit a collection. This is used to know when all nodes have
+    finished collection or how large the chunks need to be created.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self.pending = []
+        self.collection = None
+        if log is None:
+            self.log = Producer("loadsched")
+        else:
+            self.log = log.loadsched
+        self.config = config
+        self.maxschedchunk = self.config.getoption("maxschedchunk")
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node
+
+        The duration it took to execute the item is used as a hint to
+        the scheduler.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule(node, duration=duration)
+
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
+    def check_schedule(self, node, duration=0):
+        """Maybe schedule new items on the node
+
+        If there are any globally pending nodes left then this will
+        check if the given node should be given any more tests.  The
+        ``duration`` of the last test is optionally used as a
+        heuristic to influence how many tests the node is assigned.
+        """
+        if node.shutting_down:
+            return
+
+        if self.pending:
+            # how many nodes do we have?
+            num_nodes = len(self.node2pending)
+            # if our node goes below a heuristic minimum, fill it out to
+            # heuristic maximum
+            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
+            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
+            node_pending = self.node2pending[node]
+            if len(node_pending) < items_per_node_min:
+                if duration >= 0.1 and len(node_pending) >= 2:
+                    # seems the node is doing long-running tests
+                    # and has enough items to continue
+                    # so let's rather wait with sending new items
+                    return
+                num_send = items_per_node_max - len(node_pending)
+                # keep at least 2 tests pending even if --maxschedchunk=1
+                maxschedchunk = max(2 - len(node_pending), self.maxschedchunk)
+                self._send_tests(node, min(num_send, maxschedchunk))
+        else:
+            node.shutdown()
+
+        self.log("num items waiting for node:", len(self.pending))
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+
+        """
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+
+        # The node crashed, reassing pending items
+        crashitem = self.collection[pending.pop(0)]
+        self.pending.extend(pending)
+        for node in self.node2pending:
+            self.check_schedule(node)
+        return crashitem
+
+    def schedule(self):
+        """Initiate distribution of the test collection
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self.check_schedule(node)
+            return
+
+        # XXX allow nodes to have different collections
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = list(self.node2collection.values())[0]
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        if self.maxschedchunk is None:
+            self.maxschedchunk = len(self.collection)
+
+        # Send a batch of tests to run. If we don't have at least two
+        # tests per node, we have to send them all so that we can send
+        # shutdown signals and get all nodes working.
+        if len(self.pending) < 2 * len(self.nodes):
+            # Distribute tests round-robin. Try to load all nodes if there are
+            # enough tests. The other branch tries sends at least 2 tests
+            # to each node - which is suboptimal when you have less than
+            # 2 * len(nodes) tests.
+            nodes = cycle(self.nodes)
+            for i in range(len(self.pending)):
+                self._send_tests(next(nodes), 1)
+        else:
+            # Send batches of consecutive tests. By default, pytest sorts tests
+            # in order for optimal single-threaded execution, minimizing the
+            # number of necessary fixture setup/teardown. Try to keep that
+            # optimal order for every worker.
+
+            # how many items per node do we have about?
+            items_per_node = len(self.collection) // len(self.node2pending)
+            # take a fraction of tests for initial distribution
+            node_chunksize = min(items_per_node // 4, self.maxschedchunk)
+            node_chunksize = max(node_chunksize, 2)
+            # and initialize each node with a chunk of tests
+            for node in self.nodes:
+                self._send_tests(node, node_chunksize)
+
+        if not self.pending:
+            # initial distribution sent all tests, start node shutdown
+            for node in self.nodes:
+                node.shutdown()
+
+    def _send_tests(self, node, num):
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = CollectReport(
+                        node.gateway.id, "failed", longrepr=msg, result=[]
+                    )
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/scheduler/loadfile.py

@@ -0,0 +1,52 @@
+from .loadscope import LoadScopeScheduling
+from xdist.remote import Producer
+
+
+class LoadFileScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test test file.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test file,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    This class behaves very much like LoadScopeScheduling, but with a file-level scope.
+    """
+
+    def __init__(self, config, log=None):
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadfilesched")
+        else:
+            self.log = log.loadfilesched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the left. That is, test will be grouped in a
+        single work unit when they reside in the same file.
+         In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.split("::", 1)[0]

src/xdist/scheduler/loadgroup.py

@@ -0,0 +1,54 @@
+from .loadscope import LoadScopeScheduling
+from xdist.remote import Producer
+
+
+class LoadGroupScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test by xdist_group mark.
+
+    This class behaves very much like LoadScopeScheduling, but it groups tests by xdist_group mark
+    instead of the module or class to which they belong to.
+    """
+
+    def __init__(self, config, log=None):
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadgroupsched")
+        else:
+            self.log = log.loadgroupsched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        With loadgroup, two cases are added::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            example/loadsuite/test/test_gamma.py::test_beta0@gname
+            example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname
+
+        This function will group tests with the scope determined by splitting the first ``@``
+        from the right. That is, test will be grouped in a single work unit when they have
+        same group name. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            gname
+            gname
+        """
+        if nodeid.rfind("@") > nodeid.rfind("]"):
+            # check the index of ']' to avoid the case: parametrize mark value has '@'
+            return nodeid.split("@")[-1]
+        else:
+            return nodeid

src/xdist/scheduler/loadscope.py

@@ -0,0 +1,410 @@
+from collections import OrderedDict
+
+from _pytest.runner import CollectReport
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_spec_config
+
+
+class LoadScopeScheduling:
+    """Implement load scheduling across nodes, but grouping test by scope.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test scope,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual number of
+       nodes will vary during the scheduler's lifetime as nodes are added by
+       the DSession as they are brought up and removed either because of a dead
+       node or normal shutdown.  This number is primarily used to know when the
+       initial collection is completed.
+
+    :collection: The final list of tests collected by all nodes once it is
+       validated to be identical between all the nodes.  It is initialised to
+       None until ``.schedule()`` is called.
+
+    :workqueue: Ordered dictionary that maps all available scopes with their
+       associated tests (nodeid). Nodeids are in turn associated with their
+       completion status. One entry of the workqueue is called a work unit.
+       In turn, a collection of work unit is called a workload.
+
+       ::
+
+            workqueue = {
+                '<full>/<path>/<to>/test_module.py': {
+                    '<full>/<path>/<to>/test_module.py::test_case1': False,
+                    '<full>/<path>/<to>/test_module.py::test_case2': False,
+                    (...)
+                },
+                (...)
+            }
+
+    :assigned_work: Ordered dictionary that maps worker nodes with their
+       assigned work units.
+
+       ::
+
+            assigned_work = {
+                '<worker node A>': {
+                    '<full>/<path>/<to>/test_module.py': {
+                        '<full>/<path>/<to>/test_module.py::test_case1': False,
+                        '<full>/<path>/<to>/test_module.py::test_case2': False,
+                        (...)
+                    },
+                    (...)
+                },
+                (...)
+            }
+
+    :registered_collections: Ordered dictionary that maps worker nodes with
+       their collection of tests gathered during test discovery.
+
+       ::
+
+            registered_collections = {
+                '<worker node A>': [
+                    '<full>/<path>/<to>/test_module.py::test_case1',
+                    '<full>/<path>/<to>/test_module.py::test_case2',
+                ],
+                (...)
+            }
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.collection = None
+
+        self.workqueue = OrderedDict()
+        self.assigned_work = OrderedDict()
+        self.registered_collections = OrderedDict()
+
+        if log is None:
+            self.log = Producer("loadscopesched")
+        else:
+            self.log = log.loadscopesched
+
+        self.config = config
+
+    @property
+    def nodes(self):
+        """A list of all active nodes in the scheduler."""
+        return list(self.assigned_work.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes have
+        finished collection.  The required number of initial nodes is defined
+        by ``.numnodes``.
+        """
+        return len(self.registered_collections) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+
+        if self.workqueue:
+            return False
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) >= 2:
+                return False
+
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are still
+        processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.workqueue:
+            return True
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) > 0:
+                return True
+
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be assigned work units to be executed.
+
+        Called by the ``DSession.worker_workerready`` hook when it successfully
+        bootstraps a new node.
+        """
+        assert node not in self.assigned_work
+        self.assigned_work[node] = OrderedDict()
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at shutdown time.
+        In the former case any pending items assigned to the node will be
+        re-scheduled.
+
+        Called by the hooks:
+
+        - ``DSession.worker_workerfinished``.
+        - ``DSession.worker_errordown``.
+
+        Return the item being executed while the node crashed or None if the
+        node has no more pending items.
+        """
+        workload = self.assigned_work.pop(node)
+        if not self._pending_of(workload):
+            return None
+
+        # The node crashed, identify test that crashed
+        for work_unit in workload.values():
+            for nodeid, completed in work_unit.items():
+                if not completed:
+                    crashitem = nodeid
+                    break
+            else:
+                continue
+            break
+        else:
+            raise RuntimeError(
+                "Unable to identify crashitem on a workload with pending items"
+            )
+
+        # Made uncompleted work unit available again
+        self.workqueue.update(workload)
+
+        for node in self.assigned_work:
+            self._reschedule(node)
+
+        return crashitem
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.registered_collections`` dictionary.
+
+        Called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+
+        # Check that add_node() was called on the node before
+        assert node in self.assigned_work
+
+        # A new node has been added later, perhaps an original one died.
+        if self.collection_is_completed:
+            # Assert that .schedule() should have been called by now
+            assert self.collection
+
+            # Check that the new collection matches the official collection
+            if collection != self.collection:
+                other_node = next(iter(self.registered_collections.keys()))
+
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+
+        self.registered_collections[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node.
+
+        Called by the hook:
+
+        - ``DSession.worker_testreport``.
+        """
+        nodeid = self.registered_collections[node][item_index]
+        scope = self._split_scope(nodeid)
+
+        self.assigned_work[node][scope][nodeid] = True
+        self._reschedule(node)
+
+    def mark_test_pending(self, item):
+        raise NotImplementedError()
+
+    def _assign_work_unit(self, node):
+        """Assign a work unit to a node."""
+        assert self.workqueue
+
+        # Grab a unit of work
+        scope, work_unit = self.workqueue.popitem(last=False)
+
+        # Keep track of the assigned work
+        assigned_to_node = self.assigned_work.setdefault(node, default=OrderedDict())
+        assigned_to_node[scope] = work_unit
+
+        # Ask the node to execute the workload
+        worker_collection = self.registered_collections[node]
+        nodeids_indexes = [
+            worker_collection.index(nodeid)
+            for nodeid, completed in work_unit.items()
+            if not completed
+        ]
+
+        node.send_runtest_some(nodeids_indexes)
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the right. That is, classes will be grouped in a
+        single work unit, and functions from a test module will be grouped by
+        their module. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py::Delta1
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.rsplit("::", 1)[0]
+
+    def _pending_of(self, workload):
+        """Return the number of pending tests in a workload."""
+        pending = sum(list(scope.values()).count(False) for scope in workload.values())
+        return pending
+
+    def _reschedule(self, node):
+        """Maybe schedule new items on the node.
+
+        If there are any globally pending work units left then this will check
+        if the given node should be given any more tests.
+        """
+
+        # Do not add more work to a node shutting down
+        if node.shutting_down:
+            return
+
+        # Check that more work is available
+        if not self.workqueue:
+            node.shutdown()
+            return
+
+        self.log("Number of units waiting for node:", len(self.workqueue))
+
+        # Check that the node is almost depleted of work
+        # 2: Heuristic of minimum tests to enqueue more work
+        if self._pending_of(self.assigned_work[node]) > 2:
+            return
+
+        # Pop one unit of work and assign it
+        self._assign_work_unit(node)
+
+    def schedule(self):
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this gets called
+        again later it behaves the same as calling ``._reschedule()`` on all
+        nodes so that newly added nodes will start to be used.
+
+        If ``.collection_is_completed`` is True, this is called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self._reschedule(node)
+            return
+
+        # Check that all nodes collected the same tests
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the final list of items
+        self.collection = list(next(iter(self.registered_collections.values())))
+        if not self.collection:
+            return
+
+        # Determine chunks of work (scopes)
+        for nodeid in self.collection:
+            scope = self._split_scope(nodeid)
+            work_unit = self.workqueue.setdefault(scope, default=OrderedDict())
+            work_unit[nodeid] = False
+
+        # Avoid having more workers than work
+        extra_nodes = len(self.nodes) - len(self.workqueue)
+
+        if extra_nodes > 0:
+            self.log(f"Shutting down {extra_nodes} nodes")
+
+            for _ in range(extra_nodes):
+                unused_node, assigned = self.assigned_work.popitem(last=True)
+
+                self.log(f"Shutting down unused node {unused_node}")
+                unused_node.shutdown()
+
+        # Assign initial workload
+        for node in self.nodes:
+            self._assign_work_unit(node)
+
+        # Ensure nodes start with at least two work units if possible (#277)
+        for node in self.nodes:
+            self._reschedule(node)
+
+        # Initial distribution sent all tests, start node shutdown
+        if not self.workqueue:
+            for node in self.nodes:
+                node.shutdown()
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.registered_collections.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if not msg:
+                continue
+
+            same_collection = False
+            self.log(msg)
+
+            if self.config is None:
+                continue
+
+            rep = CollectReport(node.gateway.id, "failed", longrepr=msg, result=[])
+            self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/scheduler/worksteal.py

@@ -0,0 +1,325 @@
+from collections import namedtuple
+
+from _pytest.runner import CollectReport
+
+from xdist.remote import Producer
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+NodePending = namedtuple("NodePending", ["node", "pending"])
+
+# Every worker needs at least 2 tests in queue - the current and the next one.
+MIN_PENDING = 2
+
+
+class WorkStealingScheduling:
+    """Implement work-stealing scheduling.
+
+    Initially, tests are distributed evenly among all nodes.
+
+    When some node completes most of its assigned tests (when only one pending
+    test remains), an attempt is made to reassign ("steal") some tests from
+    other nodes to this node.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+
+    :steal_requested_from_node: The node to which the current "steal" request
+       was sent. ``None`` if there is no request in progress. Only one request
+       can be in progress at any time, the scheduler doesn't send multiple
+       simultaneous requests.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self.pending = []
+        self.collection = None
+        if log is None:
+            self.log = Producer("workstealsched")
+        else:
+            self.log = log.workstealsched
+        self.config = config
+        self.steal_requested_from_node = None
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        if self.steal_requested_from_node is not None:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= MIN_PENDING:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=None):
+        """Mark test item as completed by node
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule()
+
+    def mark_test_pending(self, item):
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        self.check_schedule()
+
+    def remove_pending_tests_from_node(self, node, indices):
+        """Node returned some test indices back in response to 'steal' command.
+
+        This is called by ``DSession.worker_unscheduled``.
+        """
+        assert node is self.steal_requested_from_node
+        self.steal_requested_from_node = None
+
+        indices_set = set(indices)
+        self.node2pending[node] = [
+            i for i in self.node2pending[node] if i not in indices_set
+        ]
+        self.pending.extend(indices)
+        self.check_schedule()
+
+    def check_schedule(self):
+        """Reschedule tests/perform load balancing."""
+        nodes_up = [
+            NodePending(node, pending)
+            for node, pending in self.node2pending.items()
+            if not node.shutting_down
+        ]
+
+        def get_idle_nodes():
+            return [node for node, pending in nodes_up if len(pending) < MIN_PENDING]
+
+        idle_nodes = get_idle_nodes()
+        if not idle_nodes:
+            return
+
+        if self.pending:
+            # Distribute pending tests evenly among idle nodes
+            for i, node in enumerate(idle_nodes):
+                nodes_remaining = len(idle_nodes) - i
+                num_send = len(self.pending) // nodes_remaining
+                self._send_tests(node, num_send)
+
+            idle_nodes = get_idle_nodes()
+            # No need to steal anything if all nodes have enough work to continue
+            if not idle_nodes:
+                return
+
+        # Only one active stealing request is allowed
+        if self.steal_requested_from_node is not None:
+            return
+
+        # Find the node that has the longest test queue
+        steal_from = max(
+            nodes_up, key=lambda node_pending: len(node_pending.pending), default=None
+        )
+
+        if steal_from is None:
+            num_steal = 0
+        else:
+            # Steal half of the test queue - but keep that node running too.
+            # If the node has 2 or less tests queued, stealing will fail
+            # anyway.
+            max_steal = max(0, len(steal_from.pending) - MIN_PENDING)
+            num_steal = min(len(steal_from.pending) // 2, max_steal)
+
+        if num_steal == 0:
+            # Can't get more work - shutdown idle nodes. This will force them
+            # to run the last test now instead of waiting for more tests.
+            for node in idle_nodes:
+                node.shutdown()
+            return
+
+        steal_from.node.send_steal(steal_from.pending[-num_steal:])
+        self.steal_requested_from_node = steal_from.node
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+
+        """
+        pending = self.node2pending.pop(node)
+
+        # If node was removed without completing its assigned tests - it crashed
+        if pending:
+            crashitem = self.collection[pending.pop(0)]
+        else:
+            crashitem = None
+
+        self.pending.extend(pending)
+
+        # Dead node won't respond to "steal" request
+        if self.steal_requested_from_node is node:
+            self.steal_requested_from_node = None
+
+        self.check_schedule()
+        return crashitem
+
+    def schedule(self):
+        """Initiate distribution of the test collection
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            self.check_schedule()
+            return
+
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = list(self.node2collection.values())[0]
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        self.check_schedule()
+
+    def _send_tests(self, node, num):
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = CollectReport(
+                        node.gateway.id, "failed", longrepr=msg, result=[]
+                    )
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/workermanage.py

@@ -0,0 +1,450 @@
+import fnmatch
+import os
+import re
+import sys
+import uuid
+from pathlib import Path
+from typing import List, Union, Sequence, Optional, Any, Tuple, Set
+
+import pytest
+import execnet
+
+import xdist.remote
+from xdist.remote import Producer
+from xdist.plugin import _sys_path
+
+
+def parse_spec_config(config):
+    xspeclist = []
+    for xspec in config.getvalue("tx"):
+        i = xspec.find("*")
+        try:
+            num = int(xspec[:i])
+        except ValueError:
+            xspeclist.append(xspec)
+        else:
+            xspeclist.extend([xspec[i + 1 :]] * num)
+    if not xspeclist:
+        raise pytest.UsageError(
+            "MISSING test execution (tx) nodes: please specify --tx"
+        )
+    return xspeclist
+
+
+class NodeManager:
+    EXIT_TIMEOUT = 10
+    DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
+
+    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache") -> None:
+        self.config = config
+        self.trace = self.config.trace.get("nodemanager")
+        self.testrunuid = self.config.getoption("testrunuid")
+        if self.testrunuid is None:
+            self.testrunuid = uuid.uuid4().hex
+        self.group = execnet.Group()
+        if specs is None:
+            specs = self._getxspecs()
+        self.specs = []
+        for spec in specs:
+            if not isinstance(spec, execnet.XSpec):
+                spec = execnet.XSpec(spec)
+            if not spec.chdir and not spec.popen:
+                spec.chdir = defaultchdir
+            self.group.allocate_id(spec)
+            self.specs.append(spec)
+        self.roots = self._getrsyncdirs()
+        self.rsyncoptions = self._getrsyncoptions()
+        self._rsynced_specs: Set[Tuple[Any, Any]] = set()
+
+    def rsync_roots(self, gateway):
+        """Rsync the set of roots to the node's gateway cwd."""
+        if self.roots:
+            for root in self.roots:
+                self.rsync(gateway, root, **self.rsyncoptions)
+
+    def setup_nodes(self, putevent):
+        self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
+        self.trace("setting up nodes")
+        return [self.setup_node(spec, putevent) for spec in self.specs]
+
+    def setup_node(self, spec, putevent):
+        gw = self.group.makegateway(spec)
+        self.config.hook.pytest_xdist_newgateway(gateway=gw)
+        self.rsync_roots(gw)
+        node = WorkerController(self, gw, self.config, putevent)
+        gw.node = node  # keep the node alive
+        node.setup()
+        self.trace("started node %r" % node)
+        return node
+
+    def teardown_nodes(self):
+        self.group.terminate(self.EXIT_TIMEOUT)
+
+    def _getxspecs(self):
+        return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
+
+    def _getrsyncdirs(self) -> List[Path]:
+        for spec in self.specs:
+            if not spec.popen or spec.chdir:
+                break
+        else:
+            return []
+        import pytest
+        import _pytest
+
+        def get_dir(p):
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
+        config = self.config
+        candidates = [pytestpath, pytestdir]
+        candidates += config.option.rsyncdir
+        rsyncroots = config.getini("rsyncdirs")
+        if rsyncroots:
+            candidates.extend(rsyncroots)
+        roots = []
+        for root in candidates:
+            root = Path(root).resolve()
+            if not root.exists():
+                raise pytest.UsageError(f"rsyncdir doesn't exist: {root!r}")
+            if root not in roots:
+                roots.append(root)
+        return roots
+
+    def _getrsyncoptions(self):
+        """Get options to be passed for rsync."""
+        ignores = list(self.DEFAULT_IGNORES)
+        ignores += [str(path) for path in self.config.option.rsyncignore]
+        ignores += [str(path) for path in self.config.getini("rsyncignore")]
+
+        return {
+            "ignores": ignores,
+            "verbose": getattr(self.config.option, "verbose", 0),
+        }
+
+    def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
+        """Perform rsync to remote hosts for node."""
+        # XXX This changes the calling behaviour of
+        #     pytest_xdist_rsyncstart and pytest_xdist_rsyncfinish to
+        #     be called once per rsync target.
+        rsync = HostRSync(source, verbose=verbose, ignores=ignores)
+        spec = gateway.spec
+        if spec.popen and not spec.chdir:
+            # XXX This assumes that sources are python-packages
+            #     and that adding the basedir does not hurt.
+            gateway.remote_exec(
+                """
+                import sys ; sys.path.insert(0, %r)
+            """
+                % os.path.dirname(str(source))
+            ).waitclose()
+            return
+        if (spec, source) in self._rsynced_specs:
+            return
+
+        def finished():
+            if notify:
+                notify("rsyncrootready", spec, source)
+
+        rsync.add_target_host(gateway, finished=finished)
+        self._rsynced_specs.add((spec, source))
+        self.config.hook.pytest_xdist_rsyncstart(source=source, gateways=[gateway])
+        rsync.send()
+        self.config.hook.pytest_xdist_rsyncfinish(source=source, gateways=[gateway])
+
+
+class HostRSync(execnet.RSync):
+    """RSyncer that filters out common files"""
+
+    PathLike = Union[str, "os.PathLike[str]"]
+
+    def __init__(
+        self,
+        sourcedir: PathLike,
+        *,
+        ignores: Optional[Sequence[PathLike]] = None,
+        **kwargs: object
+    ) -> None:
+        if ignores is None:
+            ignores = []
+        self._ignores = [re.compile(fnmatch.translate(os.fspath(x))) for x in ignores]
+        super().__init__(sourcedir=Path(sourcedir), **kwargs)
+
+    def filter(self, path: PathLike) -> bool:
+        path = Path(path)
+        for cre in self._ignores:
+            if cre.match(path.name) or cre.match(str(path)):
+                return False
+        else:
+            return True
+
+    def add_target_host(self, gateway, finished=None):
+        remotepath = os.path.basename(self._sourcedir)
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
+
+    def _report_send_file(self, gateway, modified_rel_path):
+        if self._verbose > 0:
+            path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
+            remotepath = gateway.spec.chdir
+            print(f"{gateway.spec}:{remotepath} <= {path}")
+
+
+def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]:
+    # XXX introduce/use public API for splitting pytest args
+    splitcode = "::"
+    result = []
+    for arg in args:
+        parts = arg.split(splitcode)
+        fspath = Path(parts[0])
+        try:
+            exists = fspath.exists()
+        except OSError:
+            exists = False
+        if not exists:
+            result.append(arg)
+            continue
+        for root in roots:
+            x: Optional[Path]
+            try:
+                x = fspath.relative_to(root)
+            except ValueError:
+                x = None
+            if x or fspath == root:
+                parts[0] = root.name + "/" + str(x)
+                break
+        else:
+            raise ValueError(f"arg {arg} not relative to an rsync root")
+        result.append(splitcode.join(parts))
+    return result
+
+
+class WorkerController:
+    ENDMARK = -1
+
+    class RemoteHook:
+        @pytest.hookimpl(trylast=True)
+        def pytest_xdist_getremotemodule(self):
+            return xdist.remote
+
+    def __init__(self, nodemanager, gateway, config, putevent):
+        config.pluginmanager.register(self.RemoteHook())
+        self.nodemanager = nodemanager
+        self.putevent = putevent
+        self.gateway = gateway
+        self.config = config
+        self.workerinput = {
+            "workerid": gateway.id,
+            "workercount": len(nodemanager.specs),
+            "testrunuid": nodemanager.testrunuid,
+            "mainargv": sys.argv,
+        }
+        self._down = False
+        self._shutdown_sent = False
+        self.log = Producer(f"workerctl-{gateway.id}", enabled=config.option.debug)
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__} {self.gateway.id}>"
+
+    @property
+    def shutting_down(self):
+        return self._down or self._shutdown_sent
+
+    def setup(self):
+        self.log("setting up worker session")
+        spec = self.gateway.spec
+        if hasattr(self.config, "invocation_params"):
+            args = [str(x) for x in self.config.invocation_params.args or ()]
+            option_dict = {}
+        else:
+            args = self.config.args
+            option_dict = vars(self.config.option)
+        if not spec.popen or spec.chdir:
+            args = make_reltoroot(self.nodemanager.roots, args)
+        if spec.popen:
+            name = "popen-%s" % self.gateway.id
+            if hasattr(self.config, "_tmp_path_factory"):
+                basetemp = self.config._tmp_path_factory.getbasetemp()
+                option_dict["basetemp"] = str(basetemp / name)
+        self.config.hook.pytest_configure_node(node=self)
+
+        remote_module = self.config.hook.pytest_xdist_getremotemodule()
+        self.channel = self.gateway.remote_exec(remote_module)
+        # change sys.path only for remote workers
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
+        self.channel.send((self.workerinput, args, option_dict, change_sys_path))
+
+        if self.putevent:
+            self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
+
+    def ensure_teardown(self):
+        if hasattr(self, "channel"):
+            if not self.channel.isclosed():
+                self.log("closing", self.channel)
+                self.channel.close()
+            # del self.channel
+        if hasattr(self, "gateway"):
+            self.log("exiting", self.gateway)
+            self.gateway.exit()
+            # del self.gateway
+
+    def send_runtest_some(self, indices):
+        self.sendcommand("runtests", indices=indices)
+
+    def send_runtest_all(self):
+        self.sendcommand("runtests_all")
+
+    def send_steal(self, indices):
+        self.sendcommand("steal", indices=indices)
+
+    def shutdown(self):
+        if not self._down:
+            try:
+                self.sendcommand("shutdown")
+            except OSError:
+                pass
+            self._shutdown_sent = True
+
+    def sendcommand(self, name, **kwargs):
+        """send a named parametrized command to the other side."""
+        self.log(f"sending command {name}(**{kwargs})")
+        self.channel.send((name, kwargs))
+
+    def notify_inproc(self, eventname, **kwargs):
+        self.log(f"queuing {eventname}(**{kwargs})")
+        self.putevent((eventname, kwargs))
+
+    def process_from_remote(self, eventcall):  # noqa too complex
+        """this gets called for each object we receive from
+        the other side and if the channel closes.
+
+        Note that channel callbacks run in the receiver
+        thread of execnet gateways - we need to
+        avoid raising exceptions or doing heavy work.
+        """
+        try:
+            if eventcall == self.ENDMARK:
+                err = self.channel._getremoteerror()
+                if not self._down:
+                    if not err or isinstance(err, EOFError):
+                        err = "Not properly terminated"  # lost connection?
+                    self.notify_inproc("errordown", node=self, error=err)
+                    self._down = True
+                return
+            eventname, kwargs = eventcall
+            if eventname in ("collectionstart",):
+                self.log(f"ignoring {eventname}({kwargs})")
+            elif eventname == "workerready":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "workerfinished":
+                self._down = True
+                self.workeroutput = kwargs["workeroutput"]
+                self.notify_inproc("workerfinished", node=self)
+            elif eventname in ("logstart", "logfinish"):
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname in ("testreport", "collectreport", "teardownreport"):
+                item_index = kwargs.pop("item_index", None)
+                rep = self.config.hook.pytest_report_from_serializable(
+                    config=self.config, data=kwargs["data"]
+                )
+                if item_index is not None:
+                    rep.item_index = item_index
+                self.notify_inproc(eventname, node=self, rep=rep)
+            elif eventname == "collectionfinish":
+                self.notify_inproc(eventname, node=self, ids=kwargs["ids"])
+            elif eventname == "runtest_protocol_complete":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "unscheduled":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "logwarning":
+                self.notify_inproc(
+                    eventname,
+                    message=kwargs["message"],
+                    code=kwargs["code"],
+                    nodeid=kwargs["nodeid"],
+                    fslocation=kwargs["nodeid"],
+                )
+            elif eventname == "warning_captured":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    item=kwargs["item"],
+                )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
+            else:
+                raise ValueError(f"unknown event: {eventname}")
+        except KeyboardInterrupt:
+            # should not land in receiver-thread
+            raise
+        except:  # noqa
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            print("!" * 20, excinfo)
+            self.config.notify_exception(excinfo)
+            self.shutdown()
+            self.notify_inproc("errordown", node=self, error=excinfo)
+
+
+def unserialize_warning_message(data):
+    import warnings
+    import importlib
+
+    if data["message_module"]:
+        mod = importlib.import_module(data["message_module"])
+        cls = getattr(mod, data["message_class_name"])
+        message = None
+        if data["message_args"] is not None:
+            try:
+                message = cls(*data["message_args"])
+            except TypeError:
+                pass
+        if message is None:
+            # could not recreate the original warning instance;
+            # create a generic Warning instance with the original
+            # message at least
+            message_text = "{mod}.{cls}: {msg}".format(
+                mod=data["message_module"],
+                cls=data["message_class_name"],
+                msg=data["message_str"],
+            )
+            message = Warning(message_text)
+    else:
+        message = data["message_str"]
+
+    if data["category_module"]:
+        mod = importlib.import_module(data["category_module"])
+        category = getattr(mod, data["category_class_name"])
+    else:
+        category = None
+
+    kwargs = {"message": message, "category": category}
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:  # type: ignore[attr-defined]
+        if attr_name in ("message", "category"):
+            continue
+        kwargs[attr_name] = data[attr_name]
+
+    return warnings.WarningMessage(**kwargs)  # type: ignore[arg-type]

testing/conftest.py

@@ -1,39 +1,61 @@
-import py
 import execnet
+import pytest
+import shutil
+from typing import List
 
 pytest_plugins = "pytester"
 
-#rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
 
-def pytest_addoption(parser):
-    parser.addoption('--gx',
-       action="append", dest="gspecs", default=None,
-       help=("add a global test environment, XSpec-syntax. "))
+@pytest.fixture(autouse=True)
+def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
+    import atexit
 
-def pytest_funcarg__specssh(request):
+    finalizers = []
+
+    def fake_register(func, *args, **kwargs):
+        finalizers.append((func, args, kwargs))
+
+    monkeypatch.setattr(atexit, "register", fake_register)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
+
+
+def pytest_addoption(parser) -> None:
+    parser.addoption(
+        "--gx",
+        action="append",
+        dest="gspecs",
+        help="add a global test environment, XSpec-syntax. ",
+    )
+
+
+@pytest.fixture
+def specssh(request) -> str:
     return getspecssh(request.config)
-def getgspecs(config):
-    return [execnet.XSpec(spec)
-                for spec in config.getvalueorskip("gspecs")]
+
 
 # configuration information for tests
-def getgspecs(config):
-    return [execnet.XSpec(spec)
-                for spec in config.getvalueorskip("gspecs")]
+def getgspecs(config) -> List[execnet.XSpec]:
+    return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
 
-def getspecssh(config):
+
+def getspecssh(config) -> str:  # type: ignore[return]
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.ssh:
-            if not py.path.local.sysfind("ssh"):
-                py.test.skip("command not found: ssh")
+            if not shutil.which("ssh"):
+                pytest.skip("command not found: ssh")
             return str(spec)
-    py.test.skip("need '--gx ssh=...'")
+    pytest.skip("need '--gx ssh=...'")
 
-def getsocketspec(config):
+
+def getsocketspec(config) -> execnet.XSpec:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.socket:
             return spec
-    py.test.skip("need '--gx socket=...'")
-
+    pytest.skip("need '--gx socket=...'")

testing/test_boxed.py

@@ -1,28 +0,0 @@
-import py
-
-@py.test.mark.skipif("not hasattr(os, 'fork')")
-def test_functional_boxed(testdir):
-    p1 = testdir.makepyfile("""
-        import os
-        def test_function():
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed")
-    result.stdout.fnmatch_lines([
-        "*CRASHED*",
-        "*1 failed*"
-    ])
-
-class TestOptionEffects:
-    def test_boxed_option_default(self, testdir):
-        tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
-        config = testdir.parseconfig()
-        assert not config.option.boxed
-        py.test.importorskip("execnet")
-        config = testdir.parseconfig('-d', tmpdir)
-        assert not config.option.boxed
-
-    def test_is_not_boxed_by_default(self, testdir):
-        config = testdir.parseconfig(testdir.tmpdir)
-        assert not config.option.boxed
-

testing/test_dsession.py

@@ -1,221 +1,529 @@
-from xdist.dsession import (
-    DSession,
-    LoadScheduling,
-    EachScheduling,
-    report_collection_diff,
-)
-from _pytest import main as outcome
-import py
+from xdist.dsession import DSession, get_default_max_worker_restart
+from xdist.report import report_collection_diff
+from xdist.scheduler import EachScheduling, LoadScheduling, WorkStealingScheduling
+from typing import Optional
+
 import pytest
 import execnet
 
-XSpec = execnet.XSpec
-
-def run(item, node, excinfo=None):
-    runner = item.config.pluginmanager.getplugin("runner")
-    rep = runner.ItemTestReport(item=item,
-        excinfo=excinfo, when="call")
-    rep.node = node
-    return rep
 
 class MockGateway:
-    _count = 0
-    def __init__(self):
+    def __init__(self) -> None:
+        self._count = 0
         self.id = str(self._count)
         self._count += 1
 
-class MockNode:
-    def __init__(self):
-        self.sent = []
-        self.gateway = MockGateway()
 
-    def send_runtest_some(self, indices):
+class MockNode:
+    def __init__(self) -> None:
+        self.sent = []  # type: ignore[var-annotated]
+        self.stolen = []  # type: ignore[var-annotated]
+        self.gateway = MockGateway()
+        self._shutdown = False
+
+    def send_runtest_some(self, indices) -> None:
         self.sent.extend(indices)
 
-    def send_runtest_all(self):
+    def send_runtest_all(self) -> None:
         self.sent.append("ALL")
 
-    def shutdown(self):
-        self._shutdown=True
+    def send_steal(self, indices) -> None:
+        self.stolen.extend(indices)
+
+    def shutdown(self) -> None:
+        self._shutdown = True
+
+    @property
+    def shutting_down(self) -> bool:
+        return self._shutdown
 
-def dumpqueue(queue):
-    while queue.qsize():
-        print(queue.get())
 
 class TestEachScheduling:
-    def test_schedule_load_simple(self):
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
         node2 = MockNode()
-        sched = EachScheduling(2)
-        sched.addnode(node1)
-        sched.addnode(node2)
-        collection = ["a.py::test_1", ]
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = EachScheduling(config)
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = ["a.py::test_1"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
-        assert sched.tests_finished()
-        assert node1.sent == ['ALL']
-        assert node2.sent == ['ALL']
-        sched.remove_item(node1, 0)
-        assert sched.tests_finished()
-        sched.remove_item(node2, 0)
-        assert sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
+        assert node1.sent == ["ALL"]
+        assert node2.sent == ["ALL"]
+        sched.mark_test_complete(node1, 0)
+        assert sched.tests_finished
+        sched.mark_test_complete(node2, 0)
+        assert sched.tests_finished
 
-    def test_schedule_remove_node(self):
+    def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
         node1 = MockNode()
-        sched = EachScheduling(1)
-        sched.addnode(node1)
-        collection = ["a.py::test_1", ]
+        config = pytester.parseconfig("--tx=popen")
+        sched = EachScheduling(config)
+        sched.add_node(node1)
+        collection = ["a.py::test_1"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
-        sched.init_distribute()
-        assert sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
         crashitem = sched.remove_node(node1)
         assert crashitem
-        assert sched.tests_finished()
-        assert not sched.hasnodes()
+        assert sched.tests_finished
+        assert not sched.nodes
+
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self):
-        node1 = MockNode()
-        node2 = MockNode()
-        sched = LoadScheduling(2)
-        sched.addnode(node1)
-        sched.addnode(node2)
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
         collection = ["a.py::test_1", "a.py::test_2"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
-        assert sched.tests_finished()
+        sched.schedule()
+        assert not sched.pending
+        assert sched.tests_finished
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
-        x = sorted(node1.sent + node2.sent)
-        assert x == [0, 1]
-        sched.remove_item(node1, node1.sent[0])
-        sched.remove_item(node2, node2.sent[0])
-        assert sched.tests_finished()
-        assert not sched.pending
+        assert node1.sent == [0]
+        assert node2.sent == [1]
+        sched.mark_test_complete(node1, node1.sent[0])
+        assert sched.tests_finished
 
-    def test_init_distribute_chunksize(self):
-        sched = LoadScheduling(2)
-        node1 = MockNode()
-        node2 = MockNode()
-        sched.addnode(node1)
-        sched.addnode(node2)
-        col = ["xyz"] * (3)
-        sched.addnode_collection(node1, col)
-        sched.addnode_collection(node2, col)
-        sched.init_distribute()
-        #assert not sched.tests_finished()
+    def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
+        col = ["xyz"] * 6
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
-        chunkitems = col[:1]
-        assert (sent1 == [0] and sent2 == [1]) or (
-                sent1 == [1] and sent2 == [0])
+        assert sent1 == [0, 1]
+        assert sent2 == [2, 3]
+        assert sched.pending == [4, 5]
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
-        assert len(sched.pending) == 1
-        for node in (node1, node2):
-            for i in sched.node2pending[node]:
-                sched.remove_item(node, i)
+        assert len(sched.pending) == 2
+        sched.mark_test_complete(node1, 0)
+        assert node1.sent == [0, 1, 4]
+        assert sched.pending == [5]
+        assert node2.sent == [2, 3]
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
         assert not sched.pending
 
-    def test_add_remove_node(self):
-        node = MockNode()
-        sched = LoadScheduling(1)
-        sched.addnode(node)
-        collection = ["test_file.py::test_func"]
-        sched.addnode_collection(node, collection)
+    def test_schedule_maxchunk_none(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
+        col = [f"test{i}" for i in range(16)]
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+        sched.mark_test_complete(node1, 0)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+
+        for i in range(7, 16):
+            sched.mark_test_complete(node1, i - 3)
+            assert node1.sent == [0, 1] + list(range(4, i))
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(i, 16))
+
+    def test_schedule_maxchunk_1(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen", "--maxschedchunk=1")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
+        col = [f"test{i}" for i in range(16)]
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+
+        for complete_index, first_pending in enumerate(range(5, 16)):
+            sched.mark_test_complete(node1, node1.sent[complete_index])
+            assert node1.sent == [0, 1] + list(range(4, first_pending))
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(first_pending, 16))
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2, node3 = sched.nodes
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
         assert sched.collection_is_completed
-        sched.init_distribute()
+        sched.schedule()
+        # assert not sched.tests_finished
+        assert node1.sent == [0]
+        assert node2.sent == [1]
+        assert node3.sent == []
+        assert not sched.pending
+
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2, node3 = sched.nodes
+        col = ["xyz"] * 5
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        assert sched.collection_is_completed
+        sched.schedule()
+        # assert not sched.tests_finished
+        assert node1.sent == [0, 3]
+        assert node2.sent == [1, 4]
+        assert node3.sent == [2]
+        assert not sched.pending
+
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = LoadScheduling(config)
+        sched.add_node(node)
+        collection = ["test_file.py::test_func"]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
         assert not sched.pending
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
 
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
+        """
+        Test that LoadScheduling is reporting collection errors when
+        different test ids are collected by workers.
+        """
+
+        class CollectHook:
+            """
+            Dummy hook that stores collection reports.
+            """
+
+            def __init__(self):
+                self.reports = []
+
+            def pytest_collectreport(self, report):
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = pytester.parseconfig("--tx=2*popen")
+        config.pluginmanager.register(collect_hook, "collect_hook")
+        node1 = MockNode()
+        node2 = MockNode()
+        sched = LoadScheduling(config)
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert "Different tests were collected between" in rep.longrepr
+
+
+class TestWorkStealingScheduling:
+    def test_ideal_case(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node1, collection)
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        assert sched.node2collection[node1] == collection
+        assert sched.node2collection[node2] == collection
+        sched.schedule()
+        assert not sched.pending
+        assert not sched.tests_finished
+        assert node1.sent == list(range(0, 8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sched.mark_test_complete(node1, node1.sent[i])
+            sched.mark_test_complete(node2, node2.sent[i])
+        assert sched.tests_finished
+        assert node1.stolen == []
+        assert node2.stolen == []
+
+    def test_stealing(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2 = sched.nodes
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        sched.add_node_collection(node1, collection)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node1.sent == list(range(0, 8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sched.mark_test_complete(node1, node1.sent[i])
+        assert node2.stolen == list(range(12, 16))
+        sched.remove_pending_tests_from_node(node2, node2.stolen)
+        for i in range(4):
+            sched.mark_test_complete(node2, node2.sent[i])
+        assert node1.stolen == [14, 15]
+        sched.remove_pending_tests_from_node(node1, node1.stolen)
+        sched.mark_test_complete(node1, 12)
+        sched.mark_test_complete(node2, 14)
+        assert node2.stolen == list(range(12, 16))
+        assert node1.stolen == [14, 15]
+        assert sched.tests_finished
+
+    def test_steal_on_add_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(5)]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        sched.mark_test_complete(node, 0)
+        node2 = MockNode()
+        sched.add_node(node2)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node.stolen == [3, 4]
+        sched.remove_pending_tests_from_node(node, node.stolen)
+        sched.mark_test_complete(node, 1)
+        sched.mark_test_complete(node2, 3)
+        assert sched.tests_finished
+        assert node2.stolen == []
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2, node3 = sched.nodes
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == []
+        assert node1.stolen == []
+        assert node2.sent == [0]
+        assert node2.stolen == []
+        assert node3.sent == [1]
+        assert node3.stolen == []
+        assert not sched.pending
+        assert sched.tests_finished
+
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        node1, node2, node3 = sched.nodes
+        col = ["xyz"] * 5
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == [0]
+        assert node2.sent == [1, 2]
+        assert node3.sent == [3, 4]
+        assert not sched.pending
+        assert not sched.tests_finished
+        sched.mark_test_complete(node1, node1.sent[0])
+        sched.mark_test_complete(node2, node2.sent[0])
+        sched.mark_test_complete(node3, node3.sent[0])
+        sched.mark_test_complete(node3, node3.sent[1])
+        assert sched.tests_finished
+        assert node1.stolen == []
+        assert node2.stolen == []
+        assert node3.stolen == []
+
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = ["test_file.py::test_func"]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        crashitem = sched.remove_node(node)
+        assert crashitem == collection[0]
+
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
+        class CollectHook:
+            def __init__(self):
+                self.reports = []
+
+            def pytest_collectreport(self, report):
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = pytester.parseconfig("--tx=2*popen")
+        config.pluginmanager.register(collect_hook, "collect_hook")
+        node1 = MockNode()
+        node2 = MockNode()
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert "Different tests were collected between" in rep.longrepr
+
 
 class TestDistReporter:
-
-    @py.test.mark.xfail
-    def test_rsync_printing(self, testdir, linecomp):
-        config = testdir.parseconfig()
+    @pytest.mark.xfail
+    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
+        config = pytester.parseconfig()
         from _pytest.pytest_terminal import TerminalReporter
+
         rep = TerminalReporter(config, file=linecomp.stringio)
         config.pluginmanager.register(rep, "terminalreporter")
         dsession = DSession(config)
+
         class gw1:
             id = "X1"
             spec = execnet.XSpec("popen")
+
         class gw2:
             id = "X2"
             spec = execnet.XSpec("popen")
-        #class rinfo:
+
+        # class rinfo:
         #    version_info = (2, 5, 1, 'final', 0)
         #    executable = "hello"
         #    platform = "xyz"
         #    cwd = "qwe"
 
-        #dsession.pytest_xdist_newgateway(gw1, rinfo)
-        #linecomp.assert_contains_lines([
-        #    "*X1*popen*xyz*2.5*"
-        #])
-        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
-        linecomp.assert_contains_lines([
-            "[X1,X2] rsyncing: hello",
-        ])
+        # dsession.pytest_xdist_newgateway(gw1, rinfo)
+        # linecomp.assert_contains_lines([
+        #     "*X1*popen*xyz*2.5*"
+        # ])
+        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])  # type: ignore[attr-defined]
+        linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
 
 
-def test_report_collection_diff_equal():
+def test_report_collection_diff_equal() -> None:
     """Test reporting of equal collections."""
-    from_collection = to_collection = ['aaa', 'bbb', 'ccc']
-    assert report_collection_diff(from_collection, to_collection, 1, 2)
+    from_collection = to_collection = ["aaa", "bbb", "ccc"]
+    assert report_collection_diff(from_collection, to_collection, 1, 2) is None
 
 
-def test_report_collection_diff_different():
+def test_default_max_worker_restart() -> None:
+    class config:
+        class option:
+            maxworkerrestart: Optional[str] = None
+            numprocesses: int = 0
+
+    assert get_default_max_worker_restart(config) is None
+
+    config.option.numprocesses = 2
+    assert get_default_max_worker_restart(config) == 8
+
+    config.option.maxworkerrestart = "1"
+    assert get_default_max_worker_restart(config) == 1
+
+    config.option.maxworkerrestart = "0"
+    assert get_default_max_worker_restart(config) == 0
+
+
+def test_report_collection_diff_different() -> None:
     """Test reporting of different collections."""
-    from_collection = ['aaa', 'bbb', 'ccc', 'YYY']
-    to_collection = ['aZa', 'bbb', 'XXX', 'ccc']
+    from_collection = ["aaa", "bbb", "ccc", "YYY"]
+    to_collection = ["aZa", "bbb", "XXX", "ccc"]
     error_message = (
-        'Different tests were collected between 1 and 2. The difference is:\n'
-        '--- 1\n'
-        '\n'
-        '+++ 2\n'
-        '\n'
-        '@@ -1,4 +1,4 @@\n'
-        '\n'
-        '-aaa\n'
-        '+aZa\n'
-        ' bbb\n'
-        '+XXX\n'
-        ' ccc\n'
-        '-YYY'
+        "Different tests were collected between 1 and 2. The difference is:\n"
+        "--- 1\n"
+        "\n"
+        "+++ 2\n"
+        "\n"
+        "@@ -1,4 +1,4 @@\n"
+        "\n"
+        "-aaa\n"
+        "+aZa\n"
+        " bbb\n"
+        "+XXX\n"
+        " ccc\n"
+        "-YYY\n"
+        "To see why this happens see Known limitations in documentation"
     )
 
-    try:
-        report_collection_diff(from_collection, to_collection, 1, 2)
-    except AssertionError as e:
-        assert py.builtin._totext(e) == error_message
+    msg = report_collection_diff(from_collection, to_collection, "1", "2")
+    assert msg == error_message
+
 
 @pytest.mark.xfail(reason="duplicate test ids not supported yet")
-def test_pytest_issue419(testdir):
-    testdir.makepyfile("""
+def test_pytest_issue419(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
+        """
         import pytest
 
         @pytest.mark.parametrize('birth_year', [1988, 1988, ])
         def test_2011_table(birth_year):
             pass
-    """)
-    reprec = testdir.inline_run("-n1")
+    """
+    )
+    reprec = pytester.inline_run("-n1")
     reprec.assertoutcome(passed=2)
     assert 0

testing/test_looponfail.py

@@ -1,280 +1,374 @@
-import py
+import unittest.mock
+from typing import List
+
+import pytest
+import shutil
+import textwrap
+from pathlib import Path
+
 from xdist.looponfail import RemoteControl
 from xdist.looponfail import StatRecorder
 
+
+PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
+
+
 class TestStatRecorder:
-    def test_filechange(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
+    def test_filechange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        hello.write("world")
+        hello.write_text("world")
         changed = sd.check()
         assert changed
 
-        (hello + "c").write("hello")
+        hello.with_suffix(".pyc").write_text("hello")
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp / "new.py"
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "b", "c.py").ensure()
+        tmp.joinpath("a", "b").mkdir(parents=True)
+        tmp.joinpath("a", "b", "c.py").touch()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "c.txt").ensure()
+        tmp.joinpath("a", "c.txt").touch()
         changed = sd.check()
         assert changed
         changed = sd.check()
         assert not changed
 
-        tmp.join("a").remove()
+        shutil.rmtree(str(tmp.joinpath("a")))
         changed = sd.check()
         assert changed
 
-    def test_dirchange(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("dir", "hello.py")
+    def test_dirchange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        tmp.joinpath("dir").mkdir()
+        tmp.joinpath("dir", "hello.py").touch()
         sd = StatRecorder([tmp])
-        assert not sd.fil(tmp.join("dir"))
+        assert not sd.fil(tmp / "dir")
 
-    def test_filechange_deletion_race(self, tmpdir, monkeypatch):
-        tmp = tmpdir
+    def test_filechange_deletion_race(self, tmp_path: Path) -> None:
+        tmp = tmp_path
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp.joinpath("new.py")
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(tmp, 'visit', lambda *args: [p])
-
-        changed = sd.check()
+        dirname = str(tmp)
+        dirnames: List[str] = []
+        filenames = [str(p)]
+        with unittest.mock.patch(
+            "os.walk", return_value=[(dirname, dirnames, filenames)], autospec=True
+        ):
+            changed = sd.check()
         assert changed
 
-    def test_pycremoval(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
+    def test_pycremoval(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        pycfile = hello + "c"
-        pycfile.ensure()
-        hello.write("world")
+        pycfile = hello.with_suffix(".pyc")
+        pycfile.touch()
+        hello.write_text("world")
         changed = sd.check()
         assert changed
-        assert not pycfile.check()
+        assert not pycfile.exists()
 
-    def test_waitonchange(self, tmpdir, monkeypatch):
-        tmp = tmpdir
+    def test_waitonchange(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        tmp = tmp_path
         sd = StatRecorder([tmp])
 
-        l = [True, False]
-        monkeypatch.setattr(StatRecorder, 'check', lambda self: l.pop())
+        ret_values = [True, False]
+        monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
         sd.waitonchange(checkinterval=0.2)
-        assert not l
+        assert not ret_values
+
 
 class TestRemoteControl:
-    def test_nofailures(self, testdir):
-        item = testdir.getitem("def test_func(): pass\n")
+    def test_nofailures(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func(): pass\n")
         control = RemoteControl(item.config)
         control.setup()
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failures_somewhere(self, testdir):
-        item = testdir.getitem("def test_func():\n assert 0\n")
+    def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
         failures = control.runsession()
         assert failures
         control.setup()
-        item.fspath.write("def test_func():\n assert 1\n")
-        removepyc(item.fspath)
+        item_path = item.path if PYTEST_GTE_7 else Path(str(item.fspath))  # type: ignore[attr-defined]
+        item_path.write_text("def test_func():\n assert 1\n")
+        removepyc(item_path)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failure_change(self, testdir):
-        modcol = testdir.getitem("""
-            def test_func():
-                assert 0
-        """)
+    def test_failure_change(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
+        )
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol.fspath.write(py.code.Source("""
-            def test_func():
-                assert 1
-            def test_new():
-                assert 0
-        """))
-        removepyc(modcol.fspath)
+        if PYTEST_GTE_7:
+            modcol_path = modcol.path  # type:ignore[attr-defined]
+        else:
+            modcol_path = Path(str(modcol.fspath))
+
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 1
+                def test_new():
+                    assert 0
+                """
+            )
+        )
+        removepyc(modcol_path)
         control.loop_once()
         assert not control.failures
         control.loop_once()
         assert control.failures
         assert str(control.failures).find("test_new") != -1
 
-    def test_failure_subdir_no_init(self, testdir):
-        modcol = testdir.getitem("""
-            def test_func():
-                assert 0
-        """)
-        parent = modcol.fspath.dirpath().dirpath()
-        parent.chdir()
-        modcol.config.args = [py.path.local(x).relto(parent)
-                                for x in modcol.config.args]
+    def test_failure_subdir_no_init(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
+        )
+        if PYTEST_GTE_7:
+            parent = modcol.path.parent.parent  # type: ignore[attr-defined]
+        else:
+            parent = Path(modcol.fspath.dirpath().dirpath())
+        monkeypatch.chdir(parent)
+        modcol.config.args = [
+            str(Path(x).relative_to(parent)) for x in modcol.config.args
+        ]
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
         control.loop_once()
         assert control.failures
 
+
 class TestLooponFailing:
-    def test_looponfail_from_fail_to_ok(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                x = 0
-                assert x == 1
-            def test_two():
-                assert 1
-        """)
+    def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                def test_two():
+                    assert 1
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-        modcol.fspath.write(py.code.Source("""
-            def test_one():
-                assert 1
-            def test_two():
-                assert 1
-        """))
-        removepyc(modcol.fspath)
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1
+                def test_two():
+                    assert 1
+                """
+            )
+        )
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert not remotecontrol.failures
 
-    def test_looponfail_from_one_to_two_tests(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                assert 0
-        """)
+    def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
-        assert 'test_one' in remotecontrol.failures[0]
+        assert "test_one" in remotecontrol.failures[0]
 
-        modcol.fspath.write(py.code.Source("""
-            def test_one():
-                assert 1 # passes now
-            def test_two():
-                assert 0 # new and fails
-        """))
-        removepyc(modcol.fspath)
+        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1 # passes now
+                def test_two():
+                    assert 0 # new and fails
+                """
+            )
+        )
+        removepyc(modcol_path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
-        assert 'test_one' not in remotecontrol.failures[0]
-        assert 'test_two' in remotecontrol.failures[0]
+        assert "test_one" not in remotecontrol.failures[0]
+        assert "test_two" in remotecontrol.failures[0]
 
-    def test_looponfail_removed_test(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                assert 0
-            def test_two():
-                assert 0
-        """)
+    @pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
+    def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                def test_two():
+                    assert 0
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 2
 
-        modcol.fspath.write(py.code.Source("""
-            def test_xxx(): # renamed test
-                assert 0
-            def test_two():
-                assert 1 # pass now
-        """))
-        removepyc(modcol.fspath)
+        modcol.path.write_text(
+            textwrap.dedent(
+                """
+                def test_xxx(): # renamed test
+                    assert 0
+                def test_two():
+                    assert 1 # pass now
+                """
+            )
+        )
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
 
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-    def test_looponfail_multiple_errors(self, testdir, monkeypatch):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                assert 0
-        """)
+    def test_looponfail_multiple_errors(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         orig_runsession = remotecontrol.runsession
 
         def runsession_dups():
             # twisted.trial test cases may report multiple errors.
             failures, reports, collection_failed = orig_runsession()
-            print (failures)
+            print(failures)
             return failures * 2, reports, collection_failed
 
-        monkeypatch.setattr(remotecontrol, 'runsession', runsession_dups)
+        monkeypatch.setattr(remotecontrol, "runsession", runsession_dups)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
 
 class TestFunctional:
-    def test_fail_to_ok(self, testdir):
-        p = testdir.makepyfile("""
-            def test_one():
-                x = 0
-                assert x == 1
-        """)
-        #p = testdir.mkdir("sub").join(p1.basename)
-        #p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+    def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                """
+            )
+        )
+        # p = pytester.mkdir("sub").join(p1.basename)
+        # p1.move(p)
+        child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
         child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
-        p.write(py.code.Source("""
-            def test_one():
-                x = 1
-                assert x == 1
-        """))
+        p.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 1
+                    assert x == 1
+                """
+            ),
+        )
         child.expect(".*1 passed.*")
         child.kill(15)
 
-    def test_xfail_passes(self, testdir):
-        p = testdir.makepyfile("""
-            import py
-            @py.test.mark.xfail
-            def test_one():
-                pass
-        """)
-        child = testdir.spawn_pytest("-f %s" % p)
+    def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                import pytest
+                @pytest.mark.xfail
+                def test_one():
+                    pass
+                """
+            )
+        )
+        child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
-        child.expect("### LOOPONFAILING ####")
+        # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
         child.kill(15)
 
-def removepyc(path):
-    # XXX damn those pyc files
-    pyc = path + "c"
-    if pyc.check():
-        pyc.remove()
-    c = path.dirpath("__pycache__")
-    if c.check():
-        c.remove()
 
+def removepyc(path: Path) -> None:
+    # XXX damn those pyc files
+    pyc = path.with_suffix(".pyc")
+    if pyc.exists():
+        pyc.unlink()
+    c = path.parent / "__pycache__"
+    if c.exists():
+        shutil.rmtree(c)

testing/test_newhooks.py

@@ -0,0 +1,121 @@
+import pytest
+
+
+class TestHooks:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): pass
+            def test_c(): pass
+        """
+        )
+
+    def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
+        """Test that log reports from pytest_runtest_logreport when running
+        with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
+        """
+        pytester.makeconftest(
+            """
+            def pytest_runtest_logreport(report):
+                if hasattr(report, 'node'):
+                    if report.when == "call":
+                        workerid = report.node.workerinput['workerid']
+                        testrunuid = report.node.workerinput['testrunuid']
+                        if workerid != report.worker_id:
+                            print("HOOK: Worker id mismatch: %s %s"
+                                   % (workerid, report.worker_id))
+                        elif testrunuid != report.testrun_uid:
+                            print("HOOK: Testrun uid mismatch: %s %s"
+                                   % (testrunuid, report.testrun_uid))
+                        else:
+                            print("HOOK: %s %s %s"
+                                   % (report.nodeid, report.worker_id, report.testrun_uid))
+        """
+        )
+        res = pytester.runpytest("-n1", "-s")
+        res.stdout.fnmatch_lines(
+            [
+                "*HOOK: test_runtest_logreport.py::test_a gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_b gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_c gw0 *",
+                "*3 passed*",
+            ]
+        )
+
+    def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
+        pytester.makeconftest(
+            """
+            def pytest_xdist_node_collection_finished(node, ids):
+                workerid = node.workerinput['workerid']
+                stripped_ids = [x.split('::')[1] for x in ids]
+                print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(
+            ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
+        )
+        res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])
+
+    def test_handlecrashitem_one(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook with just one test."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n1", "-s", "-k", "test_b")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(
+            [
+                "FAILED test_handlecrashitem_one.py::test_b",
+                "FAILED test_handlecrashitem_one.py::test_b",
+            ]
+        )

testing/test_plugin.py

@@ -1,30 +1,287 @@
-import py
+from contextlib import suppress
+from pathlib import Path
+import sys
+import os
+
 import execnet
-from xdist.slavemanage import NodeManager
+from xdist.workermanage import NodeManager
 
-def test_dist_incompatibility_messages(testdir):
-    result = testdir.runpytest("--pdb", "--looponfail")
+import pytest
+
+
+@pytest.fixture
+def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch):
+    """Make pytest-xdist believe the system has 3 CPUs"""
+    # block import
+    monkeypatch.setitem(sys.modules, "psutil", None)  # type: ignore
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setattr(os, "cpu_count", lambda: 3)
+
+
+def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
+    result = pytester.runpytest("--pdb", "--looponfail")
     assert result.ret != 0
-    result = testdir.runpytest("--pdb", "-n", "3")
+    result = pytester.runpytest("--pdb", "-n", "3")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
-    result = testdir.runpytest("--pdb", "-d", "--tx", "popen")
+    result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
 
-def test_dist_options(testdir):
-    from xdist.plugin import check_options
-    config = testdir.parseconfigure("-n 2")
+
+def test_dist_options(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    config = pytester.parseconfigure("-n 2")
     check_options(config)
     assert config.option.dist == "load"
-    assert config.option.tx == ['popen'] * 2
-    config = testdir.parseconfigure("-d")
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("--numprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("-d")
+    check_options(config)
+    assert config.option.dist == "load"
+
+
+def test_auto_detect_cpus(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+
+    if hasattr(os, "sched_getaffinity"):
+        monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
+    elif hasattr(os, "cpu_count"):
+        monkeypatch.setattr(os, "cpu_count", lambda: 99)
+    else:
+        import multiprocessing
+
+        monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
+
+    config = pytester.parseconfigure("-n2")
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 99
+
+    config = pytester.parseconfigure("-nauto", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
+    config = pytester.parseconfigure("-nlogical", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setenv("TRAVIS", "true")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+
+def test_auto_detect_cpus_psutil(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_auto_detect_cpus_os(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_hook_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
+def test_hook_auto_num_workers_arg(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # config.option.numprocesses is a pytest feature,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers(config):
+            if config.option.numprocesses == 'auto':
+                return 42
+            if config.option.numprocesses == 'logical':
+                return 8
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 8
+
+
+def test_hook_auto_num_workers_none(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    # Returning None from a hook to skip it is pytest behavior,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return None
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "5")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 5
+
+
+def test_envvar_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "7")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+
+def test_envvar_auto_num_workers_warn(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "fourscore")
+
+    config = pytester.parseconfigure("-nauto")
+    with pytest.warns(UserWarning):
+        check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_auto_num_workers_hook_overrides_envvar(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "987")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 2
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+
+def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+    from xdist.plugin import pytest_configure as configure
+
+    config = pytester.parseconfigure("-n1")
+    check_options(config)
+    configure(config)
+    assert config.pluginmanager.hasplugin("dsession")
+
+    config = pytester.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    configure(config)
+    assert not config.pluginmanager.hasplugin("dsession")
+
+
+def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    nm = NodeManager(config)
+    assert nm.testrunuid == "test123"
+
+
+def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--tx=popen")
+    nm = NodeManager(config)
+    assert len(nm.testrunuid) == 32
+
 
 class TestDistOptions:
-    def test_getxspecs(self, testdir):
-        config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
+    def test_getxspecs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
         nodemanager = NodeManager(config)
         xspecs = nodemanager._getxspecs()
         assert len(xspecs) == 2
@@ -32,39 +289,39 @@ class TestDistOptions:
         assert xspecs[0].popen
         assert xspecs[1].ssh == "xyz"
 
-    def test_xspecs_multiplied(self, testdir):
-        config = testdir.parseconfigure("--tx=3*popen",)
+    def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=3*popen")
         xspecs = NodeManager(config)._getxspecs()
         assert len(xspecs) == 3
         assert xspecs[1].popen
 
-    def test_getrsyncdirs(self, testdir):
-        config = testdir.parseconfigure('--rsyncdir=' + str(testdir.tmpdir))
+    def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
         nm = NodeManager(config, specs=[execnet.XSpec("popen")])
         assert not nm._getrsyncdirs()
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert nm.roots
-        assert testdir.tmpdir in nm.roots
+        assert pytester.path in nm.roots
 
-    def test_getrsyncignore(self, testdir):
-        config = testdir.parseconfigure('--rsyncignore=fo*')
+    def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncignore=fo*")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
-        assert 'fo*' in nm.rsyncoptions['ignores']
+        assert "fo*" in nm.rsyncoptions["ignores"]
 
-    def test_getrsyncdirs_with_conftest(self, testdir):
-        p = py.path.local()
-        for bn in 'x y z'.split():
-            p.mkdir(bn)
-        testdir.makeini("""
+    def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
+        p = Path.cwd()
+        for bn in ("x", "y", "z"):
+            p.joinpath(bn).mkdir()
+        pytester.makeini(
+            """
             [pytest]
             rsyncdirs= x
-        """)
-        config = testdir.parseconfigure(
-              testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z')
+        """
+        )
+        config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
         roots = nm._getrsyncdirs()
-        #assert len(roots) == 3 + 1 # pylib
-        assert py.path.local('y') in roots
-        assert py.path.local('z') in roots
-        assert testdir.tmpdir.join('x') in roots
-
+        # assert len(roots) == 3 + 1 # pylib
+        assert Path("y").resolve() in roots
+        assert Path("z").resolve() in roots
+        assert pytester.path.joinpath("x") in roots

testing/test_remote.py

@@ -1,42 +1,52 @@
-import py
-from xdist.slavemanage import SlaveController, unserialize_report
-from xdist.remote import serialize_report
+import pprint
+import pytest
+import sys
+import uuid
+
+from xdist.workermanage import WorkerController
 import execnet
-queue = py.builtin._tryimport("queue", "Queue")
-from py.builtin import print_
 import marshal
 
+from queue import Queue
+
 WAIT_TIMEOUT = 10.0
 
+
 def check_marshallable(d):
     try:
         marshal.dumps(d)
     except ValueError:
-        py.std.pprint.pprint(d)
+        pprint.pprint(d)
         raise ValueError("not marshallable")
 
+
 class EventCall:
     def __init__(self, eventcall):
         self.name, self.kwargs = eventcall
 
     def __str__(self):
-        return "<EventCall %s(**%s)>" %(self.name, self.kwargs)
+        return f"<EventCall {self.name}(**{self.kwargs})>"
 
-class SlaveSetup:
-    use_callback = False
 
-    def __init__(self, request):
-        self.testdir = testdir = request.getfuncargvalue("testdir")
+class WorkerSetup:
+    def __init__(self, request, pytester: pytest.Pytester) -> None:
         self.request = request
-        self.events = queue.Queue()
+        self.pytester = pytester
+        self.use_callback = False
+        self.events = Queue()  # type: ignore[var-annotated]
 
-    def setup(self, ):
-        self.testdir.chdir()
-        #import os ; os.environ['EXECNET_DEBUG'] = "2"
+    def setup(self) -> None:
+        self.pytester.chdir()
+        # import os ; os.environ['EXECNET_DEBUG'] = "2"
         self.gateway = execnet.makegateway()
-        self.config = config = self.testdir.parseconfigure()
-        putevent = self.use_callback and self.events.put or None
-        self.slp = SlaveController(None, self.gateway, config, putevent)
+        self.config = config = self.pytester.parseconfigure()
+        putevent = self.events.put if self.use_callback else None
+
+        class DummyMananger:
+            testrunuid = uuid.uuid4().hex
+            specs = [0, 1]
+
+        self.slp = WorkerController(DummyMananger, self.gateway, config, putevent)
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
@@ -49,201 +59,337 @@ class SlaveSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping %s" % (ev,))
+            print(f"skipping {ev}")
 
     def sendcommand(self, name, **kwargs):
         self.slp.sendcommand(name, **kwargs)
 
-def pytest_funcarg__slave(request):
-    return SlaveSetup(request)
 
-def test_remoteinitconfig(testdir):
+@pytest.fixture
+def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
+    return WorkerSetup(request, pytester)
+
+
+@pytest.mark.xfail(reason="#59")
+def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
     from xdist.remote import remote_initconfig
-    config1 = testdir.parseconfig()
+
+    config1 = pytester.parseconfig()
     config2 = remote_initconfig(config1.option.__dict__, config1.args)
     assert config2.option.__dict__ == config1.option.__dict__
     assert config2.pluginmanager.getplugin("terminal") in (-1, None)
 
-class TestReportSerialization:
-    def test_itemreport_outcomes(self, testdir):
-        reprec = testdir.inline_runsource("""
-            import py
-            def test_pass(): pass
-            def test_fail(): 0/0
-            @py.test.mark.skipif("True")
-            def test_skip(): pass
-            def test_skip_imperative():
-                py.test.skip("hello")
-            @py.test.mark.xfail("True")
-            def test_xfail(): 0/0
-            def test_xfail_imperative():
-                py.test.xfail("hello")
-        """)
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 17 # with setup/teardown "passed" reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("testreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if newrep.skipped and not hasattr(newrep, "wasxfail"):
-                assert len(newrep.longrepr) == 3
-            assert newrep.outcome == rep.outcome
-            assert newrep.when == rep.when
-            assert newrep.keywords == rep.keywords
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
 
-    def test_collectreport_passed(self, testdir):
-        reprec = testdir.inline_runsource("def test_func(): pass")
-        reports = reprec.getreports("pytest_collectreport")
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
+class TestWorkerInteractor:
+    @pytest.fixture
+    def unserialize_report(self, pytestconfig):
+        def unserialize(data):
+            return pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
 
-    def test_collectreport_fail(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
+        return unserialize
 
-    def test_extended_report_deserialization(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            rep.extra = True
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.extra
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-
-class TestSlaveInteractor:
-    def test_basic_collect_and_runtests(self, slave):
-        p = slave.testdir.makepyfile("""
+    def test_basic_collect_and_runtests(
+        self, worker: WorkerSetup, unserialize_report
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
             def test_func():
                 pass
-        """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        """
+        )
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
-        assert ev.kwargs['topdir'] == slave.testdir.tmpdir
-        ids = ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        assert ev.kwargs["topdir"] == str(worker.pytester.path)
+        ids = ev.kwargs["ids"]
         assert len(ids) == 1
-        slave.sendcommand("runtests", indices=list(range(len(ids))))
-        slave.sendcommand("shutdown")
-        ev = slave.popevent("logstart")
+        worker.sendcommand("runtests", indices=list(range(len(ids))))
+        worker.sendcommand("shutdown")
+        ev = worker.popevent("logstart")
         assert ev.kwargs["nodeid"].endswith("test_func")
         assert len(ev.kwargs["location"]) == 3
-        ev = slave.popevent("testreport") # setup
-        ev = slave.popevent("testreport")
+        ev = worker.popevent("testreport")  # setup
+        ev = worker.popevent("testreport")
         assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, slave):
-        p = slave.testdir.makepyfile("""
-            import py
-            py.test.skip("hello")
-        """)
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+    def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
+            """
+            import pytest
+            pytest.skip("hello", allow_module_level=True)
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
-        assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
-        ev = slave.popevent("collectionfinish")
-        assert not ev.kwargs['ids']
+        assert rep.longrepr[2] == "Skipped: hello"
+        ev = worker.popevent("collectionfinish")
+        assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, slave):
-        p = slave.testdir.makepyfile("""aasd qwe""")
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+    def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile("""aasd qwe""")
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
-        assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.failed
-        ev = slave.popevent("collectionfinish")
-        assert not ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, slave):
-        p = slave.testdir.makepyfile("""
+    def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
+            """
             def test_func(): pass
             def test_func2(): pass
-        """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        """
+        )
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
-        ids = ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
         assert len(ids) == 2
-        slave.sendcommand("runtests_all", )
-        slave.sendcommand("shutdown", )
+        worker.sendcommand("runtests_all")
+        worker.sendcommand("shutdown")
         for func in "::test_func", "::test_func2":
             for i in range(3):  # setup/call/teardown
-                ev = slave.popevent("testreport")
+                ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
-                rep = unserialize_report(ev.name, ev.kwargs['data'])
+                rep = unserialize_report(ev.kwargs["data"])
                 assert rep.nodeid.endswith(func)
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, slave):
-        py.test.xfail("implement a simple test for event production")
-        assert not slave.use_callback
-        p = slave.testdir.makepyfile("""
+    def test_happy_run_events_converted(
+        self, pytester: pytest.Pytester, worker: WorkerSetup
+    ) -> None:
+        pytest.xfail("implement a simple test for event production")
+        assert not worker.use_callback  # type: ignore[unreachable]
+        worker.pytester.makepyfile(
+            """
             def test_func():
                 pass
-        """)
-        slave.setup()
-        hookrec = testdir.getreportrecorder(slave.config)
-        for data in slave.slp.channel:
-            slave.slp.process_from_remote(data)
-        slave.slp.process_from_remote(slave.slp.ENDMARK)
-        py.std.pprint.pprint(hookrec.hookrecorder.calls)
-        hookrec.hookrecorder.contains([
-            ("pytest_collectstart", "collector.fspath == aaa"),
-            ("pytest_pycollect_makeitem", "name == 'test_func'"),
-            ("pytest_collectreport", "report.collector.fspath == aaa"),
-            ("pytest_collectstart", "collector.fspath == bbb"),
-            ("pytest_pycollect_makeitem", "name == 'test_func'"),
-            ("pytest_collectreport", "report.collector.fspath == bbb"),
-        ])
+        """
+        )
+        worker.setup()
+        hookrec = pytester.getreportrecorder(worker.config)
+        for data in worker.slp.channel:
+            worker.slp.process_from_remote(data)
+        worker.slp.process_from_remote(worker.slp.ENDMARK)
+        pprint.pprint(hookrec.hookrecorder.calls)
+        hookrec.hookrecorder.contains(
+            [
+                ("pytest_collectstart", "collector.fspath == aaa"),
+                ("pytest_pycollect_makeitem", "name == 'test_func'"),
+                ("pytest_collectreport", "report.collector.fspath == aaa"),
+                ("pytest_collectstart", "collector.fspath == bbb"),
+                ("pytest_pycollect_makeitem", "name == 'test_func'"),
+                ("pytest_collectreport", "report.collector.fspath == bbb"),
+            ]
+        )
 
+    def test_process_from_remote_error_handling(
+        self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        worker.use_callback = True
+        worker.setup()
+        worker.slp.process_from_remote(("<nonono>", ()))
+        out, err = capsys.readouterr()
+        assert "INTERNALERROR> ValueError: unknown event: <nonono>" in out
+        ev = worker.popevent()
+        assert ev.name == "errordown"
+
+    def test_steal_work(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
+            """
+            import time
+            def test_func(): time.sleep(1)
+            def test_func2(): pass
+            def test_func3(): pass
+            def test_func4(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 4
+        worker.sendcommand("runtests_all")
+
+        # wait for test_func setup
+        ev = worker.popevent("testreport")
+        rep = unserialize_report(ev.kwargs["data"])
+        assert rep.nodeid.endswith("::test_func")
+        assert rep.when == "setup"
+
+        worker.sendcommand("steal", indices=[1, 2])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == [2]
+
+        reports = [
+            ("test_func", "call"),
+            ("test_func", "teardown"),
+            ("test_func2", "setup"),
+            ("test_func2", "call"),
+            ("test_func2", "teardown"),
+        ]
+
+        for func, when in reports:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith(f"::{func}")
+            assert rep.when == when
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func4")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
+    def test_steal_empty_queue(self, worker: WorkerSetup, unserialize_report) -> None:
+        worker.pytester.makepyfile(
+            """
+            def test_func(): pass
+            def test_func2(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 2
+        worker.sendcommand("runtests_all")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func")
+            assert rep.when == when
+
+        worker.sendcommand("steal", indices=[0, 1])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == []
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func2")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
+
+def test_remote_env_vars(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import os
+        def test():
+            assert len(os.environ['PYTEST_XDIST_TESTRUNUID']) == 32
+            assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
+            assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
+    """
+    )
+    result = pytester.runpytest("-n2", "--max-worker-restart=0")
+    assert result.ret == 0
+
+
+def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
+    """Test/document the behavior due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_argv():
+            assert sys.argv == ["-c"]
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0
+
+
+def test_remote_mainargv(pytester: pytest.Pytester) -> None:
+    outer_argv = sys.argv
+
+    pytester.makepyfile(
+        """
+        def test_mainargv(request):
+            assert request.config.workerinput["mainargv"] == {!r}
+        """.format(
+            outer_argv
+        )
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0
+
+
+def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
+    if not hasattr(request.config._parser, "prog"):
+        pytest.skip("prog not available in config parser")
+    pytester.makeconftest(
+        """
+        import pytest
+
+        config_parser = None
+
+        @pytest.fixture
+        def get_config_parser():
+            return config_parser
+
+        def pytest_configure(config):
+            global config_parser
+            config_parser = config._parser
+    """
+    )
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test(get_config_parser, request):
+            get_config_parser._getparser().error("my_usage_error")
+    """
+    )
+
+    result = pytester.runpytest_subprocess("-n1")
+    assert result.ret == 1
+    result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(pytester: pytest.Pytester) -> None:
+    """Work around sys.path differences due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0

testing/test_slavemanage.py

@@ -1,242 +0,0 @@
-import py
-import os
-import execnet
-from xdist.slavemanage import HostRSync, NodeManager
-
-pytest_plugins = "pytester",
-
-def pytest_funcarg__hookrecorder(request):
-    _pytest = request.getfuncargvalue('_pytest')
-    config = request.getfuncargvalue('config')
-    return _pytest.gethookrecorder(config.hook)
-
-def pytest_funcarg__config(request):
-    testdir = request.getfuncargvalue("testdir")
-    config = testdir.parseconfig()
-    return config
-
-def pytest_funcarg__mysetup(request):
-    class mysetup:
-        def __init__(self, request):
-            temp = request.getfuncargvalue("tmpdir")
-            self.source = temp.mkdir("source")
-            self.dest = temp.mkdir("dest")
-            request.getfuncargvalue("_pytest")
-    return mysetup(request)
-
-class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, config):
-        gm = NodeManager(config, ["popen"])
-        assert gm.specs[0].chdir is None
-
-    def test_default_chdir(self, config):
-        l = ["ssh=noco", "socket=xyz"]
-        for spec in NodeManager(config, l).specs:
-            assert spec.chdir == "pyexecnetcache"
-        for spec in NodeManager(config, l, defaultchdir="abc").specs:
-            assert spec.chdir == "abc"
-
-    def test_popen_makegateway_events(self, config, hookrecorder, _pytest):
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.makegateways()
-        call = hookrecorder.popcall("pytest_xdist_setupnodes")
-        assert len(call.specs) == 2
-
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.spec == execnet.XSpec("popen")
-        assert call.gateway.id == "gw0"
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.id == "gw1"
-        assert len(hm.group) == 2
-        hm.teardown_nodes()
-        assert not len(hm.group)
-
-    def test_popens_rsync(self, config, mysetup):
-        source = mysetup.source
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.makegateways()
-        assert len(hm.group) == 2
-        for gw in hm.group:
-            class pseudoexec:
-                args = []
-                def __init__(self, *args):
-                    self.args.extend(args)
-                def waitclose(self):
-                    pass
-            gw.remote_exec = pseudoexec
-        l = []
-        hm.rsync(source, notify=lambda *args: l.append(args))
-        assert not l
-        hm.teardown_nodes()
-        assert not len(hm.group)
-        assert "sys.path.insert" in gw.remote_exec.args[0]
-
-    def test_rsync_popen_with_path(self, config, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 1)
-        hm.makegateways()
-        source.ensure("dir1", "dir2", "hello")
-        l = []
-        hm.rsync(source, notify=lambda *args: l.append(args))
-        assert len(l) == 1
-        assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
-        hm.teardown_nodes()
-        dest = dest.join(source.basename)
-        assert dest.join("dir1").check()
-        assert dest.join("dir1", "dir2").check()
-        assert dest.join("dir1", "dir2", 'hello').check()
-
-    def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 2)
-        hm.makegateways()
-        source.ensure("dir1", "dir2", "hello")
-        hm.rsync(source)
-        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
-        assert call.source == source
-        assert len(call.gateways) == 1
-        assert call.gateways[0] in hm.group
-        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
-
-class TestHRSync:
-    def pytest_funcarg__mysetup(self, request):
-        class mysetup:
-            def __init__(self, request):
-                tmp = request.getfuncargvalue('tmpdir')
-                self.source = tmp.mkdir("source")
-                self.dest = tmp.mkdir("dest")
-        return mysetup(request)
-
-    def test_hrsync_filter(self, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        source.ensure("dir", "file.txt")
-        source.ensure(".svn", "entries")
-        source.ensure(".somedotfile", "moreentries")
-        source.ensure("somedir", "editfile~")
-        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        l = list(source.visit(rec=syncer.filter,
-                                   fil=syncer.filter))
-        assert len(l) == 3
-        basenames = [x.basename for x in l]
-        assert 'dir' in basenames
-        assert 'file.txt' in basenames
-        assert 'somedir' in basenames
-
-    def test_hrsync_one_host(self, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        gw = execnet.makegateway("popen//chdir=%s" % dest)
-        finished = []
-        rsync = HostRSync(source)
-        rsync.add_target_host(gw, finished=lambda: finished.append(1))
-        source.join("hello.py").write("world")
-        rsync.send()
-        gw.exit()
-        assert dest.join(source.basename, "hello.py").check()
-        assert len(finished) == 1
-
-
-class TestNodeManager:
-    @py.test.mark.xfail
-    def test_rsync_roots_no_roots(self, testdir, mysetup):
-        mysetup.source.ensure("dir1", "file1").write("hello")
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
-        #assert nodemanager.config.topdir == source == config.topdir
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        p, = nodemanager.gwmanager.multi_exec(
-            "import os ; channel.send(os.getcwd())").receive_each()
-        p = py.path.local(p)
-        py.builtin.print_("remote curdir", p)
-        assert p == mysetup.dest.join(config.topdir.basename)
-        assert p.join("dir1").check()
-        assert p.join("dir1", "file1").check()
-
-    def test_popen_rsync_subdir(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir1 = mysetup.source.mkdir("dir1")
-        dir2 = dir1.mkdir("dir2")
-        dir2.ensure("hello")
-        for rsyncroot in (dir1, source):
-            dest.remove()
-            nodemanager = NodeManager(testdir.parseconfig(
-                "--tx", "popen//chdir=%s" % dest,
-                "--rsyncdir", rsyncroot,
-                source,
-            ))
-            nodemanager.makegateways()
-            nodemanager.rsync_roots()
-            if rsyncroot == source:
-                dest = dest.join("source")
-            assert dest.join("dir1").check()
-            assert dest.join("dir1", "dir2").check()
-            assert dest.join("dir1", "dir2", 'hello').check()
-            nodemanager.teardown_nodes()
-
-    def test_init_rsync_roots(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        source.ensure("dir1", "somefile", dir=1)
-        dir2.ensure("hello")
-        source.ensure("bogusdir", "file")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
-            [pytest]
-            rsyncdirs=dir1/dir2
-        """))
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        assert dest.join("dir2").check()
-        assert not dest.join("dir1").check()
-        assert not dest.join("bogus").check()
-
-    def test_rsyncignore(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        dir5 = source.ensure("dir5", "dir6", "bogus")
-        dirf = source.ensure("dir5", "file")
-        dir2.ensure("hello")
-        dirfoo = source.ensure("foo", "bar")
-        dirbar = source.ensure("bar", "foo")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
-            [pytest]
-            rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6 foo*
-        """))
-        config = testdir.parseconfig(source)
-        config.option.rsyncignore = ['bar']
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        assert dest.join("dir1").check()
-        assert not dest.join("dir1", "dir2").check()
-        assert dest.join("dir5","file").check()
-        assert not dest.join("dir6").check()
-        assert not dest.join('foo').check()
-        assert not dest.join('bar').check()
-
-    def test_optimise_popen(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        specs = ["popen"] * 3
-        source.join("conftest.py").write("rsyncdirs = ['a']")
-        source.ensure('a', dir=1)
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, specs)
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        for gwspec in nodemanager.specs:
-            assert gwspec._samefilesystem()
-            assert not gwspec.chdir
-
-    def test_ssh_setup_nodes(self, specssh, testdir):
-        testdir.makepyfile(__init__="", test_x="""
-            def test_one():
-                pass
-        """)
-        reprec = testdir.inline_run("-d", "--rsyncdir=%s" % testdir.tmpdir,
-                "--tx", specssh, testdir.tmpdir)
-        rep, = reprec.getreports("pytest_runtest_logreport")
-        assert rep.passed
-
-

testing/test_workermanage.py

@@ -0,0 +1,405 @@
+import execnet
+import pytest
+import shutil
+import textwrap
+import warnings
+from pathlib import Path
+from util import generate_warning
+from xdist import workermanage
+from xdist._path import visit_path
+from xdist.remote import serialize_warning_message
+from xdist.workermanage import HostRSync, NodeManager, unserialize_warning_message
+
+pytest_plugins = "pytester"
+
+
+@pytest.fixture
+def hookrecorder(request, config, pytester: pytest.Pytester):
+    hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
+    return hookrecorder
+
+
+@pytest.fixture
+def config(pytester: pytest.Pytester):
+    return pytester.parseconfig()
+
+
+@pytest.fixture
+def source(tmp_path: Path) -> Path:
+    source = tmp_path / "source"
+    source.mkdir()
+    return source
+
+
+@pytest.fixture
+def dest(tmp_path: Path) -> Path:
+    dest = tmp_path / "dest"
+    dest.mkdir()
+    return dest
+
+
+@pytest.fixture
+def workercontroller(monkeypatch: pytest.MonkeyPatch):
+    class MockController:
+        def __init__(self, *args):
+            pass
+
+        def setup(self):
+            pass
+
+    monkeypatch.setattr(workermanage, "WorkerController", MockController)
+    return MockController
+
+
+class TestNodeManagerPopen:
+    def test_popen_no_default_chdir(self, config) -> None:
+        gm = NodeManager(config, ["popen"])
+        assert gm.specs[0].chdir is None
+
+    def test_default_chdir(self, config) -> None:
+        specs = ["ssh=noco", "socket=xyz"]
+        for spec in NodeManager(config, specs).specs:
+            assert spec.chdir == "pyexecnetcache"
+        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
+            assert spec.chdir == "abc"
+
+    def test_popen_makegateway_events(
+        self, config, hookrecorder, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)
+        call = hookrecorder.popcall("pytest_xdist_setupnodes")
+        assert len(call.specs) == 2
+
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
+        assert call.gateway.spec == execnet.XSpec("popen")
+        assert call.gateway.id == "gw0"
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
+        assert call.gateway.id == "gw1"
+        assert len(hm.group) == 2
+        hm.teardown_nodes()
+        assert not len(hm.group)
+
+    def test_popens_rsync(
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)
+        assert len(hm.group) == 2
+        for gw in hm.group:
+
+            class pseudoexec:
+                args = []  # type: ignore[var-annotated]
+
+                def __init__(self, *args):
+                    self.args.extend(args)
+
+                def waitclose(self):
+                    pass
+
+            gw.remote_exec = pseudoexec
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert not notifications
+        hm.teardown_nodes()
+        assert not len(hm.group)
+        assert "sys.path.insert" in gw.remote_exec.args[0]
+
+    def test_rsync_popen_with_path(
+        self, config, source: Path, dest: Path, workercontroller
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
+        hm.setup_nodes(None)
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert len(notifications) == 1
+        assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
+        hm.teardown_nodes()
+        dest = dest.joinpath(source.name)
+        assert dest.joinpath("dir1").exists()
+        assert dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir1", "dir2", "hello").exists()
+
+    def test_rsync_same_popen_twice(
+        self,
+        config,
+        source: Path,
+        dest: Path,
+        hookrecorder,
+        workercontroller,
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
+        hm.roots = []
+        hm.setup_nodes(None)
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        gw = hm.group[0]
+        hm.rsync(gw, source)
+        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
+        assert call.source == source
+        assert len(call.gateways) == 1
+        assert call.gateways[0] in hm.group
+        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
+
+
+class TestHRSync:
+    def test_hrsync_filter(self, source: Path, dest: Path) -> None:
+        source.joinpath("dir").mkdir()
+        source.joinpath("dir", "file.txt").touch()
+        source.joinpath(".svn").mkdir()
+        source.joinpath(".svn", "entries").touch()
+        source.joinpath(".somedotfile").mkdir()
+        source.joinpath(".somedotfile", "moreentries").touch()
+        source.joinpath("somedir").mkdir()
+        source.joinpath("somedir", "editfile~").touch()
+        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
+        files = list(visit_path(source, recurse=syncer.filter, filter=syncer.filter))
+        names = {x.name for x in files}
+        assert names == {"dir", "file.txt", "somedir"}
+
+    def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
+        gw = execnet.makegateway("popen//chdir=%s" % dest)
+        finished = []
+        rsync = HostRSync(source)
+        rsync.add_target_host(gw, finished=lambda: finished.append(1))
+        source.joinpath("hello.py").write_text("world")
+        rsync.send()
+        gw.exit()
+        assert dest.joinpath(source.name, "hello.py").exists()
+        assert len(finished) == 1
+
+
+class TestNodeManager:
+    @pytest.mark.xfail(run=False)
+    def test_rsync_roots_no_roots(
+        self, pytester: pytest.Pytester, source: Path, dest: Path
+    ) -> None:
+        source.joinpath("dir1").mkdir()
+        source.joinpath("dir1", "file1").write_text("hello")
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        # assert nodemanager.config.topdir == source == config.topdir
+        nodemanager.makegateways()  # type: ignore[attr-defined]
+        nodemanager.rsync_roots()  # type: ignore[call-arg]
+        (p,) = nodemanager.gwmanager.multi_exec(  # type: ignore[attr-defined]
+            "import os ; channel.send(os.getcwd())"
+        ).receive_each()
+        p = Path(p)
+        print("remote curdir", p)
+        assert p == dest.joinpath(config.rootpath.name)
+        assert p.joinpath("dir1").check()
+        assert p.joinpath("dir1", "file1").check()
+
+    def test_popen_rsync_subdir(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        dir2 = dir1 / "dir2"
+        dir2.mkdir()
+        dir2.joinpath("hello").touch()
+        for rsyncroot in (dir1, source):
+            shutil.rmtree(str(dest), ignore_errors=True)
+            nodemanager = NodeManager(
+                pytester.parseconfig(
+                    "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
+                )
+            )
+            nodemanager.setup_nodes(None)  # calls .rsync_roots()
+            if rsyncroot == source:
+                dest = dest.joinpath("source")
+            assert dest.joinpath("dir1").exists()
+            assert dest.joinpath("dir1", "dir2").exists()
+            assert dest.joinpath("dir1", "dir2", "hello").exists()
+            nodemanager.teardown_nodes()
+
+    @pytest.mark.parametrize(
+        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+    )
+    def test_rsync_report(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller,
+        capsys: pytest.CaptureFixture[str],
+        flag: str,
+        expects_report: bool,
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
+        if flag:
+            args.append(flag)
+        nodemanager = NodeManager(pytester.parseconfig(*args))
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
+    def test_init_rsync_roots(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir1", "somefile").mkdir()
+        dir2.joinpath("hello").touch()
+        source.joinpath("bogusdir").mkdir()
+        source.joinpath("bogusdir", "file").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs=dir1/dir2
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        assert dest.joinpath("dir2").exists()
+        assert not dest.joinpath("dir1").exists()
+        assert not dest.joinpath("bogus").exists()
+
+    def test_rsyncignore(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir5", "dir6").mkdir(parents=True)
+        source.joinpath("dir5", "dir6", "bogus").touch()
+        source.joinpath("dir5", "file").touch()
+        dir2.joinpath("hello").touch()
+        source.joinpath("foo").mkdir()
+        source.joinpath("foo", "bar").touch()
+        source.joinpath("bar").mkdir()
+        source.joinpath("bar", "foo").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs = dir1 dir5
+                rsyncignore = dir1/dir2 dir5/dir6 foo*
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        config.option.rsyncignore = ["bar"]
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        assert dest.joinpath("dir1").exists()
+        assert not dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir5", "file").exists()
+        assert not dest.joinpath("dir6").exists()
+        assert not dest.joinpath("foo").exists()
+        assert not dest.joinpath("bar").exists()
+
+    def test_optimise_popen(
+        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+    ) -> None:
+        specs = ["popen"] * 3
+        source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
+        source.joinpath("a").mkdir()
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, specs)
+        nodemanager.setup_nodes(None)  # calls .rysnc_roots()
+        for gwspec in nodemanager.specs:
+            assert gwspec._samefilesystem()
+            assert not gwspec.chdir
+
+    def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            __init__="",
+            test_x="""
+            def test_one():
+                pass
+        """,
+        )
+        reprec = pytester.inline_run(
+            "-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
+        )
+        (rep,) = reprec.getreports("pytest_runtest_logreport")
+        assert rep.passed
+
+
+class MyWarning(UserWarning):
+    pass
+
+
+@pytest.mark.parametrize(
+    "w_cls",
+    [
+        UserWarning,
+        MyWarning,
+        "Imported",
+        pytest.param(
+            "Nested",
+            marks=pytest.mark.xfail(reason="Nested warning classes are not supported."),
+        ),
+    ],
+)
+def test_unserialize_warning_msg(w_cls):
+    """Test that warning serialization process works well"""
+
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        if not isinstance(w_cls, str):
+            warnings.warn("hello", w_cls)
+        elif w_cls == "Imported":
+            generate_warning()
+        elif w_cls == "Nested":
+            # dynamic creation
+            class MyWarning2(UserWarning):
+                pass
+
+            warnings.warn("hello", MyWarning2)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+    w_msg2 = unserialize_warning_message(data)
+
+    # Compare the two objects
+    all_keys = set(vars(w_msg).keys()).union(set(vars(w_msg2).keys()))
+    for k in all_keys:
+        v1 = getattr(w_msg, k)
+        v2 = getattr(w_msg2, k)
+        if k == "message":
+            assert type(v1) == type(v2)
+            assert v1.args == v2.args
+        else:
+            assert v1 == v2
+
+
+class MyWarningUnknown(UserWarning):
+    # Changing the __module__ attribute is only safe if class can be imported
+    # from there
+    __module__ = "unknown"
+
+
+def test_warning_serialization_tweaked_module():
+    """Test for GH#404"""
+
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        warnings.warn("hello", MyWarningUnknown)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+
+    # __module__ cannot be found!
+    with pytest.raises(ModuleNotFoundError):
+        unserialize_warning_message(data)

testing/util.py

@@ -0,0 +1,9 @@
+import warnings
+
+
+class MyWarning2(UserWarning):
+    pass
+
+
+def generate_warning():
+    warnings.warn(MyWarning2("hello"))

tox.ini

@@ -1,27 +1,69 @@
 [tox]
-envlist=py26,py32,py33,py27,py27-pexpect,py33-pexpect,py26,py26-old,py33-old
-
+envlist=
+  linting
+  py{37,38,39,310,311}-pytestlatest
+  py310-pytestmain
+  py310-psutil
+  py310-setproctitle
+isolated_build = true
 [testenv]
-changedir=testing
-deps=pytest>=2.5.1
-commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
+extras = testing
+deps =
+  pytestlatest: pytest
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
+commands=
+  pytest {posargs}
 
-[testenv:py27-pexpect]
-deps={[testenv]deps}
-    pexpect
-[testenv:py33-pexpect]
-deps={[testenv]deps}
-    pexpect
+[testenv:py310-psutil]
+extras =
+  testing
+  psutil
+commands =
+  pytest {posargs:-k psutil}
 
-[testenv:py26-old]
-deps=
-    pytest==2.4.2
+[testenv:py310-setproctitle]
+extras =
+  testing
+  setproctitle
+deps = pytest
+commands =
+  pytest {posargs}
 
-[testenv:py33-old]
-basepython = python3.3
-deps=
-    pytest==2.4.2
+[testenv:linting]
+skip_install = True
+usedevelop = True
+passenv = PRE_COMMIT_HOME
+deps =
+  pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
+
+[testenv:release]
+changedir=
+description = do a release, required posarg of the version number
+basepython = python3.10
+skipsdist = True
+usedevelop = True
+passenv = *
+deps =
+  towncrier
+commands =
+  towncrier build --version {posargs} --yes
+
+[testenv:docs]
+basepython = python3.10
+usedevelop = True
+deps =
+    sphinx
+    sphinx_rtd_theme
+commands =
+    sphinx-build -W --keep-going -b html docs docs/_build/html {posargs:}
 
 [pytest]
-addopts = -rsfxX
-;; hello
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
+testpaths = testing
+
+[flake8]
+max-line-length = 120
+ignore = E203,W503

xdist/__init__.py

@@ -1,2 +0,0 @@
-#
-__version__ = '1.10'

xdist/dsession.py

@@ -1,435 +0,0 @@
-import sys
-import difflib
-
-import pytest
-import py
-from xdist.slavemanage import NodeManager
-
-
-queue = py.builtin._tryimport('queue', 'Queue')
-
-
-class EachScheduling:
-
-    def __init__(self, numnodes, log=None):
-        self.numnodes = numnodes
-        self.node2collection = {}
-        self.node2pending = {}
-        if log is None:
-            self.log = py.log.Producer("eachsched")
-        else:
-            self.log = log.loadsched
-        self.collection_is_completed = False
-
-    def hasnodes(self):
-        return bool(self.node2pending)
-
-    def addnode(self, node):
-        self.node2collection[node] = None
-
-    def tests_finished(self):
-        if not self.collection_is_completed:
-            return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        assert not self.collection_is_completed
-        assert self.node2collection[node] is None
-        self.node2collection[node] = list(collection)
-        self.node2pending[node] = []
-        if len(self.node2pending) >= self.numnodes:
-            self.collection_is_completed = True
-
-    def remove_item(self, node, item_index, duration=0):
-        self.node2pending[node].remove(item_index)
-
-    def remove_node(self, node):
-        # KeyError if we didn't get an addnode() yet
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-        crashitem = self.node2collection[node][pending.pop(0)]
-        # XXX what about the rest of pending?
-        return crashitem
-
-    def init_distribute(self):
-        assert self.collection_is_completed
-        for node, pending in self.node2pending.items():
-            node.send_runtest_all()
-            pending[:] = range(len(self.node2collection[node]))
-
-class LoadScheduling:
-    def __init__(self, numnodes, log=None):
-        self.numnodes = numnodes
-        self.node2pending = {}
-        self.node2collection = {}
-        self.pending = []
-        if log is None:
-            self.log = py.log.Producer("loadsched")
-        else:
-            self.log = log.loadsched
-        self.collection_is_completed = False
-
-    def hasnodes(self):
-        return bool(self.node2pending)
-
-    def addnode(self, node):
-        self.node2pending[node] = []
-
-    def tests_finished(self):
-        if not self.collection_is_completed or self.pending:
-            return False
-        #for items in self.node2pending.values():
-        #    if items:
-        #        return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        assert not self.collection_is_completed
-        assert node in self.node2pending
-        self.node2collection[node] = list(collection)
-        if len(self.node2collection) >= self.numnodes:
-            self.collection_is_completed = True
-
-    def remove_item(self, node, item_index, duration=0):
-        node_pending = self.node2pending[node]
-        node_pending.remove(item_index)
-        # pre-load items-to-test if the node may become ready
-
-        if self.pending:
-            if duration >= 0.1 and node_pending:
-                # seems the node is doing long-running tests
-                # so let's rather wait with sending new items
-                return
-            # how many nodes do we have remaining per node roughly?
-            num_nodes = len(self.node2pending)
-            # if our node goes below a heuristic minimum, fill it out to
-            # heuristic maximum
-            items_per_node_min = max(
-                    1, len(self.pending) // num_nodes // 4)
-            items_per_node_max = max(
-                    1, len(self.pending) // num_nodes // 2)
-            if len(node_pending) <= items_per_node_min:
-                num_send = items_per_node_max - len(node_pending) + 1
-                self._send_tests(node, num_send)
-
-        self.log("num items waiting for node:", len(self.pending))
-        #self.log("node2pending:", self.node2pending)
-
-    def remove_node(self, node):
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-        # the node must have crashed on the item if there are pending ones
-        crashitem = self.collection[pending.pop(0)]
-        self.pending.extend(pending)
-        return crashitem
-
-    def init_distribute(self):
-        assert self.collection_is_completed
-        # XXX allow nodes to have different collections
-        node_collection_items = list(self.node2collection.items())
-        first_node, col = node_collection_items[0]
-        for node, collection in node_collection_items[1:]:
-            report_collection_diff(
-                col,
-                collection,
-                first_node.gateway.id,
-                node.gateway.id,
-            )
-
-        # all collections are the same, good.
-        # we now create an index
-        self.collection = col
-        self.pending[:] = range(len(col))
-        if not col:
-            return
-        # how many items per node do we have about?
-        items_per_node = len(self.collection) // len(self.node2pending)
-        # take a fraction of tests for initial distribution
-        node_chunksize = max(items_per_node // 4, 1)
-        # and initialize each node with a chunk of tests
-        for node in self.node2pending:
-            self._send_tests(node, node_chunksize)
-
-    #f = open("/tmp/sent", "w")
-    def _send_tests(self, node, num):
-        tests_per_node = self.pending[:num]
-        #print >>self.f, "sent", node, tests_per_node
-        if tests_per_node:
-            del self.pending[:num]
-            self.node2pending[node].extend(tests_per_node)
-            node.send_runtest_some(tests_per_node)
-
-def report_collection_diff(from_collection, to_collection, from_id, to_id):
-    """Report the collected test difference between two nodes.
-
-    :returns: True if collections are equal.
-
-    :raises: AssertionError with a detailed error message describing the
-             difference between the collections.
-
-    """
-    if from_collection == to_collection:
-        return True
-
-    diff = difflib.unified_diff(
-        from_collection,
-        to_collection,
-        fromfile=from_id,
-        tofile=to_id,
-    )
-    error_message = py.builtin._totext(
-        'Different tests were collected between {from_id} and {to_id}. '
-        'The difference is:\n'
-        '{diff}'
-    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
-    raise AssertionError(msg)
-
-
-class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
-
-class DSession:
-    def __init__(self, config):
-        self.config = config
-        self.log = py.log.Producer("dsession")
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-        self.shuttingdown = False
-        self.countfailures = 0
-        self.maxfail = config.getvalue("maxfail")
-        self.queue = queue.Queue()
-        self._failed_collection_errors = {}
-        try:
-            self.terminal = config.pluginmanager.getplugin("terminalreporter")
-        except KeyError:
-            self.terminal = None
-        else:
-            self.trdist = TerminalDistReporter(config)
-            config.pluginmanager.register(self.trdist, "terminaldistreporter")
-
-    def report_line(self, line):
-        if self.terminal and self.config.option.verbose >= 0:
-            self.terminal.write_line(line)
-
-    @pytest.mark.trylast
-    def pytest_sessionstart(self, session):
-        self.nodemanager = NodeManager(self.config)
-        self.nodemanager.setup_nodes(putevent=self.queue.put)
-
-    def pytest_sessionfinish(self, session):
-        """ teardown any resources after a test run. """
-        nm = getattr(self, 'nodemanager', None) # if not fully initialized
-        if nm is not None:
-            nm.teardown_nodes()
-
-    def pytest_collection(self):
-        # prohibit collection of test items in master process
-        return True
-
-    def pytest_runtestloop(self):
-        numnodes = len(self.nodemanager.specs)
-        dist = self.config.getvalue("dist")
-        if dist == "load":
-            self.sched = LoadScheduling(numnodes, log=self.log)
-        elif dist == "each":
-            self.sched = EachScheduling(numnodes, log=self.log)
-        else:
-            assert 0, dist
-        self.shouldstop = False
-        self.session_finished = False
-        while not self.session_finished:
-            self.loop_once()
-            if self.shouldstop:
-                raise Interrupted(str(self.shouldstop))
-        return True
-
-    def loop_once(self):
-        """ process one callback from one of the slaves. """
-        while 1:
-            try:
-                eventcall = self.queue.get(timeout=2.0)
-                break
-            except queue.Empty:
-                continue
-        callname, kwargs = eventcall
-        assert callname, kwargs
-        method = "slave_" + callname
-        call = getattr(self, method)
-        self.log("calling method", method, kwargs)
-        call(**kwargs)
-        if self.sched.tests_finished():
-            self.triggershutdown()
-
-    #
-    # callbacks for processing events from slaves
-    #
-
-    def slave_slaveready(self, node, slaveinfo):
-        node.slaveinfo = slaveinfo
-        node.slaveinfo['id'] = node.gateway.id
-        node.slaveinfo['spec'] = node.gateway.spec
-        self.config.hook.pytest_testnodeready(node=node)
-        self.sched.addnode(node)
-        if self.shuttingdown:
-            node.shutdown()
-
-    def slave_slavefinished(self, node):
-        self.config.hook.pytest_testnodedown(node=node, error=None)
-        if node.slaveoutput['exitstatus'] == 2: # keyboard-interrupt
-            self.shouldstop = "%s received keyboard-interrupt" % (node,)
-            self.slave_errordown(node, "keyboard-interrupt")
-            return
-        crashitem = self.sched.remove_node(node)
-        #assert not crashitem, (crashitem, node)
-        if self.shuttingdown and not self.sched.hasnodes():
-            self.session_finished = True
-
-    def slave_errordown(self, node, error):
-        self.config.hook.pytest_testnodedown(node=node, error=error)
-        try:
-            crashitem = self.sched.remove_node(node)
-        except KeyError:
-            pass
-        else:
-            if crashitem:
-                self.handle_crashitem(crashitem, node)
-                #self.report_line("item crashed on node: %s" % crashitem)
-        if not self.sched.hasnodes():
-            self.session_finished = True
-
-    def slave_collectionfinish(self, node, ids):
-        self.sched.addnode_collection(node, ids)
-        if self.terminal:
-            self.trdist.setstatus(node.gateway.spec, "[%d]" %(len(ids)))
-
-        if self.sched.collection_is_completed:
-            if self.terminal:
-                self.trdist.ensure_show_status()
-                self.terminal.write_line("")
-                self.terminal.write_line("scheduling tests via %s" %(
-                    self.sched.__class__.__name__))
-
-            self.sched.init_distribute()
-
-    def slave_logstart(self, node, nodeid, location):
-        self.config.hook.pytest_runtest_logstart(
-            nodeid=nodeid, location=location)
-
-    def slave_testreport(self, node, rep):
-        if not (rep.passed and rep.when != "call"):
-            if rep.when in ("setup", "call"):
-                self.sched.remove_item(node, rep.item_index, rep.duration)
-        #self.report_line("testreport %s: %s" %(rep.id, rep.status))
-        rep.node = node
-        self.config.hook.pytest_runtest_logreport(report=rep)
-        self._handlefailures(rep)
-
-    def slave_collectreport(self, node, rep):
-        if rep.failed:
-            self._failed_slave_collectreport(node, rep)
-
-    def _failed_slave_collectreport(self, node, rep):
-        # Check we haven't already seen this report (from
-        # another slave).
-        if rep.longrepr not in self._failed_collection_errors:
-            self._failed_collection_errors[rep.longrepr] = True
-            self.config.hook.pytest_collectreport(report=rep)
-            self._handlefailures(rep)
-
-    def _handlefailures(self, rep):
-        if rep.failed:
-            self.countfailures += 1
-            if self.maxfail and self.countfailures >= self.maxfail:
-                self.shouldstop = "stopping after %d failures" % (
-                    self.countfailures)
-
-    def triggershutdown(self):
-        self.log("triggering shutdown")
-        self.shuttingdown = True
-        for node in self.sched.node2pending:
-            node.shutdown()
-
-    def handle_crashitem(self, nodeid, slave):
-        # XXX get more reporting info by recording pytest_runtest_logstart?
-        runner = self.config.pluginmanager.getplugin("runner")
-        fspath = nodeid.split("::")[0]
-        msg = "Slave %r crashed while running %r" %(slave.gateway.id, nodeid)
-        rep = runner.TestReport(nodeid, (fspath, None, fspath), (),
-            "failed", msg, "???")
-        rep.node = slave
-        self.config.hook.pytest_runtest_logreport(report=rep)
-
-class TerminalDistReporter:
-    def __init__(self, config):
-        self.config = config
-        self.tr = config.pluginmanager.getplugin("terminalreporter")
-        self._status = {}
-        self._lastlen = 0
-
-    def write_line(self, msg):
-        self.tr.write_line(msg)
-
-    def ensure_show_status(self):
-        if not self.tr.hasmarkup:
-            self.write_line(self.getstatus())
-
-    def setstatus(self, spec, status, show=True):
-        self._status[spec.id] = status
-        if show and self.tr.hasmarkup:
-            self.rewrite(self.getstatus())
-
-    def getstatus(self):
-        parts = ["%s %s" %(spec.id, self._status[spec.id])
-                   for spec in self._specs]
-        return " / ".join(parts)
-
-    def rewrite(self, line, newline=False):
-        pline = line + " " * max(self._lastlen-len(line), 0)
-        if newline:
-            self._lastlen = 0
-            pline += "\n"
-        else:
-            self._lastlen = len(line)
-        self.tr.rewrite(pline, bold=True)
-
-    def pytest_xdist_setupnodes(self, specs):
-        self._specs = specs
-        for spec in specs:
-            self.setstatus(spec, "I", show=False)
-        self.setstatus(spec, "I", show=True)
-        self.ensure_show_status()
-
-    def pytest_xdist_newgateway(self, gateway):
-        if self.config.option.verbose > 0:
-            rinfo = gateway._rinfo()
-            version = "%s.%s.%s" % rinfo.version_info[:3]
-            self.rewrite("[%s] %s Python %s cwd: %s" % (
-                gateway.id, rinfo.platform, version, rinfo.cwd),
-                newline=True)
-        self.setstatus(gateway.spec, "C")
-
-    def pytest_testnodeready(self, node):
-        if self.config.option.verbose > 0:
-            d = node.slaveinfo
-            infoline = "[%s] Python %s" %(
-                d['id'],
-                d['version'].replace('\n', ' -- '),)
-            self.rewrite(infoline, newline=True)
-        self.setstatus(node.gateway.spec, "ok")
-
-    def pytest_testnodedown(self, node, error):
-        if not error:
-            return
-        self.write_line("[%s] node down: %s" %(node.gateway.id, error))
-
-    #def pytest_xdist_rsyncstart(self, source, gateways):
-    #    targets = ",".join([gw.id for gw in gateways])
-    #    msg = "[%s] rsyncing: %s" %(targets, source)
-    #    self.write_line(msg)
-    #def pytest_xdist_rsyncfinish(self, source, gateways):
-    #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
-    #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
-

xdist/newhooks.py

@@ -1,21 +0,0 @@
-
-def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
-
-def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
-
-def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
-
-def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
-
-def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
-
-def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
-
-def pytest_testnodedown(node, error):
-    """ Test Node is down. """

xdist/plugin.py

@@ -1,127 +0,0 @@
-import py
-import pytest
-
-def pytest_addoption(parser):
-    group = parser.getgroup("xdist", "distributed and subprocess testing")
-    group._addoption('-f', '--looponfail',
-           action="store_true", dest="looponfail", default=False,
-           help="run tests in subprocess, wait for modified files "
-                "and re-run failing test set until all pass.")
-    group._addoption('-n', dest="numprocesses", metavar="numprocesses",
-           action="store", type="int",
-           help="shortcut for '--dist=load --tx=NUM*popen'")
-    group.addoption('--boxed',
-           action="store_true", dest="boxed", default=False,
-           help="box each test run in a separate process (unix)")
-    group._addoption('--dist', metavar="distmode",
-           action="store", choices=['load', 'each', 'no'],
-           type="choice", dest="dist", default="no",
-           help=("set mode for distributing tests to exec environments.\n\n"
-                 "each: send each test to each available environment.\n\n"
-                 "load: send each test to available environment.\n\n"
-                 "(default) no: run tests inprocess, don't distribute."))
-    group._addoption('--tx', dest="tx", action="append", default=[],
-           metavar="xspec",
-           help=("add a test execution environment. some examples: "
-                 "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
-                 "--tx ssh=user@codespeak.net//chdir=testcache"))
-    group._addoption('-d',
-           action="store_true", dest="distload", default=False,
-           help="load-balance tests.  shortcut for '--dist=load'")
-    group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
-           help="add directory for rsyncing to remote tx nodes.")
-    group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
-           help="add expression for ignores when rsyncing to remote tx nodes.")
-
-    parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
-         ' remote distributed testing.', type="pathlist")
-    parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
-         'for rsyncing.', type="pathlist")
-    parser.addini("looponfailroots", type="pathlist",
-        help="directories to check for changes", default=[py.path.local()])
-
-# -------------------------------------------------------------------------
-# distributed testing hooks
-# -------------------------------------------------------------------------
-def pytest_addhooks(pluginmanager):
-    from xdist import newhooks
-    pluginmanager.addhooks(newhooks)
-
-# -------------------------------------------------------------------------
-# distributed testing initialization
-# -------------------------------------------------------------------------
-
-def pytest_cmdline_main(config):
-    check_options(config)
-    if config.getvalue("looponfail"):
-        from xdist.looponfail import looponfail_main
-        looponfail_main(config)
-        return 2 # looponfail only can get stop with ctrl-C anyway
-
-def pytest_configure(config, __multicall__):
-    __multicall__.execute()
-    if config.getvalue("dist") != "no":
-        from xdist.dsession import DSession
-        session = DSession(config)
-        config.pluginmanager.register(session, "dsession")
-        tr = config.pluginmanager.getplugin("terminalreporter")
-        tr.showfspath = False
-
-def check_options(config):
-    if config.option.numprocesses:
-        config.option.dist = "load"
-        config.option.tx = ['popen'] * int(config.option.numprocesses)
-    if config.option.distload:
-        config.option.dist = "load"
-    val = config.getvalue
-    if not val("collectonly"):
-        usepdb = config.option.usepdb  # a core option
-        if val("looponfail"):
-            if usepdb:
-                raise pytest.UsageError("--pdb incompatible with --looponfail.")
-        elif val("dist") != "no":
-            if usepdb:
-                raise pytest.UsageError("--pdb incompatible with distributing tests.")
-
-
-def pytest_runtest_protocol(item):
-    if item.config.getvalue("boxed"):
-        reports = forked_run_report(item)
-        for rep in reports:
-            item.ihook.pytest_runtest_logreport(report=rep)
-        return True
-
-def forked_run_report(item):
-    # for now, we run setup/teardown in the subprocess
-    # XXX optionally allow sharing of setup/teardown
-    from _pytest.runner import runtestprotocol
-    EXITSTATUS_TESTEXIT = 4
-    import marshal
-    from xdist.remote import serialize_report
-    from xdist.slavemanage import unserialize_report
-    def runforked():
-        try:
-            reports = runtestprotocol(item, log=False)
-        except KeyboardInterrupt:
-            py.std.os._exit(EXITSTATUS_TESTEXIT)
-        return marshal.dumps([serialize_report(x) for x in reports])
-
-    ff = py.process.ForkedFunc(runforked)
-    result = ff.waitfinish()
-    if result.retval is not None:
-        report_dumps = marshal.loads(result.retval)
-        return [unserialize_report("testreport", x) for x in report_dumps]
-    else:
-        if result.exitstatus == EXITSTATUS_TESTEXIT:
-            py.test.exit("forked test item %s raised Exit" %(item,))
-        return [report_process_crash(item, result)]
-
-def report_process_crash(item, result):
-    path, lineno = item._getfslineno()
-    info = "%s:%s: running the test CRASHED with signal %d" %(
-            path, lineno, result.signal)
-    from _pytest import runner
-    call = runner.CallInfo(lambda: 0/0, "???")
-    call.excinfo = info
-    rep = runner.pytest_runtest_makereport(item, call)
-    return rep

xdist/remote.py

@@ -1,143 +0,0 @@
-"""
-    This module is executed in remote subprocesses and helps to
-    control a remote testing session and relay back information.
-    It assumes that 'py' is importable and does not have dependencies
-    on the rest of the xdist code.  This means that the xdist-plugin
-    needs not to be installed in remote environments.
-"""
-
-import sys, os
-
-class SlaveInteractor:
-    def __init__(self, config, channel):
-        self.config = config
-        self.slaveid = config.slaveinput.get('slaveid', "?")
-        self.log = py.log.Producer("slave-%s" % self.slaveid)
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-        self.channel = channel
-        config.pluginmanager.register(self)
-
-    def sendevent(self, name, **kwargs):
-        self.log("sending", name, kwargs)
-        self.channel.send((name, kwargs))
-
-    def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
-            self.log("IERROR>", line)
-
-    def pytest_sessionstart(self, session):
-        self.session = session
-        slaveinfo = getinfodict()
-        self.sendevent("slaveready", slaveinfo=slaveinfo)
-
-    def pytest_sessionfinish(self, __multicall__, exitstatus):
-        self.config.slaveoutput['exitstatus'] = exitstatus
-        res = __multicall__.execute()
-        self.sendevent("slavefinished", slaveoutput=self.config.slaveoutput)
-        return res
-
-    def pytest_collection(self, session):
-        self.sendevent("collectionstart")
-
-    def pytest_runtestloop(self, session):
-        self.log("entering main loop")
-        torun = []
-        while 1:
-            name, kwargs = self.channel.receive()
-            self.log("received command", name, kwargs)
-            if name == "runtests":
-                torun.extend(kwargs['indices'])
-            elif name == "runtests_all":
-                torun.extend(range(len(session.items)))
-            self.log("items to run:", torun)
-            while torun:
-                self.run_tests(torun)
-            if name == "shutdown":
-                break
-        return True
-
-    def run_tests(self, torun):
-        items = self.session.items
-        self.item_index = torun.pop(0)
-        if torun:
-            nextitem = items[torun[0]]
-        else:
-            nextitem = None
-        self.config.hook.pytest_runtest_protocol(
-            item=items[self.item_index],
-            nextitem=nextitem)
-
-    def pytest_collection_finish(self, session):
-        self.sendevent("collectionfinish",
-            topdir=str(session.fspath),
-            ids=[item.nodeid for item in session.items])
-
-    def pytest_runtest_logstart(self, nodeid, location):
-        self.sendevent("logstart", nodeid=nodeid, location=location)
-
-    def pytest_runtest_logreport(self, report):
-        data = serialize_report(report)
-        data["item_index"] = self.item_index
-        assert self.session.items[self.item_index].nodeid == report.nodeid
-        self.sendevent("testreport", data=data)
-
-    def pytest_collectreport(self, report):
-        data = serialize_report(report)
-        self.sendevent("collectreport", data=data)
-
-def serialize_report(rep):
-    import py
-    d = rep.__dict__.copy()
-    if hasattr(rep.longrepr, 'toterminal'):
-        d['longrepr'] = str(rep.longrepr)
-    else:
-        d['longrepr'] = rep.longrepr
-    for name in d:
-        if isinstance(d[name], py.path.local):
-            d[name] = str(d[name])
-        elif name == "result":
-            d[name] = None # for now
-    return d
-
-def getinfodict():
-    import platform
-    return dict(
-        version = sys.version,
-        version_info = tuple(sys.version_info),
-        sysplatform = sys.platform,
-        platform = platform.platform(),
-        executable = sys.executable,
-        cwd = os.getcwd(),
-    )
-
-def remote_initconfig(option_dict, args):
-    from _pytest.config import Config
-    option_dict['plugins'].append("no:terminal")
-    config = Config.fromdictargs(option_dict, args)
-    config.option.looponfail = False
-    config.option.usepdb = False
-    config.option.dist = "no"
-    config.option.distload = False
-    config.option.numprocesses = None
-    config.args = args
-    return config
-
-
-if __name__ == '__channelexec__':
-    # python3.2 is not concurrent import safe, so let's play it safe
-    # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32
-    if sys.version_info[:2] == (3,2):
-        os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
-    slaveinput,args,option_dict = channel.receive()
-    importpath = os.getcwd()
-    sys.path.insert(0, importpath) # XXX only for remote situations
-    os.environ['PYTHONPATH'] = (importpath + os.pathsep +
-        os.environ.get('PYTHONPATH', ''))
-    #os.environ['PYTHONPATH'] = importpath
-    import py
-    config = remote_initconfig(option_dict, args)
-    config.slaveinput = slaveinput
-    config.slaveoutput = {}
-    interactor = SlaveInteractor(config, channel)
-    config.hook.pytest_cmdline_main(config=config)

xdist/slavemanage.py

@@ -1,316 +0,0 @@
-import fnmatch
-import os
-
-import py
-import pytest
-import execnet
-import xdist.remote
-
-from _pytest import runner # XXX load dynamically
-
-class NodeManager(object):
-    EXIT_TIMEOUT = 10
-    DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
-    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
-        self.config = config
-        self._nodesready = py.std.threading.Event()
-        self.trace = self.config.trace.get("nodemanager")
-        self.group = execnet.Group()
-        if specs is None:
-            specs = self._getxspecs()
-        self.specs = []
-        for spec in specs:
-            if not isinstance(spec, execnet.XSpec):
-                spec = execnet.XSpec(spec)
-            if not spec.chdir and not spec.popen:
-                spec.chdir = defaultchdir
-            self.group.allocate_id(spec)
-            self.specs.append(spec)
-        self.roots = self._getrsyncdirs()
-        self.rsyncoptions = self._getrsyncoptions()
-
-    def rsync_roots(self):
-        """ make sure that all remote gateways
-            have the same set of roots in their
-            current directory.
-        """
-        if self.roots:
-            # send each rsync root
-            for root in self.roots:
-                self.rsync(root, **self.rsyncoptions)
-
-    def makegateways(self):
-        assert not list(self.group)
-        self.config.hook.pytest_xdist_setupnodes(config=self.config,
-            specs=self.specs)
-        for spec in self.specs:
-            gw = self.group.makegateway(spec)
-            self.config.hook.pytest_xdist_newgateway(gateway=gw)
-
-    def setup_nodes(self, putevent):
-        self.makegateways()
-        self.rsync_roots()
-        self.trace("setting up nodes")
-        for gateway in self.group:
-            node = SlaveController(self, gateway, self.config, putevent)
-            gateway.node = node  # to keep node alive
-            node.setup()
-            self.trace("started node %r" % node)
-
-    def teardown_nodes(self):
-        self.group.terminate(self.EXIT_TIMEOUT)
-
-    def _getxspecs(self):
-        xspeclist = []
-        for xspec in self.config.getvalue("tx"):
-            i = xspec.find("*")
-            try:
-                num = int(xspec[:i])
-            except ValueError:
-                xspeclist.append(xspec)
-            else:
-                xspeclist.extend([xspec[i+1:]] * num)
-        if not xspeclist:
-            raise pytest.UsageError(
-                "MISSING test execution (tx) nodes: please specify --tx")
-        return [execnet.XSpec(x) for x in xspeclist]
-
-    def _getrsyncdirs(self):
-        for spec in self.specs:
-            if not spec.popen or spec.chdir:
-                break
-        else:
-            return []
-        import pytest, _pytest
-        pytestpath = pytest.__file__.rstrip("co")
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
-        config = self.config
-        candidates = [py._pydir,pytestpath,pytestdir]
-        candidates += config.option.rsyncdir
-        rsyncroots = config.getini("rsyncdirs")
-        if rsyncroots:
-            candidates.extend(rsyncroots)
-        roots = []
-        for root in candidates:
-            root = py.path.local(root).realpath()
-            if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: %r" %(root,))
-            if root not in roots:
-                roots.append(root)
-        return roots
-
-    def _getrsyncoptions(self):
-        """Get options to be passed for rsync."""
-        ignores = list(self.DEFAULT_IGNORES)
-        ignores += self.config.option.rsyncignore
-        ignores += self.config.getini("rsyncignore")
-
-        return {
-            'ignores': ignores,
-            'verbose': self.config.option.verbose,
-        }
-
-
-    def rsync(self, source, notify=None, verbose=False, ignores=None):
-        """ perform rsync to all remote hosts.
-        """
-        rsync = HostRSync(source, verbose=verbose, ignores=ignores)
-        seen = py.builtin.set()
-        gateways = []
-        for gateway in self.group:
-            spec = gateway.spec
-            if spec.popen and not spec.chdir:
-                # XXX this assumes that sources are python-packages
-                # and that adding the basedir does not hurt
-                gateway.remote_exec("""
-                    import sys ; sys.path.insert(0, %r)
-                """ % os.path.dirname(str(source))).waitclose()
-                continue
-            if spec not in seen:
-                def finished():
-                    if notify:
-                        notify("rsyncrootready", spec, source)
-                rsync.add_target_host(gateway, finished=finished)
-                seen.add(spec)
-                gateways.append(gateway)
-        if seen:
-            self.config.hook.pytest_xdist_rsyncstart(
-                source=source,
-                gateways=gateways,
-            )
-            rsync.send()
-            self.config.hook.pytest_xdist_rsyncfinish(
-                source=source,
-                gateways=gateways,
-            )
-
-class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
-    """
-    def __init__(self, sourcedir, *args, **kwargs):
-        self._synced = {}
-        ignores= None
-        if 'ignores' in kwargs:
-            ignores = kwargs.pop('ignores')
-        self._ignores = ignores or []
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
-
-    def filter(self, path):
-        path = py.path.local(path)
-        for x in self._ignores:
-            x = getattr(x, 'strpath', x)
-            if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
-                return False
-        else:
-            return True
-
-    def add_target_host(self, gateway, finished=None):
-        remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(gateway, remotepath,
-                                          finishedcallback=finished,
-                                          delete=True,)
-
-    def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
-            path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
-            remotepath = gateway.spec.chdir
-            py.builtin.print_('%s:%s <= %s' %
-                              (gateway.spec, remotepath, path))
-
-
-def make_reltoroot(roots, args):
-    # XXX introduce/use public API for splitting py.test args
-    splitcode = "::"
-    l = []
-    for arg in args:
-        parts = arg.split(splitcode)
-        fspath = py.path.local(parts[0])
-        for root in roots:
-            x = fspath.relto(root)
-            if x or fspath == root:
-                parts[0] = root.basename + "/" + x
-                break
-        else:
-            raise ValueError("arg %s not relative to an rsync root" % (arg,))
-        l.append(splitcode.join(parts))
-    return l
-
-class SlaveController(object):
-    ENDMARK = -1
-
-    def __init__(self, nodemanager, gateway, config, putevent):
-        self.nodemanager = nodemanager
-        self.putevent = putevent
-        self.gateway = gateway
-        self.config = config
-        self.slaveinput = {'slaveid': gateway.id}
-        self._down = False
-        self.log = py.log.Producer("slavectl-%s" % gateway.id)
-        if not self.config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-
-    def __repr__(self):
-        return "<%s %s>" %(self.__class__.__name__, self.gateway.id,)
-
-    def setup(self):
-        self.log("setting up slave session")
-        spec = self.gateway.spec
-        args = self.config.args
-        if not spec.popen or spec.chdir:
-            args = make_reltoroot(self.nodemanager.roots, args)
-        option_dict = vars(self.config.option)
-        if spec.popen:
-            name = "popen-%s" % self.gateway.id
-            basetemp = self.config._tmpdirhandler.getbasetemp()
-            option_dict['basetemp'] = str(basetemp.join(name))
-        self.config.hook.pytest_configure_node(node=self)
-        self.channel = self.gateway.remote_exec(xdist.remote)
-        self.channel.send((self.slaveinput, args, option_dict))
-        if self.putevent:
-            self.channel.setcallback(self.process_from_remote,
-                endmarker=self.ENDMARK)
-
-    def ensure_teardown(self):
-        if hasattr(self, 'channel'):
-            if not self.channel.isclosed():
-                self.log("closing", self.channel)
-                self.channel.close()
-            #del self.channel
-        if hasattr(self, 'gateway'):
-            self.log("exiting", self.gateway)
-            self.gateway.exit()
-            #del self.gateway
-
-    def send_runtest_some(self, indices):
-        self.sendcommand("runtests", indices=indices)
-
-    def send_runtest_all(self):
-        self.sendcommand("runtests_all",)
-
-    def shutdown(self):
-        if not self._down:
-            try:
-                self.sendcommand("shutdown")
-            except IOError:
-                pass
-
-    def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
-        self.log("sending command %s(**%s)" % (name, kwargs))
-        self.channel.send((name, kwargs))
-
-    def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing %s(**%s)" % (eventname, kwargs))
-        self.putevent((eventname, kwargs))
-
-    def process_from_remote(self, eventcall):
-        """ this gets called for each object we receive from
-            the other side and if the channel closes.
-
-            Note that channel callbacks run in the receiver
-            thread of execnet gateways - we need to
-            avoid raising exceptions or doing heavy work.
-        """
-        try:
-            if eventcall == self.ENDMARK:
-                err = self.channel._getremoteerror()
-                if not self._down:
-                    if not err or isinstance(err, EOFError):
-                        err = "Not properly terminated" # lost connection?
-                    self.notify_inproc("errordown", node=self, error=err)
-                    self._down = True
-                return
-            eventname, kwargs = eventcall
-            if eventname in ("collectionstart"):
-                self.log("ignoring %s(%s)" %(eventname, kwargs))
-            elif eventname == "slaveready":
-                self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname == "slavefinished":
-                self._down = True
-                self.slaveoutput = kwargs['slaveoutput']
-                self.notify_inproc("slavefinished", node=self)
-            elif eventname == "logstart":
-                self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname in ("testreport", "collectreport", "teardownreport"):
-                item_index = kwargs.pop("item_index", None)
-                rep = unserialize_report(eventname, kwargs['data'])
-                if item_index is not None:
-                    rep.item_index = item_index
-                self.notify_inproc(eventname, node=self, rep=rep)
-            elif eventname == "collectionfinish":
-                self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
-            else:
-                raise ValueError("unknown event: %s" %(eventname,))
-        except KeyboardInterrupt:
-            # should not land in receiver-thread
-            raise
-        except:
-            excinfo = py.code.ExceptionInfo()
-            py.builtin.print_("!" * 20, excinfo)
-            self.config.pluginmanager.notify_exception(excinfo)
-
-def unserialize_report(name, reportdict):
-    if name == "testreport":
-        return runner.TestReport(**reportdict)
-    elif name == "collectreport":
-        return runner.CollectReport(**reportdict)