gotcha

for more information, see https://pre-commit.ci

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,15 @@
+Thanks for submitting a PR, your contribution is really appreciated!
+
+Here's a quick checklist that should be present in PRs:
+
+- [ ] Make sure to include reasonable tests for your change if necessary
+
+- [ ] We use [towncrier](https://pypi.python.org/pypi/towncrier) for changelog management, so please add a *news* file into the `changelog` folder following these guidelines:
+  * Name it `$issue_id.$type` for example `588.bugfix`;
+  * If you don't have an issue_id change it to the PR id after creating it
+  * Ensure type is one of `removal`, `feature`, `bugfix`, `vendor`, `doc` or `trivial`
+  * Make sure to use full sentences with correct case and punctuation, for example:
+
+    ```
+    Fix issue with non-ascii contents in doctest text files.
+    ```

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/deploy.yml

@@ -0,0 +1,57 @@
+name: deploy
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version'
+        required: true
+        default: '1.2.3'
+
+jobs:
+
+  package:
+    runs-on: ubuntu-latest
+    env:
+      SETUPTOOLS_SCM_PRETEND_VERSION: ${{ github.event.inputs.version }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build and Check Package
+      uses: hynek/build-and-inspect-python-package@v2.13
+
+  deploy:
+    needs: package
+    runs-on: ubuntu-latest
+    environment: deploy
+    permissions:
+      id-token: write  # For PyPI trusted publishers.
+      contents: write  # For tag.
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download Package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@v1.12.4
+      with:
+        attestations: true
+
+    - name: Push tag
+      run: |
+        git config user.name "pytest bot"
+        git config user.email "pytestbot@gmail.com"
+        git tag --annotate --message=v${{ github.event.inputs.version }} v${{ github.event.inputs.version }} ${{ github.sha }}
+        git push origin v${{ github.event.inputs.version }}
+
+    - name: GitHub Release
+      uses: softprops/action-gh-release@v2
+      with:
+        files: dist/*
+        tag_name: v${{ github.event.inputs.version }}

.github/workflows/test.yml

@@ -0,0 +1,93 @@
+name: test
+
+on:
+  push:
+    branches:
+      - master
+      - "test-me-*"
+
+  pull_request:
+    branches:
+      - "*"
+
+# Cancel running jobs for the same workflow and branch.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  package:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build and Check Package
+      uses: hynek/build-and-inspect-python-package@v2.13
+
+  test:
+
+    needs: [package]
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py39-pytestmin"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py311-pytestlatest"
+          - "py311-pytestmain"
+          - "py312-pytestlatest"
+          - "py313-pytestlatest"
+          - "py310-psutil"
+          - "py310-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py39-pytestmin"
+            python: "3.9"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10"
+          - tox_env: "py311-pytestlatest"
+            python: "3.11"
+          - tox_env: "py311-pytestmain"
+            python: "3.11"
+          - tox_env: "py312-pytestlatest"
+            python: "3.12"
+          - tox_env: "py313-pytestlatest"
+            python: "3.13"
+          - tox_env: "py310-psutil"
+            python: "3.10"
+          - tox_env: "py310-setproctitle"
+            python: "3.10"
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
+
+    - name: Download Package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+
+    - name: Test
+      shell: bash
+      run: |
+        tox run -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz`

.gitignore

@@ -14,9 +14,22 @@ syntax:glob
 *.class
 *.orig
 
+*.sublime-*
+.Python
+
 build/
 dist/
+include/
+lib/
+bin/
+env/
+src/xdist/_version.py*
 pytest_xdist.egg-info
 issue/
 3rdparty/
-.tox
+pytestdebug.log
+.tox/
+.cache/
+.pytest_cache/
+.eggs/
+.idea/

.hgtags

@@ -1,10 +0,0 @@
-42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0
-eca7ce17eabf296983c36812c8b8be901e7055a3 1.1
-56d8e5280be224a0ad3220a9deed55334710bd23 1.2
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-eaf8b1cb7c312883598677231be5bbeea3b5c127 1.3
-a423748bf17ee778a37853225210257699cad9c1 1.4
-cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
-4815040bdad8f182a5487f57a9da385483836e75 1.6
-20875fed94e7f3dff50bdf762df91153b15ceca6 1.7

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: "v0.12.3"
+  hooks:
+    - id: ruff
+      args: ["--fix"]
+    - id: ruff-format
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.19.1
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black==23.1.0]
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: check-yaml
+-   repo: local
+    hooks:
+    -   id: rst
+        name: rst
+        entry: rst-lint --encoding utf-8
+        files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
+        language: python
+        additional_dependencies: [pygments, restructuredtext_lint]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.16.1
+    hooks:
+    -   id: mypy
+        files: ^(src/|testing/)
+        args: []
+        additional_dependencies:
+          - pytest>=7.0.0
+          - execnet>=2.1.0
+          - types-psutil
+          - setproctitle

.readthedocs.yaml

@@ -0,0 +1,14 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - path: .
+    - requirements: docs/requirements.txt

CHANGELOG

@@ -1,82 +0,0 @@
-1.7
--------------------------
-
-- fix incompatibilities with pytest-2.2.0 (allow multiple
-  pytest_runtest_logreport reports for a test item)
-
-1.6
--------------------------
-
-- terser collection reporting
-
-- fix issue34 - distributed testing with -p plugin now works correctly
-
-- fix race condition in looponfail mode where a concurrent file removal
-  could cause a crash
-
-1.5
--------------------------
-
-- adapt to and require pytest-2.0 changes, rsyncdirs and rsyncignore can now
-  only be specified in [pytest] sections of ini files, see "py.test -h"
-  for details.
-- major internal refactoring to match the pytest-2.0 event refactoring
-  - perform test collection always at slave side instead of at the master
-  - make python2/python3 bridging work, remove usage of pickling
-- improve initial reporting by using line-rewriting
-- remove all trailing whitespace from source
-
-1.4
--------------------------
-
-- perform distributed testing related reporting in the plugin
-  rather than having dist-related code in the generic py.test
-  distribution
-
-- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to
-  gateway specification strings.
-
-- show detailed gateway setup and platform information only when
-  "-v" or "--verbose" is specified.
-
-1.3
--------------------------
-
-- fix --looponfailing - it would not actually run against the fully changed
-  source tree when initial conftest files load application state.
-
-- adapt for py-1.3.1's new --maxfailure option
-
-1.2
--------------------------
-
-- fix issue79: sessionfinish/teardown hooks are now called systematically
-  on the slave side
-- introduce a new data input/output mechanism to allow the master side
-  to send and receive data from a slave.
-- fix race condition in underlying pickling/unpickling handling
-- use and require new register hooks facility of py.test>=1.3.0
-- require improved execnet>=1.0.6 because of various race conditions
-  that can arise in xdist testing modes.
-- fix some python3 related pickling related race conditions
-- fix PyPI description
-
-1.1
--------------------------
-
-- fix an indefinite hang which would wait for events although no events
-  are pending - this happened if items arrive very quickly while
-  the "reschedule-event" tried unconditionally avoiding a busy-loop
-  and not schedule new work.
-
-1.0
--------------------------
-
-- moved code out of py-1.1.1 into its own plugin
-- use a new, faster and more sensible model to do load-balancing
-  of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing
-  works effectively even with very few tests.
-- cleaned up termination handling
-- make -x cause hard killing of test nodes to decrease wait time
-  until the traceback shows up on first failure
-

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no py.test plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

LICENSE

@@ -1,13 +1,21 @@
-The execnet package is released under the provisions of the Gnu Public
-License (GPL), version 2 or later.  
+MIT License
 
-See http://www.fsf.org/licensing/licenses/ for more information.
+Copyright (c) 2010 Holger Krekel and contributors.
 
-This package also contains some minor parts which 
-which are useable under the MIT license. 
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-If you have questions and/or want to use parts of 
-the code under a different license than the GPL 
-please contact me. 
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-holger krekel, January 2010, holger at merlinux eu 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -1,10 +0,0 @@
-include CHANGELOG
-include LICENSE 
-include README.txt
-include setup.py
-graft testing
-exclude *.orig 
-exclude *.rej 
-recursive-exclude testing *.pyc *$py.class
-prune .svn 
-prune .hg 

README.rst

@@ -0,0 +1,33 @@
+============
+pytest-xdist
+============
+
+.. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
+    :alt: PyPI version
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://img.shields.io/conda/vn/conda-forge/pytest-xdist.svg
+    :target: https://anaconda.org/conda-forge/pytest-xdist
+
+.. image:: https://img.shields.io/pypi/pyversions/pytest-xdist.svg
+    :alt: Python versions
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/test/badge.svg
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
+
+.. image:: https://img.shields.io/badge/code%20style-black-000000.svg
+    :target: https://github.com/ambv/black
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+Documentation
+=============
+
+Documentation is available at `Read The Docs <https://pytest-xdist.readthedocs.io>`__.

README.txt

@@ -1,170 +0,0 @@
-xdist: pytest distributed testing plugin
-===============================================================
-
-The `pytest-xdist`_ plugin extends py.test with some unique
-test execution modes:
-
-* Looponfail: run your tests repeatedly in a subprocess.  After each run py.test
-  waits until a file in your project changes and then re-runs the previously
-  failing tests.  This is repeated until all tests pass after which again
-  a full run is performed.
-
-* multiprocess Load-balancing: if you have multiple CPUs or hosts you can use
-  those for a combined test run.  This allows to speed up
-  development or to use special resources of remote machines.
-
-* Multi-Platform coverage: you can specify different Python interpreters
-  or different platforms and run tests in parallel on all of them.
-
-Before running tests remotely, ``py.test`` efficiently "rsyncs" your
-program source code to the remote place.  All test results
-are reported back and displayed to your local terminal.
-You may specify different Python versions and interpreters.
-
-
-Installation
------------------------
-
-Install the plugin with::
-
-    easy_install pytest-xdist
-
-    # or
-    
-    pip install pytest-xdist
-
-or use the package in develope/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
-
-    python setup.py develop
-
-Usage examples
----------------------
-
-Speed up test runs by sending tests to multiple CPUs
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-To send tests to multiple CPUs, type::
-
-    py.test -n NUM
-
-Especially for longer running tests or tests requiring
-a lot of IO this can lead to considerable speed ups.
-
-
-Running tests in a Python subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-To instantiate a python2.4 sub process and send tests to it, you may type::
-
-    py.test -d --tx popen//python=python2.4
-
-This will start a subprocess which is run with the "python2.4"
-Python interpreter, found in your system binary lookup path.
-
-If you prefix the --tx option value like this::
-
-    --tx 3*popen//python=python2.4
-
-then three subprocesses would be created and tests
-will be load-balanced across these three processes.
-
-
-Sending tests to remote SSH accounts
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Suppose you have a package ``mypkg`` which contains some
-tests that you can successfully run locally. And you
-have a ssh-reachable machine ``myhost``.  Then
-you can ad-hoc distribute your tests by typing::
-
-    py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
-
-This will synchronize your ``mypkg`` package directory
-to an remote ssh account and then locally collect tests
-and send them to remote places for execution.
-
-You can specify multiple ``--rsyncdir`` directories
-to be sent to the remote side.
-
-**NOTE:** For py.test to collect and send tests correctly
-you not only need to make sure all code and tests
-directories are rsynced, but that any test (sub) directory
-also has an ``__init__.py`` file because internally
-py.test references tests as a fully qualified python
-module path.  **You will otherwise get strange errors**
-during setup of the remote side.
-
-Sending tests to remote Socket Servers
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-Download the single-module `socketserver.py`_ Python program
-and run it like this::
-
-    python socketserver.py
-
-It will tell you that it starts listening on the default
-port.  You can now on your home machine specify this
-new socket host with something like this::
-
-    py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
-
-
-.. _`atonce`:
-
-Running tests on many platforms at once
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-The basic command to run tests on multiple platforms is::
-
-    py.test --dist=each --tx=spec1 --tx=spec2
-
-If you specify a windows host, an OSX host and a Linux
-environment this command will send each tests to all
-platforms - and report back failures from all platforms
-at once.   The specifications strings use the `xspec syntax`_.
-
-.. _`xspec syntax`: http://codespeak.net/execnet/trunk/basics.html#xspec
-
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
-
-.. _`execnet`: http://codespeak.net/execnet
-
-Specifying test exec environments in an ini file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-pytest (since version 2.0) supports ini-style cofiguration.
-You can for example make running with three subprocesses
-your default like this::
-
-    [pytest]
-    addopts = -n3
-
-You can also add default environments like this::
-
-    [pytest]
-    addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6
-
-and then just type::
-
-    py.test --dist=each
-
-to run tests in each of the environments.
-
-Specifying "rsync" dirs in an ini-file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
-you may specify directories to include or to exclude in synchronisation::
-
-    [pytest]
-    rsyncdirs = . mypkg helperpkg
-    rsyncignore = .hg
-
-These directory specifications are relative to the directory
-where the configuration file was found.
-
-.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
-.. _`pytest-xdist repository`: http://bitbucket.org/hpk42/pytest-xdist
-.. _`pytest`: http://pytest.org
-

RELEASING.rst

@@ -0,0 +1,41 @@
+======================
+Releasing pytest-xdist
+======================
+
+This document describes the steps to make a new ``pytest-xdist`` release.
+
+Version
+-------
+
+``master`` should always be green and a potential release candidate. ``pytest-xdist`` follows
+semantic versioning, so given that the current version is ``X.Y.Z``, to find the next version number
+one needs to look at the ``changelog`` folder:
+
+- If there is any file named ``*.feature``, then we must make a new **minor** release: next
+  release will be ``X.Y+1.0``.
+
+- Otherwise it is just a **bug fix** release: ``X.Y.Z+1``.
+
+
+Steps
+-----
+
+To publish a new release ``X.Y.Z``, the steps are as follows:
+
+#. Create a new branch named ``release-X.Y.Z`` from the latest ``master``.
+
+#. Install ``tox`` in a virtualenv::
+
+    $ pip install tox
+
+#. Update the necessary files with::
+
+    $ tox -e release -- X.Y.Z
+
+#. Commit and push the branch to ``upstream`` and open a PR.
+
+#. Once the PR is **green** and **approved**, start the ``deploy`` workflow manually from the branch ``release-VERSION``, passing ``VERSION`` as parameter, or execute::
+
+   gh workflow run deploy.yml -R pytest-dev/pytest-xdist --ref release-X.Y.Z --field version=X.Y.Z
+
+#. Merge the release PR to ``master``.

changelog/_template.rst

@@ -0,0 +1,38 @@
+{% for section in sections %}
+{% set underline = "-" %}
+{% if section %}
+{{section}}
+{{ underline * section|length }}{% set underline = "~" %}
+
+{% endif %}
+{% if sections[section] %}
+{% for category, val in definitions.items() if category in sections[section] %}
+
+{{ definitions[category]['name'] }}
+{{ underline * definitions[category]['name']|length }}
+
+{% if definitions[category]['showcontent'] %}
+{% for text, values in sections[section][category]|dictsort(by='value') %}
+- `{{ values[0] }} <https://github.com/pytest-dev/pytest-xdist/issues/{{ values[0][1:] }}>`_: {{ text }}
+
+{% endfor %}
+{% else %}
+- {{ sections[section][category]['']|sort|join(', ') }}
+
+
+{% endif %}
+{% if sections[section][category]|length == 0 %}
+
+No significant changes.
+
+
+{% else %}
+{% endif %}
+{% endfor %}
+{% else %}
+
+No significant changes.
+
+
+{% endif %}
+{% endfor %}

docs/.gitignore

@@ -0,0 +1 @@
+_build/

docs/changelog.rst

@@ -0,0 +1,5 @@
+=========
+Changelog
+=========
+
+.. include:: ../CHANGELOG.rst

docs/conf.py

@@ -0,0 +1,55 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "pytest-xdist"
+copyright = "2010, holger krekel and contributors"
+author = "holger krekel and contributors"
+
+master_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx_rtd_theme",
+    "sphinx.ext.autodoc",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']

docs/crash.rst

@@ -0,0 +1,7 @@
+When tests crash
+================
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.

docs/distribution.rst

@@ -0,0 +1,130 @@
+.. _parallelization:
+
+Running tests across multiple CPUs
+==================================
+
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
+
+    pytest -n auto
+
+This can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
+
+With ``-n auto``, pytest-xdist will use as many processes as your computer
+has physical CPU cores.
+
+Use ``-n logical`` to use the number of *logical* CPU cores rather than
+physical ones. This currently requires the `psutil <https://pypi.org/project/psutil/>`__ package to be installed;
+if it is not or if it fails to determine the number of logical CPUs, fall back to ``-n auto`` behavior.
+
+Pass a number, e.g. ``-n 8``, to specify the number of processes explicitly.
+
+Use ``-n 0`` to disable xdist and run all tests in the main process.
+
+To specify a different meaning for ``-n auto`` and ``-n logical`` for your
+tests, you can:
+
+* Set the environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` to the
+  desired number of processes.
+
+* Implement the ``pytest_xdist_auto_num_workers``
+  `pytest hook <https://docs.pytest.org/en/latest/how-to/writing_plugins.html>`__
+  (a ``pytest_xdist_auto_num_workers(config)`` function in e.g. ``conftest.py``)
+  that returns the number of processes to use.
+  The hook can use ``config.option.numprocesses`` to determine if the user
+  asked for ``"auto"`` or ``"logical"``, and it can return ``None`` to fall
+  back to the default.
+
+If both the hook and environment variable are specified, the hook takes
+priority.
+
+
+Parallelization can be configured further with these options:
+
+* ``--maxprocesses=maxprocesses``: limit the maximum number of workers to
+  process the tests.
+
+* ``--max-worker-restart``: maximum number of workers that can be restarted
+  when crashed (set to zero to disable this feature).
+
+The test distribution algorithm is configured with the ``--dist`` command-line option:
+
+.. _distribution modes:
+
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order. Scheduling can be fine-tuned with
+  the `--maxschedchunk` option, see output of `pytest --help`.
+
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
+
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker. If a test has
+  multiple groups, they will be joined together into a new group,
+  the order of the marks doesn't matter. This works along with marks from fixtures
+  and from the pytestmark global variable.
+
+  .. code-block:: python
+
+      @pytest.mark.xdist_group(name="group1")
+      def test1():
+          pass
+
+
+      class TestA:
+          @pytest.mark.xdist_group("group1")
+          def test2():
+              pass
+
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+
+  .. code-block:: python
+
+      @pytest.fixture(
+          scope="session",
+          params=[
+              pytest.param(
+                  "chrome",
+                  marks=pytest.mark.xdist_group("chrome"),
+              ),
+              pytest.param(
+                  "firefox",
+                  marks=pytest.mark.xdist_group("firefox"),
+              ),
+              pytest.param(
+                  "edge",
+                  marks=pytest.mark.xdist_group("edge"),
+              ),
+          ],
+      )
+      def setup_container():
+          pass
+
+
+      @pytest.mark.xdist_group(name="data-store")
+      def test_data_store(setup_container):
+          ...
+
+  This will generate 3 new groups: ``chrome_data-store``, ``data-store_firefox`` and ``data-store_edge`` (the markers are lexically sorted before being merged together).
+
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
+
+* ``--dist worksteal``: Initially, tests are distributed evenly among all
+  available workers. When a worker completes most of its assigned tests and
+  doesn't have enough tests to continue (currently, every worker needs at least
+  two tests in its queue), an attempt is made to reassign ("steal") a portion
+  of tests from some other worker's queue. The results should be similar to
+  the ``load`` method, but ``worksteal`` should handle tests with significantly
+  differing duration better, and, at the same time, it should provide similar
+  or better reuse of fixtures.
+
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).

docs/how-it-works.rst

@@ -0,0 +1,90 @@
+How it works?
+=============
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* instead of passing the full
+   test id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

docs/how-to.rst

@@ -0,0 +1,232 @@
+How-tos
+-------
+
+This section show cases how to accomplish some specialized tasks with ``pytest-xdist``.
+
+Identifying the worker process during a test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.15.*
+
+If you need to determine the identity of a worker process in
+a test or fixture, you may use the ``worker_id`` fixture to do so:
+
+.. code-block:: python
+
+    @pytest.fixture()
+    def user_account(worker_id):
+        """use a different account in each xdist worker"""
+        return "account_%s" % worker_id
+
+When ``xdist`` is disabled (running with ``-n0`` for example), then
+``worker_id`` will return ``"master"``.
+
+Worker processes also have the following environment variables
+defined:
+
+.. envvar:: PYTEST_XDIST_WORKER
+
+The name of the worker, e.g., ``"gw2"``.
+
+.. envvar:: PYTEST_XDIST_WORKER_COUNT
+
+The total number of workers in this session, e.g., ``"4"`` when ``-n 4`` is given in the command-line.
+
+The information about the worker_id in a test is stored in the ``TestReport`` as
+well, under the ``worker_id`` attribute.
+
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+
+.. autofunction:: xdist.is_xdist_worker
+.. autofunction:: xdist.is_xdist_controller
+.. autofunction:: xdist.is_xdist_master
+.. autofunction:: xdist.get_xdist_worker_id
+
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
+Uniquely identifying the current test run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """create a unique database for this particular test run"""
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """retrieve unique database"""
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+.. envvar:: PYTEST_XDIST_TESTRUNUID
+
+The unique id of the test run.
+
+Accessing ``sys.argv`` from the controller node in workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the ``sys.argv`` passed to the command-line of the controller node, use
+``request.config.workerinput["mainargv"]``.
+
+
+Specifying test exec environments in an ini file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use pytest's ini file configuration to avoid typing common options.
+You can for example make running with three subprocesses your default like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = -n3
+
+You can also add default environments like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
+
+and then just type::
+
+    pytest --dist=each
+
+to run tests in each of the environments.
+
+
+Specifying "rsync" dirs in an ini-file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
+you may specify directories to include or to exclude in synchronisation:
+
+.. code-block:: ini
+
+    [pytest]
+    rsyncdirs = . mypkg helperpkg
+    rsyncignore = .hg
+
+These directory specifications are relative to the directory
+where the configuration file was found.
+
+.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
+.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
+.. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+Creating one log file for each worker
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create one log file for each worker with ``pytest-xdist``, you can leverage :envvar:`PYTEST_XDIST_WORKER`
+to generate a unique filename for each worker.
+
+Example:
+
+.. code-block:: python
+
+    # content of conftest.py
+    def pytest_configure(config):
+        worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+        if worker_id is not None:
+            logging.basicConfig(
+                format=config.getini("log_file_format"),
+                filename=f"tests_{worker_id}.log",
+                level=config.getini("log_file_level"),
+            )
+
+
+When running the tests with ``-n3``, for example, three files will be created in the current directory:
+``tests_gw0.log``, ``tests_gw1.log`` and ``tests_gw2.log``.

docs/index.rst

@@ -0,0 +1,63 @@
+pytest-xdist
+============
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+.. note::
+    Due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+Features
+--------
+
+* Test run :ref:`parallelization`: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of :ref:`remote machines`.
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
+  pytest waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed (DEPRECATED).
+
+* :ref:`Multi-Platform` coverage: you can specify different Python interpreters
+  or different platforms and run tests in parallel on all of them.
+
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  program source code to the remote place.
+  You may specify different Python versions and interpreters. It does not
+  installs/synchronize dependencies however.
+
+  **Note**: this mode exists mostly for backward compatibility, as modern development
+  relies on continuous integration for multi-platform testing.
+
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Contents:
+
+    distribution
+    subprocess
+    remote
+    crash
+    how-to
+    how-it-works
+    known-limitations
+    changelog

docs/known-limitations.rst

@@ -0,0 +1,71 @@
+Known limitations
+=================
+
+pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist.
+
+Order and amount of test must be consistent
+-------------------------------------------
+
+It is not possible to have tests that differ in order or their amount across workers.
+
+This is especially true with ``pytest.mark.parametrize``, when values are produced with sets or other unordered iterables/generators.
+
+
+Example:
+
+.. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", {"a", "b"})
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+In the example above, the fact that ``set`` are not necessarily ordered can cause different workers
+to collect tests in different order, which will throw an error.
+
+Workarounds
+~~~~~~~~~~~
+
+A solution to this is to guarantee that the parametrized values have the same order.
+
+Some solutions:
+
+* Convert your sequence to a ``list``.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", ["a", "b"])
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+* Sort your sequence, guaranteeing order.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", sorted({"a", "b"}))
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+Output (stdout and stderr) from workers
+---------------------------------------
+
+The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time.
+
+However, this option does not work with ``pytest-xdist`` because `execnet <https://github.com/pytest-dev/execnet>`__ the underlying library used for communication between master and workers, does not support transferring stdout/stderr from workers.
+
+Currently, there are no plans to support this in ``pytest-xdist``.
+
+Debugging
+~~~~~~~~~
+
+This also means that debugging using PDB (or any other debugger that wants to use standard I/O) will not work. The ``--pdb`` option is disabled when distributing tests with ``pytest-xdist`` for this reason.
+
+It is generally likely best to use ``pytest-xdist`` to find failing tests and then debug them without distribution; however, if you need to debug from within a worker process (for example, to address failures that only happen when running tests concurrently), remote debuggers (for example, `python-remote-pdb <https://github.com/ionelmc/python-remote-pdb>`__ or `python-web-pdb <https://github.com/romanvm/python-web-pdb>`__) have been reported to work for this purpose.

docs/remote.rst

@@ -0,0 +1,100 @@
+
+.. _`Multi-Platform`:
+.. _`remote machines`:
+
+Sending tests to remote SSH accounts
+====================================
+
+.. deprecated:: 3.0
+
+.. warning::
+
+    The ``rsync`` feature is deprecated because its implementation is faulty
+    in terms of reproducing the development environment in the remote
+    worker, and there is no clear solution moving forward.
+
+    For that reason, ``rsync`` is scheduled to be removed in release 4.0, to let the team
+    focus on a smaller set of features.
+
+    Note that SSH and socket server are not planned for removal, as they are part
+    of the ``execnet`` feature set.
+
+Suppose you have a package ``mypkg`` which contains some
+tests that you can successfully run locally. And you
+have a ssh-reachable machine ``myhost``.  Then
+you can ad-hoc distribute your tests by typing::
+
+    pytest -d  --rsyncdir mypkg --tx ssh=myhostpopen mypkg/tests/unit/test_something.py
+
+This will synchronize your :code:`mypkg` package directory
+to a remote ssh account and then locally collect tests
+and send them to remote places for execution.
+
+You can specify multiple :code:`--rsyncdir` directories
+to be sent to the remote side.
+
+.. note::
+
+  For pytest to collect and send tests correctly
+  you not only need to make sure all code and tests
+  directories are rsynced, but that any test (sub) directory
+  also has an :code:`__init__.py` file because internally
+  pytest references tests as a fully qualified python
+  module path.  **You will otherwise get strange errors**
+  during setup of the remote side.
+
+
+You can specify multiple :code:`--rsyncignore` glob patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
+Sending tests to remote Socket Servers
+--------------------------------------
+
+Download the single-module `socketserver.py`_ Python program
+and run it like this::
+
+    python socketserver.py
+
+It will tell you that it starts listening on the default
+port.  You can now on your home machine specify this
+new socket host with something like this::
+
+    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg
+
+
+Using proxies to run multiple workers on remote machines
+---------------------------------------
+
+In case you want to run multiple workers on a remote machine,
+you can create a proxy gateway for the machine, and run multiple
+workers using the `via` attribute.::
+
+    pytest -d --px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy
+
+Here we declare a proxy gateway using the `--px` arg, and
+create 5 workers that run on the remote server using the proxy.
+Note that the proxy gateway does not run a worker, thus only 5
+workers are created.
+
+
+Running tests on many platforms at once
+---------------------------------------
+
+The basic command to run tests on multiple platforms is::
+
+    pytest --dist=each --tx=spec1 --tx=spec2
+
+If you specify a windows host, an OSX host and a Linux
+environment this command will send each tests to all
+platforms - and report back failures from all platforms
+at once. The specifications strings use the `xspec syntax`_.
+
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
+
+.. _`execnet`: https://codespeak.net/execnet
+
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/src/execnet/script/socketserver.py

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx
+sphinx-rtd-theme

docs/subprocess.rst

@@ -0,0 +1,16 @@
+Running tests in a Python subprocess
+====================================
+
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
+
+    pytest -d --tx popen//python=python3.9
+
+This will start a subprocess which is run with the ``python3.9``
+Python interpreter, found in your system binary lookup path.
+
+If you prefix the --tx option value like this::
+
+    --tx 3*popen//python=python3.9
+
+then three subprocesses would be created and tests
+will be load-balanced across these three processes.

example/boxed.txt

@@ -0,0 +1,68 @@
+.. warning::
+
+  Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a
+  separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_
+  which can be installed independently. The ``--boxed`` command-line option is deprecated
+  and will be removed in pytest-xdist 3.0.0; use ``--forked`` from pytest-forked instead.
+
+
+If your testing involves C or C++ libraries you might have to deal
+with crashing processes.  The xdist-plugin provides the ``--boxed`` option
+to run each test in a controlled subprocess.  Here is a basic example::
+
+    # content of test_module.py
+
+    import pytest
+    import os
+    import time
+
+    # run test function 50 times with different argument
+    @pytest.mark.parametrize("arg", range(50))
+    def test_func(arg):
+        time.sleep(0.05) # each tests takes a while
+        if arg % 19 == 0:
+            os.kill(os.getpid(), 15)
+
+If you run this with::
+
+    $ pytest -n1
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    collecting ... collected 50 items
+
+    test_module.py f..................f..................f...........
+
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 3.41 seconds ====================
+
+You'll see that a couple of tests are reported as crashing, indicated
+by lower-case ``f`` and the respective failure summary. You can also use
+the xdist-provided parallelization feature to speed up your testing::
+
+    $ pytest -n3
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    gw0 I / gw1 I / gw2 I
+    gw0 [50] / gw1 [50] / gw2 [50]
+
+    scheduling tests via LoadScheduling
+    ..f...............f..................f............
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    [gw0] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 2.03 seconds ====================

example/loadscope/epsilon/__init__.py

@@ -0,0 +1,37 @@
+def epsilon1(arg1, arg2=1000):
+    """Do epsilon1
+
+    Usage:
+
+    >>> epsilon1(10, 20)
+    40
+    >>> epsilon1(30)
+    1040
+    """
+    return arg1 + arg2 + 10
+
+
+def epsilon2(arg1, arg2=1000):
+    """Do epsilon2
+
+    Usage:
+
+    >>> epsilon2(10, 20)
+    -20
+    >>> epsilon2(30)
+    -980
+    """
+    return arg1 - arg2 - 10
+
+
+def epsilon3(arg1, arg2=1000):
+    """Do epsilon3
+
+    Usage:
+
+    >>> epsilon3(10, 20)
+    200
+    >>> epsilon3(30)
+    30000
+    """
+    return arg1 * arg2

example/loadscope/requirements.txt

@@ -0,0 +1,3 @@
+ipdb
+pytest
+../../

example/loadscope/test/test_alpha.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_alpha0():
+    sleep(5)
+    assert True
+
+
+def test_alpha1():
+    sleep(5)
+    assert True
+
+
+def test_alpha2():
+    sleep(5)
+    assert True
+
+
+def test_alpha3():
+    sleep(5)
+    assert True
+
+
+def test_alpha4():
+    sleep(5)
+    assert True
+
+
+def test_alpha5():
+    sleep(5)
+    assert True
+
+
+def test_alpha6():
+    sleep(5)
+    assert True
+
+
+def test_alpha7():
+    sleep(5)
+    assert True
+
+
+def test_alpha8():
+    sleep(5)
+    assert True
+
+
+def test_alpha9():
+    sleep(5)
+    assert True

example/loadscope/test/test_beta.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_beta0():
+    sleep(5)
+    assert True
+
+
+def test_beta1():
+    sleep(5)
+    assert True
+
+
+def test_beta2():
+    sleep(5)
+    assert True
+
+
+def test_beta3():
+    sleep(5)
+    assert True
+
+
+def test_beta4():
+    sleep(5)
+    assert True
+
+
+def test_beta5():
+    sleep(5)
+    assert True
+
+
+def test_beta6():
+    sleep(5)
+    assert True
+
+
+def test_beta7():
+    sleep(5)
+    assert True
+
+
+def test_beta8():
+    sleep(5)
+    assert True
+
+
+def test_beta9():
+    sleep(5)
+    assert True

example/loadscope/test/test_delta.py

@@ -0,0 +1,86 @@
+from time import sleep
+from unittest import TestCase
+
+
+class Delta1(TestCase):
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True
+
+
+class Delta2(TestCase):
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True

example/loadscope/test/test_gamma.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_gamma0():
+    sleep(5)
+    assert True
+
+
+def test_gamma1():
+    sleep(5)
+    assert True
+
+
+def test_gamma2():
+    sleep(5)
+    assert True
+
+
+def test_gamma3():
+    sleep(5)
+    assert True
+
+
+def test_gamma4():
+    sleep(5)
+    assert True
+
+
+def test_gamma5():
+    sleep(5)
+    assert True
+
+
+def test_gamma6():
+    sleep(5)
+    assert True
+
+
+def test_gamma7():
+    sleep(5)
+    assert True
+
+
+def test_gamma8():
+    sleep(5)
+    assert True
+
+
+def test_gamma9():
+    sleep(5)
+    assert True

example/loadscope/tox.ini

@@ -0,0 +1,17 @@
+[tox]
+envlist = test
+setupdir = {toxinidir}/../../
+
+[testenv:test]
+basepython = python3
+passenv = http_proxy https_proxy
+deps = -rrequirements.txt
+changedir = {envtmpdir}
+commands =
+    pytest -s -v \
+        --doctest-modules \
+        --junitxml=tests.xml \
+        --dist=loadscope \
+        --tx=8*popen \
+        {toxinidir}/test \
+        {toxinidir}/epsilon

pyproject.toml

@@ -0,0 +1,170 @@
+[build-system]
+requires = [
+  "setuptools>=77.0",
+  "setuptools-scm[toml]>=6.2.3",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pytest-xdist"
+description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs"
+readme = "README.rst"
+license = "MIT"
+authors = [{name = "holger krekel and contributors", email = "pytest-dev@python.org"}, {email = "holger@merlinux.eu"}]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: Pytest",
+    "Intended Audience :: Developers",
+    "Operating System :: POSIX",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: MacOS :: MacOS X",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Utilities",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+requires-python = ">=3.9"
+dependencies = [
+    "execnet>=2.1",
+    "pytest>=7.0.0",
+]
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/pytest-dev/pytest-xdist"
+Documentation = "https://pytest-xdist.readthedocs.io/en/latest"
+Changelog = "https://pytest-xdist.readthedocs.io/en/latest/changelog.html"
+Source = "https://github.com/pytest-dev/pytest-xdist"
+Tracker = "https://github.com/pytest-dev/pytest-xdist/issues"
+
+[project.entry-points.pytest11]
+xdist = "xdist.plugin"
+"xdist.looponfail" = "xdist.looponfail"
+
+[project.optional-dependencies]
+testing = ["filelock"]
+psutil = ["psutil>=3.0"]
+setproctitle = ["setproctitle"]
+
+[tool.setuptools_scm]
+write_to = "src/xdist/_version.py"
+
+[tool.pytest.ini_options]
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = "-ra -p no:pytest-services"
+testpaths = ["testing"]
+
+[tool.ruff]
+src = ["src"]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "B",        # bugbear
+    "D",        # pydocstyle
+    "E",        # pycodestyle
+    "F",        # pyflakes
+    "I",        # isort
+    "PYI",      # flake8-pyi
+    "UP",       # pyupgrade
+    "RUF",      # ruff
+    "W",        # pycodestyle
+    "T10",      # flake8-debugger
+    "PIE",      # flake8-pie
+    "FA",       # flake8-future-annotations
+    "PGH",      # pygrep-hooks
+    "PLE",      # pylint error
+    "PLW",      # pylint warning
+    "PLR1714",  # Consider merging multiple comparisons
+]
+ignore = [
+    # bugbear ignore
+    "B011",     # Do not `assert False` (`python -O` removes these calls)
+    "B028",     # No explicit `stacklevel` keyword argument found
+    # pydocstyle ignore
+    "D100",     # Missing docstring in public module
+    "D101",     # Missing docstring in public class
+    "D102",     # Missing docstring in public method
+    "D103",     # Missing docstring in public function
+    "D104",     # Missing docstring in public package
+    "D105",     # Missing docstring in magic method
+    "D106",     # Missing docstring in public nested class
+    "D107",     # Missing docstring in `__init__`
+    "D209",     # Multi-line docstring closing quotes should be on a separate line
+    "D205",     # 1 blank line required between summary line and description
+    "D400",     # First line should end with a period
+    "D401",     # First line of docstring should be in imperative mood
+    # ruff ignore
+    "RUF012",   # Mutable class attributes should be annotated with `typing.ClassVar`
+    # pylint ignore
+    "PLW0603",  # Using the global statement
+    "PLW0120",  # remove the else and dedent its contents
+    "PLW2901",  # for loop variable overwritten by assignment target
+    "PLR5501",  # Use `elif` instead of `else` then `if`
+    "UP031",  # Use format specifiers instead of percent format
+]
+
+[tool.ruff.lint.pycodestyle]
+# In order to be able to format for 88 char in ruff format
+max-line-length = 120
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"
+
+[tool.ruff.lint.isort]
+force-single-line = true
+combine-as-imports = true
+force-sort-within-sections = true
+order-by-type = false
+lines-after-imports = 2
+
+[tool.ruff.lint.per-file-ignores]
+"src/xdist/_version.py" = ["I001"]
+
+[tool.mypy]
+mypy_path = ["src"]
+files = ["src", "testing"]
+strict = true
+warn_unreachable = true
+[[tool.mypy.overrides]]
+module = ["xdist._version"]
+ignore_missing_imports = true
+
+
+[tool.towncrier]
+package = "xdist"
+filename = "CHANGELOG.rst"
+directory = "changelog/"
+title_format = "pytest-xdist {version} ({project_date})"
+template = "changelog/_template.rst"
+
+[tool.towncrier.fragment.removal]
+name = "Removals"
+
+[tool.towncrier.fragment.deprecation]
+name = "Deprecations"
+
+[tool.towncrier.fragment.feature]
+name = "Features"
+
+[tool.towncrier.fragment.bugfix]
+name = "Bug Fixes"
+
+[tool.towncrier.fragment.vendor]
+name = "Vendored Libraries"
+
+[tool.towncrier.fragment.doc]
+name = "Improved Documentation"
+
+[tool.towncrier.fragment.trivial]
+name = "Trivial Changes"

setup.py

@@ -1,30 +0,0 @@
-from setuptools import setup
-
-setup(
-    name="pytest-xdist",
-    version='1.7.dev1',
-    description='py.test xdist plugin for distributed testing and loop-on-failing modes',
-    long_description=open('README.txt').read(),
-    license='GPLv2 or later',
-    author='holger krekel and contributors',
-    author_email='py-dev@codespeak.net,holger@merlinux.eu',
-    url='http://bitbucket.org/hpk42/pytest-xdist',
-    platforms=['linux', 'osx', 'win32'],
-    packages = ['xdist'],
-    entry_points = {'pytest11': ['xdist = xdist.plugin'],},
-    zip_safe=False,
-    install_requires = ['execnet>=1.0.8', 'pytest>=2.2.0'],
-    classifiers=[
-    'Development Status :: 5 - Production/Stable',
-    'Intended Audience :: Developers',
-    'License :: OSI Approved :: GNU General Public License (GPL)',
-    'Operating System :: POSIX',
-    'Operating System :: Microsoft :: Windows',
-    'Operating System :: MacOS :: MacOS X',
-    'Topic :: Software Development :: Testing',
-    'Topic :: Software Development :: Quality Assurance',
-    'Topic :: Utilities',
-    'Programming Language :: Python',
-    'Programming Language :: Python :: 3',
-    ],
-)

src/xdist/__init__.py

@@ -0,0 +1,14 @@
+from xdist._version import version as __version__
+from xdist.plugin import get_xdist_worker_id
+from xdist.plugin import is_xdist_controller
+from xdist.plugin import is_xdist_master
+from xdist.plugin import is_xdist_worker
+
+
+__all__ = [
+    "__version__",
+    "get_xdist_worker_id",
+    "is_xdist_controller",
+    "is_xdist_master",
+    "is_xdist_worker",
+]

src/xdist/_path.py

@@ -0,0 +1,20 @@
+from collections.abc import Iterator
+from itertools import chain
+import os
+from pathlib import Path
+from typing import Callable
+
+
+def visit_path(
+    path: Path, *, filter: Callable[[Path], bool], recurse: Callable[[Path], bool]
+) -> Iterator[Path]:
+    """
+    Implements the interface of ``py.path.local.visit()`` for Path objects,
+    to simplify porting the code over from ``py.path.local``.
+    """
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames[:] = [x for x in dirnames if recurse(Path(dirpath, x))]
+        for name in chain(dirnames, filenames):
+            p = Path(dirpath, name)
+            if filter(p):
+                yield p

src/xdist/dsession.py

@@ -0,0 +1,601 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from enum import auto
+from enum import Enum
+from queue import Empty
+from queue import Queue
+import sys
+from typing import Any
+import warnings
+
+import execnet
+import pytest
+
+from xdist.remote import Producer
+from xdist.remote import WorkerInfo
+from xdist.scheduler import EachScheduling
+from xdist.scheduler import LoadFileScheduling
+from xdist.scheduler import LoadGroupScheduling
+from xdist.scheduler import LoadScheduling
+from xdist.scheduler import LoadScopeScheduling
+from xdist.scheduler import Scheduling
+from xdist.scheduler import WorkStealingScheduling
+from xdist.workermanage import NodeManager
+from xdist.workermanage import WorkerController
+
+
+class Interrupted(KeyboardInterrupt):
+    """signals an immediate interruption."""
+
+
+class DSession:
+    """A pytest plugin which runs a distributed test session.
+
+    At the beginning of the test session this creates a NodeManager
+    instance which creates and starts all nodes.  Nodes then emit
+    events processed in the pytest_runtestloop hook using the worker_*
+    methods.
+
+    Once a node is started it will automatically start running the
+    pytest mainloop with some custom hooks.  This means a node
+    automatically starts collecting tests.  Once tests are collected
+    it will wait for instructions.
+    """
+
+    shouldstop: bool | str
+
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        self.log = Producer("dsession", enabled=config.option.debug)
+        self.nodemanager: NodeManager | None = None
+        self.sched: Scheduling | None = None
+        self.shuttingdown = False
+        self.countfailures = 0
+        self.maxfail: int = config.getvalue("maxfail")
+        self.queue: Queue[tuple[str, dict[str, Any]]] = Queue()
+        self._session: pytest.Session | None = None
+        self._failed_collection_errors: dict[object, bool] = {}
+        self._active_nodes: set[WorkerController] = set()
+        self._failed_nodes_count = 0
+        self._max_worker_restart = get_default_max_worker_restart(self.config)
+        # summary message to print at the end of the session
+        self._summary_report: str | None = None
+        self.terminal = config.pluginmanager.getplugin("terminalreporter")
+        if self.terminal:
+            self.trdist = TerminalDistReporter(config)
+            config.pluginmanager.register(self.trdist, "terminaldistreporter")
+
+    @property
+    def session_finished(self) -> bool:
+        """Return True if the distributed session has finished.
+
+        This means all nodes have executed all test items.  This is
+        used by pytest_runtestloop to break out of its loop.
+        """
+        return bool(self.shuttingdown and not self._active_nodes)
+
+    def report_line(self, line: str) -> None:
+        if self.terminal and self.config.option.verbose >= 0:
+            self.terminal.write_line(line)
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_sessionstart(self, session: pytest.Session) -> None:
+        """Creates and starts the nodes.
+
+        The nodes are setup to put their events onto self.queue.  As
+        soon as nodes start they will emit the worker_workerready event.
+        """
+        self.nodemanager = NodeManager(self.config)
+        nodes = self.nodemanager.setup_nodes(putevent=self.queue.put)
+        self._active_nodes.update(nodes)
+        self._session = session
+
+    @pytest.hookimpl
+    def pytest_sessionfinish(self) -> None:
+        """Shutdown all nodes."""
+        nm = getattr(self, "nodemanager", None)  # if not fully initialized
+        if nm is not None:
+            nm.teardown_nodes()
+        self._session = None
+
+    @pytest.hookimpl
+    def pytest_collection(self) -> bool:
+        # prohibit collection of test items in controller process
+        return True
+
+    @pytest.hookimpl(trylast=True)
+    def pytest_xdist_make_scheduler(
+        self,
+        config: pytest.Config,
+        log: Producer,
+    ) -> Scheduling | None:
+        dist = config.getvalue("dist")
+        if dist == "each":
+            return EachScheduling(config, log)
+        if dist == "load":
+            return LoadScheduling(config, log)
+        if dist == "loadscope":
+            return LoadScopeScheduling(config, log)
+        if dist == "loadfile":
+            return LoadFileScheduling(config, log)
+        if dist == "loadgroup":
+            return LoadGroupScheduling(config, log)
+        if dist == "worksteal":
+            return WorkStealingScheduling(config, log)
+        return None
+
+    @pytest.hookimpl
+    def pytest_runtestloop(self) -> bool:
+        self.sched = self.config.hook.pytest_xdist_make_scheduler(
+            config=self.config, log=self.log
+        )
+        assert self.sched is not None
+
+        self.shouldstop = False
+        pending_exception = None
+        while not self.session_finished:
+            self.loop_once()
+            if self.shouldstop:
+                self.triggershutdown()
+                pending_exception = Interrupted(str(self.shouldstop))
+        if pending_exception:
+            raise pending_exception
+        return True
+
+    def loop_once(self) -> None:
+        """Process one callback from one of the workers."""
+        while 1:
+            if not self._active_nodes:
+                # If everything has died stop looping
+                self.triggershutdown()
+                raise RuntimeError("Unexpectedly no active workers available")
+            try:
+                eventcall = self.queue.get(timeout=2.0)
+                break
+            except Empty:
+                continue
+        callname, kwargs = eventcall
+        assert callname, kwargs
+        method = "worker_" + callname
+        call = getattr(self, method)
+        self.log("calling method", method, kwargs)
+        call(**kwargs)
+        assert self.sched is not None
+        if self.sched.tests_finished:
+            self.triggershutdown()
+
+    #
+    # callbacks for processing events from workers
+    #
+
+    def worker_workerready(
+        self,
+        node: WorkerController,
+        workerinfo: WorkerInfo,
+    ) -> None:
+        """Emitted when a node first starts up.
+
+        This adds the node to the scheduler, nodes continue with
+        collection without any further input.
+        """
+        node.workerinfo = workerinfo
+        node.workerinfo["id"] = node.gateway.id
+        node.workerinfo["spec"] = node.gateway.spec
+
+        self.config.hook.pytest_testnodeready(node=node)
+        if self.shuttingdown:
+            node.shutdown()
+        else:
+            assert self.sched is not None
+            self.sched.add_node(node)
+
+    def worker_workerfinished(self, node: WorkerController) -> None:
+        """Emitted when node executes its pytest_sessionfinish hook.
+
+        Removes the node from the scheduler.
+
+        The node might not be in the scheduler if it had not emitted
+        workerready before shutdown was triggered.
+        """
+        self.config.hook.pytest_testnodedown(node=node, error=None)
+        if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
+            self.shouldstop = f"{node} received keyboard-interrupt"
+            self.worker_errordown(node, "keyboard-interrupt")
+            return
+        shouldfail = node.workeroutput["shouldfail"]
+        shouldstop = node.workeroutput["shouldstop"]
+        for shouldx in [shouldfail, shouldstop]:
+            if shouldx:
+                if not self.shouldstop:
+                    self.shouldstop = shouldx
+                break
+        else:
+            assert self.sched is not None
+            if node in self.sched.nodes:
+                crashitem = self.sched.remove_node(node)
+                assert not crashitem, (crashitem, node)
+        self._active_nodes.remove(node)
+
+    def worker_internal_error(
+        self, node: WorkerController, formatted_error: str
+    ) -> None:
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            excinfo = pytest.ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
+    def worker_errordown(self, node: WorkerController, error: object | None) -> None:
+        """Emitted by the WorkerController when a node dies."""
+        self.config.hook.pytest_testnodedown(node=node, error=error)
+        assert self.sched is not None
+        try:
+            crashitem = self.sched.remove_node(node)
+        except KeyError:
+            pass
+        else:
+            if crashitem:
+                self.handle_crashitem(crashitem, node)
+
+        self._failed_nodes_count += 1
+        maximum_reached = (
+            self._max_worker_restart is not None
+            and self._failed_nodes_count > self._max_worker_restart
+        )
+        if maximum_reached:
+            if self._max_worker_restart == 0:
+                msg = f"worker {node.gateway.id} crashed and worker restarting disabled"
+            else:
+                msg = f"maximum crashed workers reached: {self._max_worker_restart}"
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
+        else:
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
+            self.shuttingdown = False
+            self._clone_node(node)
+        self._active_nodes.remove(node)
+
+    @pytest.hookimpl
+    def pytest_terminal_summary(self, terminalreporter: Any) -> None:
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", f"xdist: {self._summary_report}")
+
+    def worker_collectionfinish(
+        self, node: WorkerController, ids: Sequence[str]
+    ) -> None:
+        """Worker has finished test collection.
+
+        This adds the collection for this node to the scheduler.  If
+        the scheduler indicates collection is finished (i.e. all
+        initial nodes have submitted their collections), then tells the
+        scheduler to schedule the collected items.  When initiating
+        scheduling the first time it logs which scheduler is in use.
+        """
+        if self.shuttingdown:
+            return
+        self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
+        # tell session which items were effectively collected otherwise
+        # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
+        assert self._session is not None
+        self._session.testscollected = len(ids)
+        assert self.sched is not None
+        self.sched.add_node_collection(node, ids)
+        if self.terminal:
+            self.trdist.setstatus(
+                node.gateway.spec, WorkerStatus.CollectionDone, tests_collected=len(ids)
+            )
+        if self.sched.collection_is_completed:
+            if self.terminal and not self.sched.has_pending:
+                self.trdist.ensure_show_status()
+                self.terminal.write_line("")
+                if self.config.option.verbose > 0:
+                    self.terminal.write_line(
+                        f"scheduling tests via {self.sched.__class__.__name__}"
+                    )
+            self.sched.schedule()
+
+    def worker_logstart(
+        self,
+        node: WorkerController,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
+        """Emitted when a node calls the pytest_runtest_logstart hook."""
+        self.config.hook.pytest_runtest_logstart(nodeid=nodeid, location=location)
+
+    def worker_logfinish(
+        self,
+        node: WorkerController,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
+        """Emitted when a node calls the pytest_runtest_logfinish hook."""
+        self.config.hook.pytest_runtest_logfinish(nodeid=nodeid, location=location)
+
+    def worker_testreport(self, node: WorkerController, rep: pytest.TestReport) -> None:
+        """Emitted when a node calls the pytest_runtest_logreport hook."""
+        rep.node = node  # type: ignore[attr-defined]
+        self.config.hook.pytest_runtest_logreport(report=rep)
+        self._handlefailures(rep)
+
+    def worker_runtest_protocol_complete(
+        self, node: WorkerController, item_index: int, duration: float
+    ) -> None:
+        """
+        Emitted when a node fires the 'runtest_protocol_complete' event,
+        signalling that a test has completed the runtestprotocol and should be
+        removed from the pending list in the scheduler.
+        """
+        assert self.sched is not None
+        self.sched.mark_test_complete(node, item_index, duration)
+
+    def worker_unscheduled(
+        self, node: WorkerController, indices: Sequence[int]
+    ) -> None:
+        """
+        Emitted when a node fires the 'unscheduled' event, signalling that
+        some tests have been removed from the worker's queue and should be
+        sent to some worker again.
+
+        This should happen only in response to 'steal' command, so schedulers
+        not using 'steal' command don't have to implement it.
+        """
+        assert self.sched is not None
+        self.sched.remove_pending_tests_from_node(node, indices)
+
+    def worker_collectreport(
+        self,
+        node: WorkerController,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
+        """Emitted when a node calls the pytest_collectreport hook.
+
+        Because we only need the report when there's a failure/skip, as optimization
+        we only expect to receive failed/skipped reports from workers (#330).
+        """
+        assert not rep.passed
+        self._failed_worker_collectreport(node, rep)
+
+    def worker_warning_recorded(
+        self,
+        warning_message: warnings.WarningMessage,
+        when: str,
+        nodeid: str,
+        location: tuple[str, int, str] | None,
+    ) -> None:
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
+    def _clone_node(self, node: WorkerController) -> WorkerController:
+        """Return new node based on an existing one.
+
+        This is normally for when a node dies, this will copy the spec
+        of the existing node and create a new one with a new id.  The
+        new node will have been setup so it will start calling the
+        "worker_*" hooks and do work soon.
+        """
+        spec = node.gateway.spec
+        spec.id = None
+        assert self.nodemanager is not None
+        self.nodemanager.group.allocate_id(spec)
+        clone = self.nodemanager.setup_node(spec, self.queue.put)
+        self._active_nodes.add(clone)
+        return clone
+
+    def _failed_worker_collectreport(
+        self,
+        node: WorkerController,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
+        # Check we haven't already seen this report (from
+        # another worker).
+        if rep.longrepr not in self._failed_collection_errors:
+            self._failed_collection_errors[rep.longrepr] = True
+            self.config.hook.pytest_collectreport(report=rep)
+            self._handlefailures(rep)
+
+    def _handlefailures(
+        self,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
+        if rep.failed:
+            self.countfailures += 1
+            if (
+                self.maxfail
+                and self.countfailures >= self.maxfail
+                and not self.shouldstop
+            ):
+                self.shouldstop = f"stopping after {self.countfailures} failures"
+
+    def triggershutdown(self) -> None:
+        if not self.shuttingdown:
+            self.log("triggering shutdown")
+            self.shuttingdown = True
+            assert self.sched is not None
+            for node in self.sched.nodes:
+                node.shutdown()
+
+    def handle_crashitem(self, nodeid: str, worker: WorkerController) -> None:
+        # XXX get more reporting info by recording pytest_runtest_logstart?
+        # XXX count no of failures and retry N times
+        fspath = nodeid.split("::")[0]
+        msg = f"worker {worker.gateway.id!r} crashed while running {nodeid!r}"
+        rep = pytest.TestReport(
+            nodeid=nodeid,
+            location=(fspath, None, fspath),
+            keywords={},
+            outcome="failed",
+            longrepr=msg,
+            when="???",  # type: ignore[arg-type]
+        )
+        rep.node = worker  # type: ignore[attr-defined]
+
+        self.config.hook.pytest_handlecrashitem(
+            crashitem=nodeid,
+            report=rep,
+            sched=self.sched,
+        )
+        self.config.hook.pytest_runtest_logreport(report=rep)
+
+
+class WorkerStatus(Enum):
+    """Status of each worker during creation/collection."""
+
+    # Worker spec has just been created.
+    Created = auto()
+
+    # Worker has been initialized.
+    Initialized = auto()
+
+    # Worker is now ready for collection.
+    ReadyForCollection = auto()
+
+    # Worker has finished collection.
+    CollectionDone = auto()
+
+
+class TerminalDistReporter:
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        self.tr = config.pluginmanager.getplugin("terminalreporter")
+        self._status: dict[object, tuple[WorkerStatus, int]] = {}
+        self._lastlen = 0
+        self._isatty = getattr(self.tr, "isatty", self.tr.hasmarkup)
+
+    def write_line(self, msg: str) -> None:
+        self.tr.write_line(msg)
+
+    def ensure_show_status(self) -> None:
+        if not self._isatty:
+            self.write_line(self.getstatus())
+
+    def setstatus(
+        self,
+        spec: execnet.XSpec,
+        status: WorkerStatus,
+        *,
+        tests_collected: int,
+        show: bool = True,
+    ) -> None:
+        self._status[spec.id] = (status, tests_collected)
+        if show and self._isatty:
+            self.rewrite(self.getstatus())
+
+    def getstatus(self) -> str:
+        if self.config.option.verbose >= 0:
+            line = get_workers_status_line(list(self._status.values()))
+            if line:
+                return line
+
+        return "bringing up nodes..."
+
+    def rewrite(self, line: str, newline: bool = False) -> None:
+        pline = line + " " * max(self._lastlen - len(line), 0)
+        if newline:
+            self._lastlen = 0
+            pline += "\n"
+        else:
+            self._lastlen = len(line)
+        self.tr.rewrite(pline, bold=True)
+
+    @pytest.hookimpl
+    def pytest_xdist_setupnodes(self, specs: Sequence[execnet.XSpec]) -> None:
+        self._specs = specs
+        for spec in specs:
+            self.setstatus(spec, WorkerStatus.Created, tests_collected=0, show=False)
+        self.setstatus(spec, WorkerStatus.Created, tests_collected=0, show=True)
+        self.ensure_show_status()
+
+    @pytest.hookimpl
+    def pytest_xdist_newgateway(self, gateway: execnet.Gateway) -> None:
+        if self.config.option.verbose > 0:
+            rinfo = gateway._rinfo()
+            different_interpreter = rinfo.executable != sys.executable
+            if different_interpreter:
+                version = "{}.{}.{}".format(*rinfo.version_info[:3])
+                self.rewrite(
+                    f"[{gateway.id}] {rinfo.platform} Python {version} cwd: {rinfo.cwd}",
+                    newline=True,
+                )
+        self.setstatus(gateway.spec, WorkerStatus.Initialized, tests_collected=0)
+
+    @pytest.hookimpl
+    def pytest_testnodeready(self, node: WorkerController) -> None:
+        if self.config.option.verbose > 0:
+            d = node.workerinfo
+            different_interpreter = d.get("executable") != sys.executable
+            if different_interpreter:
+                version = d["version"].replace("\n", " -- ")
+                self.rewrite(f"[{d['id']}] Python {version}", newline=True)
+        self.setstatus(
+            node.gateway.spec, WorkerStatus.ReadyForCollection, tests_collected=0
+        )
+
+    @pytest.hookimpl
+    def pytest_testnodedown(self, node: WorkerController, error: object) -> None:
+        if not error:
+            return
+        self.write_line(f"[{node.gateway.id}] node down: {error}")
+
+
+def get_default_max_worker_restart(config: pytest.Config) -> int | None:
+    """Gets the default value of --max-worker-restart option if it is not provided.
+
+    Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
+    """
+    result_str: str | None = config.option.maxworkerrestart
+    if result_str is not None:
+        result = int(result_str)
+    elif config.option.numprocesses:
+        # if --max-worker-restart was not provided, use a reasonable default (#226)
+        result = config.option.numprocesses * 4
+    else:
+        result = None
+    return result
+
+
+def get_workers_status_line(
+    status_and_items: Sequence[tuple[WorkerStatus, int]],
+) -> str:
+    """
+    Return the line to display during worker setup/collection based on the
+    status of the workers and number of tests collected for each.
+    """
+    statuses = [s for s, c in status_and_items]
+    total_workers = len(statuses)
+    workers_noun = "worker" if total_workers == 1 else "workers"
+    if status_and_items and all(s == WorkerStatus.CollectionDone for s in statuses):
+        # All workers collect the same number of items, so we grab
+        # the total number of items from the first worker.
+        first = status_and_items[0]
+        status, tests_collected = first
+        tests_noun = "item" if tests_collected == 1 else "items"
+        return f"{total_workers} {workers_noun} [{tests_collected} {tests_noun}]"
+    if WorkerStatus.CollectionDone in statuses:
+        done = sum(1 for s, c in status_and_items if c > 0)
+        return f"collecting: {done}/{total_workers} {workers_noun}"
+    if WorkerStatus.ReadyForCollection in statuses:
+        ready = statuses.count(WorkerStatus.ReadyForCollection)
+        return f"ready: {ready}/{total_workers} {workers_noun}"
+    if WorkerStatus.Initialized in statuses:
+        initialized = statuses.count(WorkerStatus.Initialized)
+        return f"initialized: {initialized}/{total_workers} {workers_noun}"
+    if WorkerStatus.Created in statuses:
+        created = statuses.count(WorkerStatus.Created)
+        return f"created: {created}/{total_workers} {workers_noun}"
+
+    return ""

src/xdist/looponfail.py

@@ -0,0 +1,299 @@
+"""
+Implement -f aka looponfailing for pytest.
+
+NOTE that we try to avoid loading and depending on application modules
+within the controlling process (the one that starts repeatedly test
+processes) otherwise changes to source code can crash
+the controlling process which should best never happen.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import os
+from pathlib import Path
+import sys
+import time
+from typing import Any
+
+from _pytest._io import TerminalWriter
+import execnet
+import pytest
+
+from xdist._path import visit_path
+
+
+@pytest.hookimpl
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("xdist", "distributed and subprocess testing")
+    group._addoption(
+        "-f",
+        "--looponfail",
+        action="store_true",
+        dest="looponfail",
+        default=False,
+        help="Run tests in subprocess: wait for files to be modified, then "
+        "re-run failing test set until all pass.",
+    )
+
+
+@pytest.hookimpl
+def pytest_cmdline_main(config: pytest.Config) -> int | None:
+    if config.getoption("looponfail"):
+        usepdb = config.getoption("usepdb", False)  # a core option
+        if usepdb:
+            raise pytest.UsageError("--pdb is incompatible with --looponfail.")
+        looponfail_main(config)
+        return 2  # looponfail only can get stop with ctrl-C anyway
+    return None
+
+
+def looponfail_main(config: pytest.Config) -> None:
+    remotecontrol = RemoteControl(config)
+    config_roots = config.getini("looponfailroots")
+    if not config_roots:
+        config_roots = [Path.cwd()]
+    rootdirs = [Path(root) for root in config_roots]
+    statrecorder = StatRecorder(rootdirs)
+    try:
+        while 1:
+            remotecontrol.loop_once()
+            if not remotecontrol.failures and remotecontrol.wasfailing:
+                # the last failures passed, let's immediately rerun all
+                continue
+            repr_pytest_looponfailinfo(
+                failreports=remotecontrol.failures, rootdirs=rootdirs
+            )
+            statrecorder.waitonchange(checkinterval=2.0)
+    except KeyboardInterrupt:
+        print()
+
+
+class RemoteControl:
+    gateway: execnet.Gateway
+
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        self.failures: list[str] = []
+
+    def trace(self, *args: object) -> None:
+        if self.config.option.debug:
+            msg = " ".join(str(x) for x in args)
+            print("RemoteControl:", msg)
+
+    def initgateway(self) -> execnet.Gateway:
+        return execnet.makegateway("execmodel=main_thread_only//popen")
+
+    def setup(self) -> None:
+        if hasattr(self, "gateway"):
+            raise ValueError("already have gateway %r" % self.gateway)
+        self.trace("setting up worker session")
+        self.gateway = self.initgateway()
+        self.channel = channel = self.gateway.remote_exec(
+            init_worker_session,
+            args=self.config.args,
+            option_dict=vars(self.config.option),
+        )
+        remote_outchannel: execnet.Channel = channel.receive()
+
+        out = TerminalWriter()
+
+        def write(s: str) -> None:
+            out._file.write(s)
+            out._file.flush()
+
+        remote_outchannel.setcallback(write)
+
+    def ensure_teardown(self) -> None:
+        if hasattr(self, "channel"):
+            if not self.channel.isclosed():
+                self.trace("closing", self.channel)
+                self.channel.close()
+            del self.channel
+        if hasattr(self, "gateway"):
+            self.trace("exiting", self.gateway)
+            self.gateway.exit()
+            del self.gateway
+
+    def runsession(self) -> tuple[list[str], list[str], bool]:
+        try:
+            self.trace("sending", self.failures)
+            self.channel.send(self.failures)
+            try:
+                return self.channel.receive()  # type: ignore[no-any-return]
+            except self.channel.RemoteError:
+                e = sys.exc_info()[1]
+                self.trace("ERROR", e)
+                raise
+        finally:
+            self.ensure_teardown()
+
+    def loop_once(self) -> None:
+        self.setup()
+        self.wasfailing = self.failures and len(self.failures)
+        result = self.runsession()
+        failures, reports, collection_failed = result
+        if collection_failed:
+            pass  # "Collection failed, keeping previous failure set"
+        else:
+            uniq_failures = []
+            for failure in failures:
+                if failure not in uniq_failures:
+                    uniq_failures.append(failure)
+            self.failures = uniq_failures
+
+
+def repr_pytest_looponfailinfo(
+    failreports: Sequence[str], rootdirs: Sequence[Path]
+) -> None:
+    tr = TerminalWriter()
+    if failreports:
+        tr.sep("#", "LOOPONFAILING", bold=True)
+        for report in failreports:
+            if report:
+                tr.line(report, red=True)
+    tr.sep("#", "waiting for changes", bold=True)
+    for rootdir in rootdirs:
+        tr.line(f"### Watching:   {rootdir}", bold=True)
+
+
+def init_worker_session(
+    channel: "execnet.Channel",  # noqa: UP037
+    args: list[str],
+    option_dict: dict[str, "Any"],  # noqa: UP037
+) -> None:
+    import os
+    import sys
+
+    outchannel = channel.gateway.newchannel()
+    sys.stdout = sys.stderr = outchannel.makefile("w")
+    channel.send(outchannel)
+    # prune sys.path to not contain relative paths
+    newpaths = []
+    for p in sys.path:
+        if p:
+            # Ignore path placeholders created for editable installs
+            if not os.path.isabs(p) and not p.endswith(".__path_hook__"):
+                p = os.path.abspath(p)
+            newpaths.append(p)
+    sys.path[:] = newpaths
+
+    # fullwidth, hasmarkup = channel.receive()
+    from pytest import Config
+
+    config = Config.fromdictargs(option_dict, list(args))
+    config.args = args
+    from xdist.looponfail import WorkerFailSession
+
+    WorkerFailSession(config, channel).main()
+
+
+class WorkerFailSession:
+    def __init__(self, config: pytest.Config, channel: execnet.Channel) -> None:
+        self.config = config
+        self.channel = channel
+        self.recorded_failures: list[pytest.CollectReport | pytest.TestReport] = []
+        self.collection_failed = False
+        config.pluginmanager.register(self)
+        config.option.looponfail = False
+        config.option.usepdb = False
+
+    def DEBUG(self, *args: object) -> None:
+        if self.config.option.debug:
+            print(" ".join(map(str, args)))
+
+    @pytest.hookimpl
+    def pytest_collection(self, session: pytest.Session) -> bool:
+        self.session = session
+        self.trails = self.current_command
+        hook = self.session.ihook
+        try:
+            items = session.perform_collect(self.trails or None)
+        except pytest.UsageError:
+            items = session.perform_collect(None)
+        hook.pytest_collection_modifyitems(
+            session=session, config=session.config, items=items
+        )
+        hook.pytest_collection_finish(session=session)
+        return True
+
+    @pytest.hookimpl
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+        if report.failed:
+            self.recorded_failures.append(report)
+
+    @pytest.hookimpl
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+        if report.failed:
+            self.recorded_failures.append(report)
+            self.collection_failed = True
+
+    def main(self) -> None:
+        self.DEBUG("WORKER: received configuration, waiting for command trails")
+        try:
+            command = self.channel.receive()
+        except KeyboardInterrupt:
+            return  # in the worker we can't do much about this
+        self.DEBUG("received", command)
+        self.current_command = command
+        self.config.hook.pytest_cmdline_main(config=self.config)
+        trails, failreports = [], []
+        for rep in self.recorded_failures:
+            trails.append(rep.nodeid)
+            loc = rep.longrepr
+            loc = str(getattr(loc, "reprcrash", loc))
+            failreports.append(loc)
+        result = (trails, failreports, self.collection_failed)
+        self.channel.send(result)
+
+
+class StatRecorder:
+    def __init__(self, rootdirlist: Sequence[Path]) -> None:
+        self.rootdirlist = rootdirlist
+        self.statcache: dict[Path, os.stat_result] = {}
+        self.check()  # snapshot state
+
+    def fil(self, p: Path) -> bool:
+        return p.is_file() and not p.name.startswith(".") and p.suffix != ".pyc"
+
+    def rec(self, p: Path) -> bool:
+        return not p.name.startswith(".") and p.exists()
+
+    def waitonchange(self, checkinterval: float = 1.0) -> None:
+        while 1:
+            changed = self.check()
+            if changed:
+                return
+            time.sleep(checkinterval)
+
+    def check(self, removepycfiles: bool = True) -> bool:
+        changed = False
+        newstat: dict[Path, os.stat_result] = {}
+        for rootdir in self.rootdirlist:
+            for path in visit_path(rootdir, filter=self.fil, recurse=self.rec):
+                oldstat = self.statcache.pop(path, None)
+                try:
+                    curstat = path.stat()
+                except OSError:
+                    if oldstat:
+                        changed = True
+                else:
+                    newstat[path] = curstat
+                    if oldstat is not None:
+                        if (
+                            oldstat.st_mtime != curstat.st_mtime
+                            or oldstat.st_size != curstat.st_size
+                        ):
+                            changed = True
+                            print("# MODIFIED", path)
+                            if removepycfiles and path.suffix == ".py":
+                                pycfile = path.with_suffix(".pyc")
+                                if pycfile.is_file():
+                                    os.unlink(pycfile)
+
+                    else:
+                        changed = True
+        if self.statcache:
+            changed = True
+        self.statcache = newstat
+        return changed

src/xdist/newhooks.py

@@ -0,0 +1,129 @@
+"""
+xdist hooks.
+
+Additionally, pytest-xdist will also decorate a few other hooks
+with the worker instance that executed the hook originally:
+
+``pytest_runtest_logreport``: ``rep`` parameter has a ``node`` attribute.
+
+You can use this hooks just as you would use normal pytest hooks, but some care
+must be taken in plugins in case ``xdist`` is not installed. Please see:
+
+    http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import os
+from typing import Any
+from typing import TYPE_CHECKING
+
+import execnet
+import pytest
+
+
+if TYPE_CHECKING:
+    from xdist.remote import Producer
+    from xdist.scheduler.protocol import Scheduling
+    from xdist.workermanage import WorkerController
+
+
+@pytest.hookspec()
+def pytest_xdist_setupnodes(
+    config: pytest.Config, specs: Sequence[execnet.XSpec]
+) -> None:
+    """Called before any remote node is set up."""
+
+
+@pytest.hookspec()
+def pytest_xdist_newgateway(gateway: execnet.Gateway) -> None:
+    """Called on new raw gateway creation."""
+
+
+@pytest.hookspec(
+    warn_on_impl=DeprecationWarning(
+        "rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+    )
+)
+def pytest_xdist_rsyncstart(
+    source: str | os.PathLike[str],
+    gateways: Sequence[execnet.Gateway],
+) -> None:
+    """Called before rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(
+    warn_on_impl=DeprecationWarning(
+        "rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+    )
+)
+def pytest_xdist_rsyncfinish(
+    source: str | os.PathLike[str],
+    gateways: Sequence[execnet.Gateway],
+) -> None:
+    """Called after rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_getremotemodule() -> Any:
+    """Called when creating remote node."""
+
+
+@pytest.hookspec()
+def pytest_configure_node(node: WorkerController) -> None:
+    """Configure node information before it gets instantiated."""
+
+
+@pytest.hookspec()
+def pytest_testnodeready(node: WorkerController) -> None:
+    """Test Node is ready to operate."""
+
+
+@pytest.hookspec()
+def pytest_testnodedown(node: WorkerController, error: object | None) -> None:
+    """Test Node is down."""
+
+
+@pytest.hookspec()
+def pytest_xdist_node_collection_finished(
+    node: WorkerController, ids: Sequence[str]
+) -> None:
+    """Called by the controller node when a worker node finishes collecting."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_make_scheduler(
+    config: pytest.Config, log: Producer
+) -> Scheduling | None:
+    """Return a node scheduler implementation."""
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_xdist_auto_num_workers(config: pytest.Config) -> int:
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """
+    raise NotImplementedError()
+
+
+@pytest.hookspec(firstresult=True)
+def pytest_handlecrashitem(
+    crashitem: str, report: pytest.TestReport, sched: Scheduling
+) -> None:
+    """
+    Handle a crashitem, modifying the report if necessary.
+
+    The scheduler is provided as a parameter to reschedule the test if desired with
+    `sched.mark_test_pending`.
+
+    def pytest_handlecrashitem(crashitem, report, sched):
+        if should_rerun(crashitem):
+            sched.mark_test_pending(crashitem)
+            report.outcome = "rerun"
+
+    .. versionadded:: 2.2.1
+    """

src/xdist/plugin.py

@@ -0,0 +1,407 @@
+from __future__ import annotations
+
+import os
+import sys
+from typing import Literal
+import uuid
+import warnings
+
+import pytest
+
+
+_sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
+
+
+@pytest.hookimpl
+def pytest_xdist_auto_num_workers(config: pytest.Config) -> int:
+    env_var = os.environ.get("PYTEST_XDIST_AUTO_NUM_WORKERS")
+    if env_var:
+        try:
+            return int(env_var)
+        except ValueError:
+            warnings.warn(
+                "PYTEST_XDIST_AUTO_NUM_WORKERS is not a number: {env_var!r}. Ignoring it."
+            )
+
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        use_logical: bool = config.option.numprocesses == "logical"
+        count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
+        if count:
+            return count
+    try:
+        from os import sched_getaffinity
+
+        def cpu_count() -> int:
+            return len(sched_getaffinity(0))
+
+    except ImportError:
+        if os.environ.get("TRAVIS") == "true":
+            # workaround https://bitbucket.org/pypy/pypy/issues/2375
+            return 2
+        try:
+            from os import cpu_count  # type: ignore[assignment]
+        except ImportError:
+            from multiprocessing import cpu_count
+    try:
+        n = cpu_count()
+    except NotImplementedError:
+        return 1
+    return n if n else 1
+
+
+def parse_numprocesses(s: str) -> int | Literal["auto", "logical"]:
+    if s in ("auto", "logical"):
+        return s  # type: ignore[return-value]
+    elif s is not None:
+        return int(s)
+
+
+@pytest.hookimpl
+def pytest_addoption(parser: pytest.Parser) -> None:
+    # 'Help' formatting (same rules as pytest's):
+    # Start with capitalized letters.
+    # If a single phrase, do not end with period. If more than one phrase, all phrases end with periods.
+    # Use \n to separate logical lines.
+    group = parser.getgroup("xdist", "distributed and subprocess testing")
+    group._addoption(
+        "-n",
+        "--numprocesses",
+        dest="numprocesses",
+        metavar="numprocesses",
+        action="store",
+        type=parse_numprocesses,
+        help="Shortcut for '--dist=load --tx=NUM*popen'.\n"
+        "With 'logical', attempt to detect logical CPU count (requires psutil, falls back to 'auto').\n"
+        "With 'auto', attempt to detect physical CPU count. If physical CPU count cannot be determined, "
+        "falls back to 1.\n"
+        "Forced to 0 (disabled) when used with --pdb.",
+    )
+    group.addoption(
+        "--maxprocesses",
+        dest="maxprocesses",
+        metavar="maxprocesses",
+        action="store",
+        type=int,
+        help="Limit the maximum number of workers to process the tests when using --numprocesses "
+        "with 'auto' or 'logical'",
+    )
+    group.addoption(
+        "--max-worker-restart",
+        action="store",
+        default=None,
+        dest="maxworkerrestart",
+        help="Maximum number of workers that can be restarted "
+        "when crashed (set to zero to disable this feature)",
+    )
+    group.addoption(
+        "--dist",
+        metavar="distmode",
+        action="store",
+        choices=[
+            "each",
+            "load",
+            "loadscope",
+            "loadfile",
+            "loadgroup",
+            "worksteal",
+            "no",
+        ],
+        dest="dist",
+        default="no",
+        help=(
+            "Set mode for distributing tests to exec environments.\n\n"
+            "each: Send each test to all available environments.\n\n"
+            "load: Load balance by sending any pending test to any"
+            " available environment.\n\n"
+            "loadscope: Load balance by sending pending groups of tests in"
+            " the same scope to any available environment.\n\n"
+            "loadfile: Load balance by sending test grouped by file"
+            " to any available environment.\n\n"
+            "loadgroup: Like 'load', but sends tests marked with 'xdist_group' to the same worker.\n\n"
+            "worksteal: Split the test suite between available environments,"
+            " then re-balance when any worker runs out of tests.\n\n"
+            "(default) no: Run tests inprocess, don't distribute."
+        ),
+    )
+    group.addoption(
+        "--loadscope-reorder",
+        dest="loadscopereorder",
+        action="store_true",
+        default=True,
+        help=(
+            "Pytest-xdist will default reorder tests by number of tests per scope "
+            "when used in conjunction with loadscope.\n"
+            "This option will enable loadscope reorder which will improve the "
+            "parallelism of the test suite.\n"
+            "However, the partial order of tests might not be retained.\n"
+        ),
+    )
+    group.addoption(
+        "--no-loadscope-reorder",
+        dest="loadscopereorder",
+        action="store_false",
+        help=(
+            "Pytest-xdist will default reorder tests by number of tests per scope "
+            "when used in conjunction with loadscope.\n"
+            "This option will disable loadscope reorder, "
+            "and the partial order of tests can be retained.\n"
+            "This is useful when pytest-xdist is used together with "
+            "other plugins that specify tests in a specific order."
+        ),
+    )
+    group.addoption(
+        "--tx",
+        dest="tx",
+        action="append",
+        default=[],
+        metavar="xspec",
+        help=(
+            "Add a test execution environment. Some examples:\n"
+            "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888\n"
+            "--tx ssh=user@codespeak.net//chdir=testcache"
+        ),
+    )
+    group.addoption(
+        "--px",
+        dest="px",
+        action="append",
+        default=[],
+        metavar="xspec",
+        help=(
+            "Add a proxy gateway to pass to test execution environments using `via`. Example:\n"
+            "--px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy"
+        ),
+    )
+    group._addoption(
+        "-d",
+        action="store_true",
+        dest="distload",
+        default=False,
+        help="Load-balance tests. Shortcut for '--dist=load'.",
+    )
+    group.addoption(
+        "--rsyncdir",
+        action="append",
+        default=[],
+        metavar="DIR",
+        help="Add directory for rsyncing to remote tx nodes",
+    )
+    group.addoption(
+        "--rsyncignore",
+        action="append",
+        default=[],
+        metavar="GLOB",
+        help="Add expression for ignores when rsyncing to remote tx nodes",
+    )
+    group.addoption(
+        "--testrunuid",
+        action="store",
+        help=(
+            "Provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture.\n"
+            "If not provided, 'testrun_uid' is filled with a new unique string "
+            "on every test run."
+        ),
+    )
+    group.addoption(
+        "--maxschedchunk",
+        action="store",
+        type=int,
+        help=(
+            "Maximum number of tests scheduled in one step for --dist=load.\n"
+            "Setting it to 1 will force pytest to send tests to workers one by "
+            "one - might be useful for a small number of slow tests.\n"
+            "Larger numbers will allow the scheduler to submit consecutive "
+            "chunks of tests to workers - allows reusing fixtures.\n"
+            "Due to implementation reasons, at least 2 tests are scheduled per "
+            "worker at the start. Only later tests can be scheduled one by one.\n"
+            "Unlimited if not set."
+        ),
+    )
+
+    parser.addini(
+        "rsyncdirs",
+        "list of (relative) paths to be rsynced for remote distributed testing.",
+        type="paths",
+    )
+    parser.addini(
+        "rsyncignore",
+        "list of (relative) glob-style paths to be ignored for rsyncing.",
+        type="paths",
+    )
+    parser.addini(
+        "looponfailroots",
+        type="paths",
+        help="directories to check for changes. Default: current directory.",
+    )
+
+
+# -------------------------------------------------------------------------
+# distributed testing hooks
+# -------------------------------------------------------------------------
+
+
+@pytest.hookimpl
+def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None:
+    from xdist import newhooks
+
+    pluginmanager.add_hookspecs(newhooks)
+
+
+# -------------------------------------------------------------------------
+# distributed testing initialization
+# -------------------------------------------------------------------------
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_configure(config: pytest.Config) -> None:
+    config_line = (
+        "xdist_group: specify group for tests should run in same session."
+        "in relation to one another. Provided by pytest-xdist."
+    )
+    config.addinivalue_line("markers", config_line)
+
+    # Skip this plugin entirely when only doing collection.
+    if config.getvalue("collectonly"):
+        return
+
+    # Create the distributed session in case we have a valid distribution
+    # mode and test environments.
+    if _is_distribution_mode(config):
+        from xdist.dsession import DSession
+
+        session = DSession(config)
+        config.pluginmanager.register(session, "dsession")
+        tr = config.pluginmanager.getplugin("terminalreporter")
+        if tr:
+            tr.showfspath = False
+
+    # Deprecation warnings for deprecated command-line/configuration options.
+    if config.getoption("looponfail", None) or config.getini("looponfailroots"):
+        warning = DeprecationWarning(
+            "The --looponfail command line argument and looponfailroots config variable are deprecated.\n"
+            "The loop-on-fail feature will be removed in pytest-xdist 4.0."
+        )
+        config.issue_config_time_warning(warning, 2)
+
+    if config.getoption("rsyncdir", None) or config.getini("rsyncdirs"):
+        warning = DeprecationWarning(
+            "The --rsyncdir command line argument and rsyncdirs config variable are deprecated.\n"
+            "The rsync feature will be removed in pytest-xdist 4.0."
+        )
+        config.issue_config_time_warning(warning, 2)
+
+
+def _is_distribution_mode(config: pytest.Config) -> bool:
+    """Whether distribution mode is on."""
+    return config.getoption("dist") != "no" and bool(config.getoption("tx"))
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_cmdline_main(config: pytest.Config) -> None:
+    if config.option.distload:
+        config.option.dist = "load"
+
+    usepdb = config.getoption("usepdb", False)  # a core option
+    if config.option.numprocesses in ("auto", "logical"):
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
+
+    if config.option.numprocesses:
+        if config.option.dist == "no":
+            config.option.dist = "load"
+        numprocesses = config.option.numprocesses
+        if config.option.maxprocesses:
+            numprocesses = min(numprocesses, config.option.maxprocesses)
+        config.option.tx = ["popen"] * numprocesses
+
+    if config.option.numprocesses == 0:
+        config.option.dist = "no"
+        config.option.tx = []
+
+    val = config.getvalue
+    if not val("collectonly") and _is_distribution_mode(config) and usepdb:
+        raise pytest.UsageError(
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
+        )
+
+
+# -------------------------------------------------------------------------
+# fixtures and API to easily know the role of current node
+# -------------------------------------------------------------------------
+
+
+def is_xdist_worker(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_controller(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise.
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+# ALIAS: TODO, deprecate (#592)
+is_xdist_master = is_xdist_controller
+
+
+def get_xdist_worker_id(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> str:
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the controller node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        workerid: str = request_or_session.config.workerinput["workerid"]
+        return workerid
+    else:
+        # TODO: remove "master", ideally for a None
+        return "master"
+
+
+@pytest.fixture(scope="session")
+def worker_id(request: pytest.FixtureRequest) -> str:
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the master node.
+    """
+    # TODO: remove "master", ideally for a None
+    return get_xdist_worker_id(request)
+
+
+@pytest.fixture(scope="session")
+def testrun_uid(request: pytest.FixtureRequest) -> str:
+    """Return the unique id of the current test."""
+    if hasattr(request.config, "workerinput"):
+        testrunid: str = request.config.workerinput["testrunuid"]
+        return testrunid
+    else:
+        return uuid.uuid4().hex

src/xdist/remote.py

@@ -0,0 +1,427 @@
+"""
+This module is executed in remote subprocesses and helps to
+control a remote testing session and relay back information.
+It assumes that 'py' is importable and does not have dependencies
+on the rest of the xdist code.  This means that the xdist-plugin
+needs not to be installed in remote environments.
+"""
+
+from __future__ import annotations
+
+import collections
+from collections.abc import Generator
+from collections.abc import Iterable
+from collections.abc import Sequence
+import contextlib
+import enum
+import os
+import sys
+import time
+from typing import Any
+from typing import Literal
+from typing import TypedDict
+from typing import Union
+import warnings
+
+from _pytest.config import _prepareconfig
+import execnet
+import pytest
+
+
+try:
+    from setproctitle import setproctitle
+except ImportError:
+
+    def setproctitle(title: str) -> None:
+        pass
+
+
+class Producer:
+    """
+    Simplified implementation of the same interface as py.log, for backward compatibility
+    since we dropped the dependency on pylib.
+    Note: this is defined here because this module can't depend on xdist, so we need
+    to have the other way around.
+    """
+
+    def __init__(self, name: str, *, enabled: bool = True) -> None:
+        self.name = name
+        self.enabled = enabled
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r}, enabled={self.enabled})"
+
+    def __call__(self, *a: Any, **k: Any) -> None:
+        if self.enabled:
+            print(f"[{self.name}]", *a, **k, file=sys.stderr)
+
+    def __getattr__(self, name: str) -> Producer:
+        return type(self)(name, enabled=self.enabled)
+
+
+def worker_title(title: str) -> None:
+    try:
+        setproctitle(title)
+    except Exception:
+        # changing the process name is very optional, no errors please
+        pass
+
+
+class Marker(enum.Enum):
+    SHUTDOWN = 0
+
+
+class TestQueue:
+    """A simple queue that can be inspected and modified while the lock is held via the ``lock()`` method."""
+
+    Item = Union[int, Literal[Marker.SHUTDOWN]]
+
+    def __init__(self, execmodel: execnet.gateway_base.ExecModel):
+        self._items: collections.deque[TestQueue.Item] = collections.deque()
+        self._lock = execmodel.RLock()  # type: ignore[no-untyped-call]
+        self._has_items_event = execmodel.Event()
+
+    def get(self) -> Item:
+        while True:
+            with self.lock() as locked_items:
+                if locked_items:
+                    return locked_items.popleft()
+
+            self._has_items_event.wait()
+
+    def put(self, item: Item) -> None:
+        with self.lock() as locked_items:
+            locked_items.append(item)
+
+    def replace(self, iterable: Iterable[Item]) -> None:
+        with self.lock():
+            self._items = collections.deque(iterable)
+
+    @contextlib.contextmanager
+    def lock(self) -> Generator[collections.deque[Item]]:
+        with self._lock:
+            try:
+                yield self._items
+            finally:
+                if self._items:
+                    self._has_items_event.set()
+                else:
+                    self._has_items_event.clear()
+
+
+class WorkerInteractor:
+    def __init__(self, config: pytest.Config, channel: execnet.Channel) -> None:
+        self.config = config
+        workerinput: dict[str, Any] = config.workerinput  # type: ignore[attr-defined]
+        self.workerid = workerinput.get("workerid", "?")
+        self.testrunuid = workerinput["testrunuid"]
+        self.log = Producer(f"worker-{self.workerid}", enabled=config.option.debug)
+        self.channel = channel
+        self.torun = TestQueue(self.channel.gateway.execmodel)
+        self.nextitem_index: int | None | Literal[Marker.SHUTDOWN] = None
+        config.pluginmanager.register(self)
+
+    def sendevent(self, name: str, **kwargs: object) -> None:
+        self.log("sending", name, kwargs)
+        self.channel.send((name, kwargs))
+
+    @pytest.hookimpl
+    def pytest_internalerror(self, excrepr: object) -> None:
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
+            self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
+
+    @pytest.hookimpl
+    def pytest_sessionstart(self, session: pytest.Session) -> None:
+        self.session = session
+        workerinfo = getinfodict()
+        self.sendevent("workerready", workerinfo=workerinfo)
+
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_sessionfinish(self, exitstatus: int) -> Generator[None, object, None]:
+        workeroutput: dict[str, Any] = self.config.workeroutput  # type: ignore[attr-defined]
+        # in pytest 5.0+, exitstatus is an IntEnum object
+        workeroutput["exitstatus"] = int(exitstatus)
+        workeroutput["shouldfail"] = self.session.shouldfail
+        workeroutput["shouldstop"] = self.session.shouldstop
+        yield
+        self.sendevent("workerfinished", workeroutput=workeroutput)
+
+    @pytest.hookimpl
+    def pytest_collection(self) -> None:
+        self.sendevent("collectionstart")
+
+    def handle_command(
+        self, command: tuple[str, dict[str, Any]] | Literal[Marker.SHUTDOWN]
+    ) -> None:
+        if command is Marker.SHUTDOWN:
+            self.torun.put(Marker.SHUTDOWN)
+            return
+
+        name, kwargs = command
+
+        self.log("received command", name, kwargs)
+        if name == "runtests":
+            for i in kwargs["indices"]:
+                self.torun.put(i)
+        elif name == "runtests_all":
+            for i in range(len(self.session.items)):
+                self.torun.put(i)
+        elif name == "shutdown":
+            self.torun.put(Marker.SHUTDOWN)
+        elif name == "steal":
+            self.steal(kwargs["indices"])
+
+    def steal(self, indices: Sequence[int]) -> None:
+        """
+        Remove tests from the queue.
+
+        Removes either all requested tests, or none, if some of these tests
+        are not in the queue (for example, if they were processed already).
+
+        :param indices: indices of the tests to remove.
+        """
+        requested_set = set(indices)
+
+        with self.torun.lock() as locked_queue:
+            stolen = list(item for item in locked_queue if item in requested_set)
+
+            # Stealing only if all requested tests are still pending
+            if len(stolen) == len(requested_set):
+                self.torun.replace(
+                    item for item in locked_queue if item not in requested_set
+                )
+            else:
+                stolen = []
+
+        self.sendevent("unscheduled", indices=stolen)
+
+    @pytest.hookimpl
+    def pytest_runtestloop(self, session: pytest.Session) -> bool:
+        self.log("entering main loop")
+        self.channel.setcallback(self.handle_command, endmarker=Marker.SHUTDOWN)
+        self.nextitem_index = self.torun.get()
+        while self.nextitem_index is not Marker.SHUTDOWN:
+            self.run_one_test()
+            if session.shouldfail or session.shouldstop:
+                break
+        return True
+
+    def run_one_test(self) -> None:
+        assert isinstance(self.nextitem_index, int)
+        self.item_index = self.nextitem_index
+        self.nextitem_index = self.torun.get()
+
+        items = self.session.items
+        item = items[self.item_index]
+        if self.nextitem_index is Marker.SHUTDOWN:
+            nextitem = None
+        else:
+            assert self.nextitem_index is not None
+            nextitem = items[self.nextitem_index]
+
+        worker_title("[pytest-xdist running] %s" % item.nodeid)
+
+        start = time.perf_counter()
+        self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
+        duration = time.perf_counter() - start
+
+        worker_title("[pytest-xdist idle]")
+
+        self.sendevent(
+            "runtest_protocol_complete", item_index=self.item_index, duration=duration
+        )
+
+    def pytest_collection_modifyitems(
+        self,
+        config: pytest.Config,
+        items: list[pytest.Item],
+    ) -> None:
+        # add the group name to nodeid as suffix if --dist=loadgroup
+        if config.getvalue("loadgroup"):
+            for item in items:
+                gnames: set[str] = set()
+                for mark in item.iter_markers("xdist_group"):
+                    name = (
+                        mark.args[0]
+                        if len(mark.args) > 0
+                        else mark.kwargs.get("name", "default")
+                    )
+                    gnames.add(str(name))
+                if not gnames:
+                    continue
+                item._nodeid = f"{item.nodeid}@{'_'.join(sorted(gnames))}"
+
+    @pytest.hookimpl
+    def pytest_collection_finish(self, session: pytest.Session) -> None:
+        self.sendevent(
+            "collectionfinish",
+            topdir=str(self.config.rootpath),
+            ids=[item.nodeid for item in session.items],
+        )
+
+    @pytest.hookimpl
+    def pytest_runtest_logstart(
+        self,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
+        self.sendevent("logstart", nodeid=nodeid, location=location)
+
+    @pytest.hookimpl
+    def pytest_runtest_logfinish(
+        self,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
+        self.sendevent("logfinish", nodeid=nodeid, location=location)
+
+    @pytest.hookimpl
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+        data = self.config.hook.pytest_report_to_serializable(
+            config=self.config, report=report
+        )
+        data["item_index"] = self.item_index
+        data["worker_id"] = self.workerid
+        data["testrun_uid"] = self.testrunuid
+        assert self.session.items[self.item_index].nodeid == report.nodeid
+        self.sendevent("testreport", data=data)
+
+    @pytest.hookimpl
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+        # send only reports that have not passed to controller as optimization (#330)
+        if not report.passed:
+            data = self.config.hook.pytest_report_to_serializable(
+                config=self.config, report=report
+            )
+            self.sendevent("collectreport", data=data)
+
+    @pytest.hookimpl
+    def pytest_warning_recorded(
+        self,
+        warning_message: warnings.WarningMessage,
+        when: str,
+        nodeid: str,
+        location: tuple[str, int, str] | None,
+    ) -> None:
+        self.sendevent(
+            "warning_recorded",
+            warning_message_data=serialize_warning_message(warning_message),
+            when=when,
+            nodeid=nodeid,
+            location=location,
+        )
+
+
+def serialize_warning_message(
+    warning_message: warnings.WarningMessage,
+) -> dict[str, Any]:
+    if isinstance(warning_message.message, Warning):
+        message_module = type(warning_message.message).__module__
+        message_class_name = type(warning_message.message).__name__
+        message_str = str(warning_message.message)
+        # check now if we can serialize the warning arguments (#349)
+        # if not, we will just use the exception message on the controller node
+        try:
+            execnet.dumps(warning_message.message.args)
+        except execnet.DumpError:
+            message_args = None
+        else:
+            message_args = warning_message.message.args
+    else:
+        message_str = warning_message.message
+        message_module = None
+        message_class_name = None
+        message_args = None
+    if warning_message.category:
+        category_module = warning_message.category.__module__
+        category_class_name = warning_message.category.__name__
+    else:
+        category_module = None
+        category_class_name = None
+
+    result = {
+        "message_str": message_str,
+        "message_module": message_module,
+        "message_class_name": message_class_name,
+        "message_args": message_args,
+        "category_module": category_module,
+        "category_class_name": category_class_name,
+    }
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warning_message._WARNING_DETAILS:  # type: ignore[attr-defined]
+        if attr_name in ("message", "category"):
+            continue
+        attr = getattr(warning_message, attr_name)
+        # Check if we can serialize the warning detail, marking `None` otherwise
+        # Note that we need to define the attr (even as `None`) to allow deserializing
+        try:
+            execnet.dumps(attr)
+        except execnet.DumpError:
+            result[attr_name] = repr(attr)
+        else:
+            result[attr_name] = attr
+    return result
+
+
+class WorkerInfo(TypedDict):
+    version: str
+    version_info: tuple[int, int, int, str, int]
+    sysplatform: str
+    platform: str
+    executable: str
+    cwd: str
+    id: str
+    spec: execnet.XSpec
+
+
+def getinfodict() -> WorkerInfo:
+    import platform
+
+    return dict(
+        version=sys.version,
+        version_info=tuple(sys.version_info),  # type: ignore[typeddict-item]
+        sysplatform=sys.platform,
+        platform=platform.platform(),
+        executable=sys.executable,
+        cwd=os.getcwd(),
+    )
+
+
+def setup_config(config: pytest.Config, basetemp: str | None) -> None:
+    config.option.loadgroup = config.getvalue("dist") == "loadgroup"
+    config.option.looponfail = False
+    config.option.usepdb = False
+    config.option.dist = "no"
+    config.option.distload = False
+    config.option.numprocesses = None
+    config.option.maxprocesses = None
+    config.option.basetemp = basetemp
+
+
+if __name__ == "__channelexec__":
+    channel: execnet.Channel = channel  # type: ignore[name-defined] # noqa: F821, PLW0127
+    workerinput, args, option_dict, change_sys_path = channel.receive()  # type: ignore[name-defined]
+
+    if change_sys_path is None:
+        importpath = os.getcwd()
+        sys.path.insert(0, importpath)
+        os.environ["PYTHONPATH"] = (
+            importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
+        )
+    else:
+        sys.path = change_sys_path
+
+    os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
+    os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
+    os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
+
+    config = _prepareconfig(args, None)
+
+    setup_config(config, option_dict.get("basetemp"))
+    config._parser.prog = os.path.basename(workerinput["mainargv"][0])
+    config.workerinput = workerinput  # type: ignore[attr-defined]
+    config.workeroutput = {}  # type: ignore[attr-defined]
+    interactor = WorkerInteractor(config, channel)  # type: ignore[name-defined]
+    config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from difflib import unified_diff
+
+
+def report_collection_diff(
+    from_collection: Sequence[str],
+    to_collection: Sequence[str],
+    from_id: str,
+    to_id: str,
+) -> str | None:
+    """Report the collected test difference between two nodes.
+
+    :returns: detailed message describing the difference between the given
+    collections, or None if they are equal.
+    """
+    if from_collection == to_collection:
+        return None
+
+    diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
+    error_message = (
+        "Different tests were collected between {from_id} and {to_id}. "
+        "The difference is:\n"
+        "{diff}\n"
+        "To see why this happens see 'Known limitations' in documentation "
+        "for pytest-xdist"
+    ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
+    msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
+    return msg

src/xdist/scheduler/__init__.py

@@ -0,0 +1,7 @@
+from xdist.scheduler.each import EachScheduling as EachScheduling
+from xdist.scheduler.load import LoadScheduling as LoadScheduling
+from xdist.scheduler.loadfile import LoadFileScheduling as LoadFileScheduling
+from xdist.scheduler.loadgroup import LoadGroupScheduling as LoadGroupScheduling
+from xdist.scheduler.loadscope import LoadScopeScheduling as LoadScopeScheduling
+from xdist.scheduler.protocol import Scheduling as Scheduling
+from xdist.scheduler.worksteal import WorkStealingScheduling as WorkStealingScheduling

src/xdist/scheduler/each.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import pytest
+
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
+
+
+class EachScheduling:
+    """Implement scheduling of test items on all nodes.
+
+    If a node gets added after the test run is started then it is
+    assumed to replace a node which got removed before it finished
+    its collection.  In this case it will only be used if a node
+    with the same spec got removed earlier.
+
+    Any nodes added after the run is started will only get items
+    assigned if a node with a matching spec was removed before it
+    finished all its pending items.  The new node will then be
+    assigned the remaining items from the removed node.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.config = config
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self._started: list[WorkerController] = []
+        self._removed2pending: dict[WorkerController, list[int]] = {}
+        if log is None:
+            self.log = Producer("eachsched")
+        else:
+            self.log = log.eachsched
+        self.collection_is_completed = False
+
+    @property
+    def nodes(self) -> list[WorkerController]:
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def tests_finished(self) -> bool:
+        if not self.collection_is_completed:
+            return False
+        if self._removed2pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node: WorkerController) -> None:
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
+
+        Collection is complete once all nodes have submitted their
+        collection.  In this case its pending list is set to an empty
+        list.  When the collection is already completed this
+        submission is from a node which was restarted to replace a
+        dead node.  In this case we already assign the pending items
+        here.  In either case ``.schedule()`` will instruct the
+        node to start running the required tests.
+        """
+        assert node in self.node2pending
+        if not self.collection_is_completed:
+            self.node2collection[node] = list(collection)
+            self.node2pending[node] = []
+            if len(self.node2collection) >= self.numnodes:
+                self.collection_is_completed = True
+        elif self._removed2pending:
+            for deadnode in self._removed2pending:
+                if deadnode.gateway.spec == node.gateway.spec:
+                    dead_collection = self.node2collection[deadnode]
+                    if collection != dead_collection:
+                        msg = report_collection_diff(
+                            dead_collection,
+                            collection,
+                            deadnode.gateway.id,
+                            node.gateway.id,
+                        )
+                        self.log(msg)
+                        return
+                    pending = self._removed2pending.pop(deadnode)
+                    self.node2pending[node] = pending
+                    break
+
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
+        self.node2pending[node].remove(item_index)
+
+    def mark_test_pending(self, item: str) -> None:
+        raise NotImplementedError()
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def remove_node(self, node: WorkerController) -> str | None:
+        # KeyError if we didn't get an add_node() yet
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return None
+        crashitem = self.node2collection[node][pending.pop(0)]
+        if pending:
+            self._removed2pending[node] = pending
+        return crashitem
+
+    def schedule(self) -> None:
+        """Schedule the test items on the nodes.
+
+        If the node's pending list is empty it is a new node which
+        needs to run all the tests.  If the pending list is already
+        populated (by ``.add_node_collection()``) then it replaces a
+        dead node and we only need to run those tests.
+        """
+        assert self.collection_is_completed
+        for node, pending in self.node2pending.items():
+            if node in self._started:
+                continue
+            if not pending:
+                pending[:] = range(len(self.node2collection[node]))
+                node.send_runtest_all()
+                node.shutdown()
+            else:
+                node.send_runtest_some(pending)
+            self._started.append(node)

src/xdist/scheduler/load.py

@@ -0,0 +1,335 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from itertools import cycle
+
+import pytest
+
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
+
+
+class LoadScheduling:
+    """Implement load scheduling across nodes.
+
+    This distributes the tests collected across all nodes so each test
+    is run just once.  All nodes collect and submit the test suite and
+    when all collections are received it is verified they are
+    identical collections.  Then the collection gets divided up in
+    chunks and chunks get submitted to nodes.  Whenever a node finishes
+    an item, it calls ``.mark_test_complete()`` which will trigger the
+    scheduler to assign more tests if the number of pending tests for
+    the node falls below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to
+    submit a collection. This is used to know when all nodes have
+    finished collection or how large the chunks need to be created.
+
+    Attributes::
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self.pending: list[int] = []
+        self.collection: list[str] | None = None
+        if log is None:
+            self.log = Producer("loadsched")
+        else:
+            self.log = log.loadsched
+        self.config = config
+        self.maxschedchunk = self.config.getoption("maxschedchunk")
+
+    @property
+    def nodes(self) -> list[WorkerController]:
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self) -> bool:
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self) -> bool:
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node: WorkerController) -> None:
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
+        """Mark test item as completed by node.
+
+        The duration it took to execute the item is used as a hint to
+        the scheduler.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule(node, duration=duration)
+
+    def mark_test_pending(self, item: str) -> None:
+        assert self.collection is not None
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        for node in self.node2pending:
+            self.check_schedule(node)
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def check_schedule(self, node: WorkerController, duration: float = 0) -> None:
+        """Maybe schedule new items on the node.
+
+        If there are any globally pending nodes left then this will
+        check if the given node should be given any more tests.  The
+        ``duration`` of the last test is optionally used as a
+        heuristic to influence how many tests the node is assigned.
+        """
+        if node.shutting_down:
+            return
+
+        if self.pending:
+            # how many nodes do we have?
+            num_nodes = len(self.node2pending)
+            # if our node goes below a heuristic minimum, fill it out to
+            # heuristic maximum
+            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
+            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
+            node_pending = self.node2pending[node]
+            if len(node_pending) < items_per_node_min:
+                if duration >= 0.1 and len(node_pending) >= 2:
+                    # seems the node is doing long-running tests
+                    # and has enough items to continue
+                    # so let's rather wait with sending new items
+                    return
+                num_send = items_per_node_max - len(node_pending)
+                # keep at least 2 tests pending even if --maxschedchunk=1
+                maxschedchunk = max(2 - len(node_pending), self.maxschedchunk)
+                self._send_tests(node, min(num_send, maxschedchunk))
+        else:
+            node.shutdown()
+
+        self.log("num items waiting for node:", len(self.pending))
+
+    def remove_node(self, node: WorkerController) -> str | None:
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+
+        """
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return None
+
+        # The node crashed, reassing pending items
+        assert self.collection is not None
+        crashitem = self.collection[pending.pop(0)]
+        self.pending.extend(pending)
+        for node in self.node2pending:
+            self.check_schedule(node)
+        return crashitem
+
+    def schedule(self) -> None:
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self.check_schedule(node)
+            return
+
+        # XXX allow nodes to have different collections
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = next(iter(self.node2collection.values()))
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        if self.maxschedchunk is None:
+            self.maxschedchunk = len(self.collection)
+
+        # Send a batch of tests to run. If we don't have at least two
+        # tests per node, we have to send them all so that we can send
+        # shutdown signals and get all nodes working.
+        if len(self.pending) < 2 * len(self.nodes):
+            # Distribute tests round-robin. Try to load all nodes if there are
+            # enough tests. The other branch tries sends at least 2 tests
+            # to each node - which is suboptimal when you have less than
+            # 2 * len(nodes) tests.
+            nodes = cycle(self.nodes)
+            for _ in range(len(self.pending)):
+                self._send_tests(next(nodes), 1)
+        else:
+            # Send batches of consecutive tests. By default, pytest sorts tests
+            # in order for optimal single-threaded execution, minimizing the
+            # number of necessary fixture setup/teardown. Try to keep that
+            # optimal order for every worker.
+
+            # how many items per node do we have about?
+            items_per_node = len(self.collection) // len(self.node2pending)
+            # take a fraction of tests for initial distribution
+            node_chunksize = min(items_per_node // 4, self.maxschedchunk)
+            node_chunksize = max(node_chunksize, 2)
+            # and initialize each node with a chunk of tests
+            for node in self.nodes:
+                self._send_tests(node, node_chunksize)
+
+        if not self.pending:
+            # initial distribution sent all tests, start node shutdown
+            for node in self.nodes:
+                node.shutdown()
+
+    def _send_tests(self, node: WorkerController, num: int) -> None:
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self) -> bool:
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = pytest.CollectReport(
+                        nodeid=node.gateway.id,
+                        outcome="failed",
+                        longrepr=msg,
+                        result=[],
+                    )
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/scheduler/loadfile.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import pytest
+
+from xdist.remote import Producer
+
+from .loadscope import LoadScopeScheduling
+
+
+class LoadFileScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test test file.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test file,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    This class behaves very much like LoadScopeScheduling, but with a file-level scope.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadfilesched")
+        else:
+            self.log = log.loadfilesched
+
+    def _split_scope(self, nodeid: str) -> str:
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the left. That is, test will be grouped in a
+        single work unit when they reside in the same file.
+
+        In the above example, scopes will be::
+
+        .. code-block:: text
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.split("::", 1)[0]

src/xdist/scheduler/loadgroup.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pytest
+
+from xdist.remote import Producer
+
+from .loadscope import LoadScopeScheduling
+
+
+class LoadGroupScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test by xdist_group mark.
+
+    This class behaves very much like LoadScopeScheduling, but it groups tests by xdist_group mark
+    instead of the module or class to which they belong to.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        super().__init__(config, log)
+        if log is None:
+            self.log = Producer("loadgroupsched")
+        else:
+            self.log = log.loadgroupsched
+
+    def _split_scope(self, nodeid: str) -> str:
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        With loadgroup, two cases are added::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            example/loadsuite/test/test_gamma.py::test_beta0@gname
+            example/loadsuite/test/test_delta.py::Gamma1::test_gamma0@gname
+
+        This function will group tests with the scope determined by splitting the first ``@``
+        from the right. That is, test will be grouped in a single work unit when they have
+        same group name. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+            gname
+            gname
+        """
+        if nodeid.rfind("@") > nodeid.rfind("]"):
+            # check the index of ']' to avoid the case: parametrize mark value has '@'
+            return nodeid.split("@")[-1]
+        else:
+            return nodeid

src/xdist/scheduler/loadscope.py

@@ -0,0 +1,438 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+from collections.abc import Sequence
+from typing import NoReturn
+
+import pytest
+
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
+
+
+class LoadScopeScheduling:
+    """Implement load scheduling across nodes, but grouping test by scope.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test scope,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    Attributes::
+
+    :numnodes: The expected number of nodes taking part.  The actual number of
+       nodes will vary during the scheduler's lifetime as nodes are added by
+       the DSession as they are brought up and removed either because of a dead
+       node or normal shutdown.  This number is primarily used to know when the
+       initial collection is completed.
+
+    :collection: The final list of tests collected by all nodes once it is
+       validated to be identical between all the nodes.  It is initialised to
+       None until ``.schedule()`` is called.
+
+    :workqueue: Ordered dictionary that maps all available scopes with their
+       associated tests (nodeid). Nodeids are in turn associated with their
+       completion status. One entry of the workqueue is called a work unit.
+       In turn, a collection of work unit is called a workload.
+
+       ::
+
+            workqueue = {
+                '<full>/<path>/<to>/test_module.py': {
+                    '<full>/<path>/<to>/test_module.py::test_case1': False,
+                    '<full>/<path>/<to>/test_module.py::test_case2': False,
+                    (...)
+                },
+                (...)
+            }
+
+    :assigned_work: Ordered dictionary that maps worker nodes with their
+       assigned work units.
+
+       ::
+
+            assigned_work = {
+                '<worker node A>': {
+                    '<full>/<path>/<to>/test_module.py': {
+                        '<full>/<path>/<to>/test_module.py::test_case1': False,
+                        '<full>/<path>/<to>/test_module.py::test_case2': False,
+                        (...)
+                    },
+                    (...)
+                },
+                (...)
+            }
+
+    :registered_collections: Ordered dictionary that maps worker nodes with
+       their collection of tests gathered during test discovery.
+
+       ::
+
+            registered_collections = {
+                '<worker node A>': [
+                    '<full>/<path>/<to>/test_module.py::test_case1',
+                    '<full>/<path>/<to>/test_module.py::test_case2',
+                ],
+                (...)
+            }
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.collection: list[str] | None = None
+
+        self.workqueue: OrderedDict[str, dict[str, bool]] = OrderedDict()
+        self.assigned_work: dict[WorkerController, dict[str, dict[str, bool]]] = {}
+        self.registered_collections: dict[WorkerController, list[str]] = {}
+
+        if log is None:
+            self.log = Producer("loadscopesched")
+        else:
+            self.log = log.loadscopesched
+
+        self.config = config
+
+    @property
+    def nodes(self) -> list[WorkerController]:
+        """A list of all active nodes in the scheduler."""
+        return list(self.assigned_work.keys())
+
+    @property
+    def collection_is_completed(self) -> bool:
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes have
+        finished collection.  The required number of initial nodes is defined
+        by ``.numnodes``.
+        """
+        return len(self.registered_collections) >= self.numnodes
+
+    @property
+    def tests_finished(self) -> bool:
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+
+        if self.workqueue:
+            return False
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) >= 2:
+                return False
+
+        return True
+
+    @property
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are still
+        processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.workqueue:
+            return True
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) > 0:
+                return True
+
+        return False
+
+    def add_node(self, node: WorkerController) -> None:
+        """Add a new node to the scheduler.
+
+        From now on the node will be assigned work units to be executed.
+
+        Called by the ``DSession.worker_workerready`` hook when it successfully
+        bootstraps a new node.
+        """
+        assert node not in self.assigned_work
+        self.assigned_work[node] = {}
+
+    def remove_node(self, node: WorkerController) -> str | None:
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at shutdown time.
+        In the former case any pending items assigned to the node will be
+        re-scheduled.
+
+        Called by the hooks:
+
+        - ``DSession.worker_workerfinished``.
+        - ``DSession.worker_errordown``.
+
+        Return the item being executed while the node crashed or None if the
+        node has no more pending items.
+        """
+        workload = self.assigned_work.pop(node)
+        if not self._pending_of(workload):
+            return None
+
+        # The node crashed, identify test that crashed
+        for work_unit in workload.values():
+            for nodeid, completed in work_unit.items():
+                if not completed:
+                    crashitem = nodeid
+                    break
+            else:
+                continue
+            break
+        else:
+            raise RuntimeError(
+                "Unable to identify crashitem on a workload with pending items"
+            )
+
+        # Made uncompleted work unit available again
+        self.workqueue.update(workload)
+
+        for node in self.assigned_work:
+            self._reschedule(node)
+
+        return crashitem
+
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.registered_collections`` dictionary.
+
+        Called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+        # Check that add_node() was called on the node before
+        assert node in self.assigned_work
+
+        # A new node has been added later, perhaps an original one died.
+        if self.collection_is_completed:
+            # Assert that .schedule() should have been called by now
+            assert self.collection
+
+            # Check that the new collection matches the official collection
+            if collection != self.collection:
+                other_node = next(iter(self.registered_collections.keys()))
+
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+
+        self.registered_collections[node] = list(collection)
+
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
+        """Mark test item as completed by node.
+
+        Called by the hook:
+
+        - ``DSession.worker_testreport``.
+        """
+        nodeid = self.registered_collections[node][item_index]
+        scope = self._split_scope(nodeid)
+
+        self.assigned_work[node][scope][nodeid] = True
+        self._reschedule(node)
+
+    def mark_test_pending(self, item: str) -> NoReturn:
+        raise NotImplementedError()
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def _assign_work_unit(self, node: WorkerController) -> None:
+        """Assign a work unit to a node."""
+        assert self.workqueue
+
+        # Grab a unit of work
+        scope, work_unit = self.workqueue.popitem(last=False)
+
+        # Keep track of the assigned work
+        assigned_to_node = self.assigned_work.setdefault(node, {})
+        assigned_to_node[scope] = work_unit
+
+        # Ask the node to execute the workload
+        worker_collection = self.registered_collections[node]
+        nodeids_indexes = [
+            worker_collection.index(nodeid)
+            for nodeid, completed in work_unit.items()
+            if not completed
+        ]
+
+        node.send_runtest_some(nodeids_indexes)
+
+    def _split_scope(self, nodeid: str) -> str:
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the right. That is, classes will be grouped in a
+        single work unit, and functions from a test module will be grouped by
+        their module. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py::Delta1
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.rsplit("::", 1)[0]
+
+    def _pending_of(self, workload: dict[str, dict[str, bool]]) -> int:
+        """Return the number of pending tests in a workload."""
+        pending = sum(list(scope.values()).count(False) for scope in workload.values())
+        return pending
+
+    def _reschedule(self, node: WorkerController) -> None:
+        """Maybe schedule new items on the node.
+
+        If there are any globally pending work units left then this will check
+        if the given node should be given any more tests.
+        """
+        # Do not add more work to a node shutting down
+        if node.shutting_down:
+            return
+
+        # Check that more work is available
+        if not self.workqueue:
+            node.shutdown()
+            return
+
+        self.log("Number of units waiting for node:", len(self.workqueue))
+
+        # Check that the node is almost depleted of work
+        # 2: Heuristic of minimum tests to enqueue more work
+        if self._pending_of(self.assigned_work[node]) > 2:
+            return
+
+        # Pop one unit of work and assign it
+        self._assign_work_unit(node)
+
+    def schedule(self) -> None:
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this gets called
+        again later it behaves the same as calling ``._reschedule()`` on all
+        nodes so that newly added nodes will start to be used.
+
+        If ``.collection_is_completed`` is True, this is called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self._reschedule(node)
+            return
+
+        # Check that all nodes collected the same tests
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the final list of items
+        self.collection = list(next(iter(self.registered_collections.values())))
+        if not self.collection:
+            return
+
+        # Determine chunks of work (scopes)
+        unsorted_workqueue: dict[str, dict[str, bool]] = {}
+        for nodeid in self.collection:
+            scope = self._split_scope(nodeid)
+            work_unit = unsorted_workqueue.setdefault(scope, {})
+            work_unit[nodeid] = False
+
+        if self.config.option.loadscopereorder:
+            # Insert tests scopes into work queue ordered by number of tests.
+            for scope, nodeids in sorted(
+                unsorted_workqueue.items(), key=lambda item: -len(item[1])
+            ):
+                self.workqueue[scope] = nodeids
+        else:
+            for scope, nodeids in unsorted_workqueue.items():
+                self.workqueue[scope] = nodeids
+
+        # Avoid having more workers than work
+        extra_nodes = len(self.nodes) - len(self.workqueue)
+
+        if extra_nodes > 0:
+            self.log(f"Shutting down {extra_nodes} nodes")
+
+            for _ in range(extra_nodes):
+                unused_node, assigned = self.assigned_work.popitem()
+
+                self.log(f"Shutting down unused node {unused_node}")
+                unused_node.shutdown()
+
+        # Assign initial workload
+        for node in self.nodes:
+            self._assign_work_unit(node)
+
+        # Ensure nodes start with at least two work units if possible (#277)
+        for node in self.nodes:
+            self._reschedule(node)
+
+        # Initial distribution sent all tests, start node shutdown
+        if not self.workqueue:
+            for node in self.nodes:
+                node.shutdown()
+
+    def _check_nodes_have_same_collection(self) -> bool:
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.registered_collections.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if not msg:
+                continue
+
+            same_collection = False
+            self.log(msg)
+
+            rep = pytest.CollectReport(
+                nodeid=node.gateway.id,
+                outcome="failed",
+                longrepr=msg,
+                result=[],
+            )
+            self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/scheduler/protocol.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Protocol
+
+from xdist.workermanage import WorkerController
+
+
+class Scheduling(Protocol):
+    @property
+    def nodes(self) -> list[WorkerController]: ...
+
+    @property
+    def collection_is_completed(self) -> bool: ...
+
+    @property
+    def tests_finished(self) -> bool: ...
+
+    @property
+    def has_pending(self) -> bool: ...
+
+    def add_node(self, node: WorkerController) -> None: ...
+
+    def add_node_collection(
+        self,
+        node: WorkerController,
+        collection: Sequence[str],
+    ) -> None: ...
+
+    def mark_test_complete(
+        self,
+        node: WorkerController,
+        item_index: int,
+        duration: float = 0,
+    ) -> None: ...
+
+    def mark_test_pending(self, item: str) -> None: ...
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None: ...
+
+    def remove_node(self, node: WorkerController) -> str | None: ...
+
+    def schedule(self) -> None: ...

src/xdist/scheduler/worksteal.py

@@ -0,0 +1,345 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import NamedTuple
+
+import pytest
+
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
+
+
+class NodePending(NamedTuple):
+    node: WorkerController
+    pending: list[int]
+
+
+# Every worker needs at least 2 tests in queue - the current and the next one.
+MIN_PENDING = 2
+
+
+class WorkStealingScheduling:
+    """Implement work-stealing scheduling.
+
+    Initially, tests are distributed evenly among all nodes.
+
+    When some node completes most of its assigned tests (when only one pending
+    test remains), an attempt is made to reassign ("steal") some tests from
+    other nodes to this node.
+
+    Attributes::
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+
+    :steal_requested_from_node: The node to which the current "steal" request
+       was sent. ``None`` if there is no request in progress. Only one request
+       can be in progress at any time, the scheduler doesn't send multiple
+       simultaneous requests.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self.pending: list[int] = []
+        self.collection: list[str] | None = None
+        if log is None:
+            self.log = Producer("workstealsched")
+        else:
+            self.log = log.workstealsched
+        self.config = config
+        self.steal_requested_from_node: WorkerController | None = None
+
+    @property
+    def nodes(self) -> list[WorkerController]:
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self) -> bool:
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self) -> bool:
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        if self.steal_requested_from_node is not None:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= MIN_PENDING:
+                return False
+        return True
+
+    @property
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node: WorkerController) -> None:
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float | None = None
+    ) -> None:
+        """Mark test item as completed by node.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule()
+
+    def mark_test_pending(self, item: str) -> None:
+        assert self.collection is not None
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        self.check_schedule()
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        """Node returned some test indices back in response to 'steal' command.
+
+        This is called by ``DSession.worker_unscheduled``.
+        """
+        assert node is self.steal_requested_from_node
+        self.steal_requested_from_node = None
+
+        indices_set = set(indices)
+        self.node2pending[node] = [
+            i for i in self.node2pending[node] if i not in indices_set
+        ]
+        self.pending.extend(indices)
+        self.check_schedule()
+
+    def check_schedule(self) -> None:
+        """Reschedule tests/perform load balancing."""
+        nodes_up = [
+            NodePending(node, pending)
+            for node, pending in self.node2pending.items()
+            if not node.shutting_down
+        ]
+
+        def get_idle_nodes() -> list[WorkerController]:
+            return [node for node, pending in nodes_up if len(pending) < MIN_PENDING]
+
+        idle_nodes = get_idle_nodes()
+        if not idle_nodes:
+            return
+
+        if self.pending:
+            # Distribute pending tests evenly among idle nodes
+            for i, node in enumerate(idle_nodes):
+                nodes_remaining = len(idle_nodes) - i
+                num_send = len(self.pending) // nodes_remaining
+                self._send_tests(node, num_send)
+
+            idle_nodes = get_idle_nodes()
+            # No need to steal anything if all nodes have enough work to continue
+            if not idle_nodes:
+                return
+
+        # Only one active stealing request is allowed
+        if self.steal_requested_from_node is not None:
+            return
+
+        # Find the node that has the longest test queue
+        steal_from = max(
+            nodes_up, key=lambda node_pending: len(node_pending.pending), default=None
+        )
+
+        if steal_from is None:
+            num_steal = 0
+        else:
+            # Steal half of the test queue - but keep that node running too.
+            # If the node has 2 or less tests queued, stealing will fail
+            # anyway.
+            max_steal = max(0, len(steal_from.pending) - MIN_PENDING)
+            num_steal = min(len(steal_from.pending) // 2, max_steal)
+
+        if num_steal == 0:
+            # Can't get more work - shutdown idle nodes. This will force them
+            # to run the last test now instead of waiting for more tests.
+            for node in idle_nodes:
+                node.shutdown()
+            return
+
+        assert steal_from is not None
+        steal_from.node.send_steal(steal_from.pending[-num_steal:])
+        self.steal_requested_from_node = steal_from.node
+
+    def remove_node(self, node: WorkerController) -> str | None:
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+        """
+        pending = self.node2pending.pop(node)
+
+        # If node was removed without completing its assigned tests - it crashed
+        if pending:
+            assert self.collection is not None
+            crashitem = self.collection[pending.pop(0)]
+        else:
+            crashitem = None
+
+        self.pending.extend(pending)
+
+        # Dead node won't respond to "steal" request
+        if self.steal_requested_from_node is node:
+            self.steal_requested_from_node = None
+
+        self.check_schedule()
+        return crashitem
+
+    def schedule(self) -> None:
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            self.check_schedule()
+            return
+
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = next(iter(self.node2collection.values()))
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        self.check_schedule()
+
+    def _send_tests(self, node: WorkerController, num: int) -> None:
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self) -> bool:
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = pytest.CollectReport(
+                        nodeid=node.gateway.id,
+                        outcome="failed",
+                        longrepr=msg,
+                        result=[],
+                    )
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/workermanage.py

@@ -0,0 +1,517 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+import enum
+import fnmatch
+import os
+from pathlib import Path
+import re
+import sys
+from typing import Any
+from typing import Callable
+from typing import Literal
+from typing import Union
+import uuid
+import warnings
+
+import execnet
+import pytest
+
+from xdist.plugin import _sys_path
+import xdist.remote
+from xdist.remote import Producer
+from xdist.remote import WorkerInfo
+
+
+def parse_tx_spec_config(config: pytest.Config) -> list[str]:
+    xspeclist = []
+    tx: list[str] = config.getvalue("tx")
+    for xspec in tx:
+        i = xspec.find("*")
+        try:
+            num = int(xspec[:i])
+        except ValueError:
+            xspeclist.append(xspec)
+        else:
+            xspeclist.extend([xspec[i + 1 :]] * num)
+    if not xspeclist:
+        raise pytest.UsageError(
+            "MISSING test execution (tx) nodes: please specify --tx"
+        )
+    return xspeclist
+
+
+class NodeManager:
+    EXIT_TIMEOUT = 10
+    DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
+
+    def __init__(
+        self,
+        config: pytest.Config,
+        specs: Sequence[execnet.XSpec | str] | None = None,
+        defaultchdir: str = "pyexecnetcache",
+    ) -> None:
+        self.config = config
+        self.trace = self.config.trace.get("nodemanager")
+        self.testrunuid = self.config.getoption("testrunuid")
+        if self.testrunuid is None:
+            self.testrunuid = uuid.uuid4().hex
+        self.group = execnet.Group(execmodel="main_thread_only")
+        for proxy_spec in self._getpxspecs():
+            # Proxy gateways do not run workers, and are meant to be passed with the `via` attribute
+            # to additional gateways.
+            # They are useful for running multiple workers on remote machines.
+            if getattr(proxy_spec, "id", None) is None:
+                raise pytest.UsageError(
+                    f"Proxy gateway {proxy_spec} must include an id"
+                )
+            self.group.makegateway(proxy_spec)
+        if specs is None:
+            specs = self._gettxspecs()
+        self.specs: list[execnet.XSpec] = []
+        for spec in specs:
+            if not isinstance(spec, execnet.XSpec):
+                spec = execnet.XSpec(spec)
+            if getattr(spec, "execmodel", None) != "main_thread_only":
+                spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
+            if not spec.chdir and not spec.popen:
+                spec.chdir = defaultchdir
+            self.group.allocate_id(spec)
+            self.specs.append(spec)
+        self.roots = self._getrsyncdirs()
+        self.rsyncoptions = self._getrsyncoptions()
+        self._rsynced_specs: set[tuple[Any, Any]] = set()
+
+    def rsync_roots(self, gateway: execnet.Gateway) -> None:
+        """Rsync the set of roots to the node's gateway cwd."""
+        if self.roots:
+            for root in self.roots:
+                self.rsync(gateway, root, **self.rsyncoptions)
+
+    def setup_nodes(
+        self,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+    ) -> list[WorkerController]:
+        self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
+        self.trace("setting up nodes")
+        with ThreadPoolExecutor(max_workers=len(self.specs)) as executor:
+            futs = [
+                executor.submit(self.setup_node, spec, putevent, idx)
+                for idx, spec in enumerate(self.specs)
+            ]
+            return [f.result() for f in futs]
+
+    def setup_node(
+        self,
+        spec: execnet.XSpec,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+        idx: int | None = None,
+    ) -> WorkerController:
+        if getattr(spec, "execmodel", None) != "main_thread_only":
+            spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
+        if idx is not None:
+            spec = execnet.XSpec(f"{spec}//id=gw{idx}")
+        gw = self.group.makegateway(spec)
+        self.config.hook.pytest_xdist_newgateway(gateway=gw)
+        self.rsync_roots(gw)
+        node = WorkerController(self, gw, self.config, putevent)
+        # Keep the node alive.
+        gw.node = node  # type: ignore[attr-defined]
+        node.setup()
+        self.trace("started node %r" % node)
+        return node
+
+    def teardown_nodes(self) -> None:
+        self.group.terminate(self.EXIT_TIMEOUT)
+
+    def _gettxspecs(self) -> list[execnet.XSpec]:
+        return [execnet.XSpec(x) for x in parse_tx_spec_config(self.config)]
+
+    def _getpxspecs(self) -> list[execnet.XSpec]:
+        return [execnet.XSpec(x) for x in self.config.getoption("px")]
+
+    def _getrsyncdirs(self) -> list[Path]:
+        for spec in self.specs:
+            if not spec.popen or spec.chdir:
+                break
+        else:
+            return []
+        import _pytest
+        import pytest
+
+        def get_dir(p: str) -> str:
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
+        config = self.config
+        candidates = [pytestpath, pytestdir]
+        candidates += config.option.rsyncdir
+        rsyncroots = config.getini("rsyncdirs")
+        if rsyncroots:
+            candidates.extend(rsyncroots)
+        roots = []
+        for root in candidates:
+            root_path = Path(root).resolve()
+            if not root_path.exists():
+                raise pytest.UsageError(f"rsyncdir doesn't exist: {root!r}")
+            if root_path not in roots:
+                roots.append(root_path)
+        return roots
+
+    def _getrsyncoptions(self) -> dict[str, Any]:
+        """Get options to be passed for rsync."""
+        ignores = list(self.DEFAULT_IGNORES)
+        ignores += [str(path) for path in self.config.option.rsyncignore]
+        ignores += [str(path) for path in self.config.getini("rsyncignore")]
+
+        return {
+            "ignores": ignores,
+            "verbose": getattr(self.config.option, "verbose", 0),
+        }
+
+    def rsync(
+        self,
+        gateway: execnet.Gateway,
+        source: str | os.PathLike[str],
+        notify: (
+            Callable[[str, execnet.XSpec, str | os.PathLike[str]], Any] | None
+        ) = None,
+        verbose: int = False,
+        ignores: Sequence[str] | None = None,
+    ) -> None:
+        """Perform rsync to remote hosts for node."""
+        # XXX This changes the calling behaviour of
+        #     pytest_xdist_rsyncstart and pytest_xdist_rsyncfinish to
+        #     be called once per rsync target.
+        rsync = HostRSync(source, verbose=verbose > 0, ignores=ignores)
+        spec = gateway.spec
+        if spec.popen and not spec.chdir:
+            # XXX This assumes that sources are python-packages
+            #     and that adding the basedir does not hurt.
+            gateway.remote_exec(
+                """
+                import sys ; sys.path.insert(0, %r)
+            """
+                % os.path.dirname(str(source))
+            ).waitclose()
+            return
+        if (spec, source) in self._rsynced_specs:
+            return
+
+        def finished() -> None:
+            if notify:
+                notify("rsyncrootready", spec, source)
+
+        rsync.add_target_host(gateway, finished=finished)
+        self._rsynced_specs.add((spec, source))
+        self.config.hook.pytest_xdist_rsyncstart(source=source, gateways=[gateway])
+        rsync.send()
+        self.config.hook.pytest_xdist_rsyncfinish(source=source, gateways=[gateway])
+
+
+class HostRSync(execnet.RSync):
+    """RSyncer that filters out common files."""
+
+    PathLike = Union[str, "os.PathLike[str]"]
+
+    def __init__(
+        self,
+        sourcedir: PathLike,
+        *,
+        ignores: Sequence[PathLike] | None = None,
+        verbose: bool = True,
+    ) -> None:
+        if ignores is None:
+            ignores = []
+        self._ignores = [re.compile(fnmatch.translate(os.fspath(x))) for x in ignores]
+        super().__init__(sourcedir=Path(sourcedir), verbose=verbose)
+
+    def filter(self, path: PathLike) -> bool:
+        path = Path(path)
+        for cre in self._ignores:
+            if cre.match(path.name) or cre.match(str(path)):
+                return False
+        else:
+            return True
+
+    def add_target_host(
+        self,
+        gateway: execnet.Gateway,
+        finished: Callable[[], None] | None = None,
+    ) -> None:
+        remotepath = os.path.basename(self._sourcedir)
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
+
+    def _report_send_file(
+        self,
+        gateway: execnet.Gateway,  # type: ignore[override]
+        modified_rel_path: str,
+    ) -> None:
+        if self._verbose > 0:
+            path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
+            remotepath = gateway.spec.chdir
+            print(f"{gateway.spec}:{remotepath} <= {path}")
+
+
+def make_reltoroot(roots: Sequence[Path], args: list[str]) -> list[str]:
+    # XXX introduce/use public API for splitting pytest args
+    splitcode = "::"
+    result = []
+    for arg in args:
+        parts = arg.split(splitcode)
+        fspath = Path(parts[0])
+        try:
+            exists = fspath.exists()
+        except OSError:
+            exists = False
+        if not exists:
+            result.append(arg)
+            continue
+        for root in roots:
+            x: Path | None
+            try:
+                x = fspath.relative_to(root)
+            except ValueError:
+                x = None
+            if x or fspath == root:
+                parts[0] = root.name + "/" + str(x)
+                break
+        else:
+            raise ValueError(f"arg {arg} not relative to an rsync root")
+        result.append(splitcode.join(parts))
+    return result
+
+
+class Marker(enum.Enum):
+    END = -1
+
+
+class WorkerController:
+    # Set when the worker is ready.
+    workerinfo: WorkerInfo
+
+    class RemoteHook:
+        @pytest.hookimpl(trylast=True)
+        def pytest_xdist_getremotemodule(self) -> Any:
+            return xdist.remote
+
+    def __init__(
+        self,
+        nodemanager: NodeManager,
+        gateway: execnet.Gateway,
+        config: pytest.Config,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+    ) -> None:
+        config.pluginmanager.register(self.RemoteHook())
+        self.nodemanager = nodemanager
+        self.putevent = putevent
+        self.gateway = gateway
+        self.config = config
+        self.workerinput = {
+            "workerid": gateway.id,
+            "workercount": len(nodemanager.specs),
+            "testrunuid": nodemanager.testrunuid,
+            "mainargv": sys.argv,
+        }
+        self._down = False
+        self._shutdown_sent = False
+        self.log = Producer(f"workerctl-{gateway.id}", enabled=config.option.debug)
+
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.gateway.id}>"
+
+    @property
+    def shutting_down(self) -> bool:
+        return self._down or self._shutdown_sent
+
+    def setup(self) -> None:
+        self.log("setting up worker session")
+        # Cache rinfo for backward compatibility, since pytest-cov
+        # accesses rinfo while the main thread is busy executing our
+        # remote_exec call, which triggers a deadlock error for the
+        # main_thread_only execmodel if the rinfo has not been cached.
+        self.gateway._rinfo()
+        spec = self.gateway.spec
+        args = [str(x) for x in self.config.invocation_params.args or ()]
+        option_dict = {}
+        if not spec.popen or spec.chdir:
+            args = make_reltoroot(self.nodemanager.roots, args)
+        if spec.popen:
+            name = "popen-%s" % self.gateway.id
+            if hasattr(self.config, "_tmp_path_factory"):
+                basetemp = self.config._tmp_path_factory.getbasetemp()
+                option_dict["basetemp"] = str(basetemp / name)
+        self.config.hook.pytest_configure_node(node=self)
+
+        remote_module = self.config.hook.pytest_xdist_getremotemodule()
+        self.channel = self.gateway.remote_exec(remote_module)
+        # change sys.path only for remote workers
+        # restore sys.path from a frozen copy for local workers
+        change_sys_path = _sys_path if self.gateway.spec.popen else None
+        self.channel.send((self.workerinput, args, option_dict, change_sys_path))
+
+        # putevent is only None in a test.
+        if self.putevent:  # type: ignore[truthy-function]
+            self.channel.setcallback(self.process_from_remote, endmarker=Marker.END)
+
+    def ensure_teardown(self) -> None:
+        if hasattr(self, "channel"):
+            if not self.channel.isclosed():
+                self.log("closing", self.channel)
+                self.channel.close()
+            # del self.channel
+        if hasattr(self, "gateway"):
+            self.log("exiting", self.gateway)
+            self.gateway.exit()
+            # del self.gateway
+
+    def send_runtest_some(self, indices: Sequence[int]) -> None:
+        self.sendcommand("runtests", indices=indices)
+
+    def send_runtest_all(self) -> None:
+        self.sendcommand("runtests_all")
+
+    def send_steal(self, indices: Sequence[int]) -> None:
+        self.sendcommand("steal", indices=indices)
+
+    def shutdown(self) -> None:
+        if not self._down:
+            try:
+                self.sendcommand("shutdown")
+            except OSError:
+                pass
+            self._shutdown_sent = True
+
+    def sendcommand(self, name: str, **kwargs: object) -> None:
+        """Send a named parametrized command to the other side."""
+        self.log(f"sending command {name}(**{kwargs})")
+        self.channel.send((name, kwargs))
+
+    def notify_inproc(self, eventname: str, **kwargs: object) -> None:
+        self.log(f"queuing {eventname}(**{kwargs})")
+        self.putevent((eventname, kwargs))
+
+    def process_from_remote(
+        self, eventcall: tuple[str, dict[str, Any]] | Literal[Marker.END]
+    ) -> None:
+        """This gets called for each object we receive from
+        the other side and if the channel closes.
+
+        Note that channel callbacks run in the receiver
+        thread of execnet gateways - we need to
+        avoid raising exceptions or doing heavy work.
+        """
+        try:
+            if eventcall is Marker.END:
+                err: object | None = self.channel._getremoteerror()  # type: ignore[no-untyped-call]
+                if not self._down:
+                    if not err or isinstance(err, EOFError):
+                        err = "Not properly terminated"  # lost connection?
+                    self.notify_inproc("errordown", node=self, error=err)
+                    self._down = True
+                return
+            eventname, kwargs = eventcall
+            if eventname in ("collectionstart",):
+                self.log(f"ignoring {eventname}({kwargs})")
+            elif eventname == "workerready":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "workerfinished":
+                self._down = True
+                self.workeroutput = kwargs["workeroutput"]
+                self.notify_inproc("workerfinished", node=self)
+            elif eventname in ("logstart", "logfinish"):
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname in ("testreport", "collectreport", "teardownreport"):
+                item_index = kwargs.pop("item_index", None)
+                rep = self.config.hook.pytest_report_from_serializable(
+                    config=self.config, data=kwargs["data"]
+                )
+                if item_index is not None:
+                    rep.item_index = item_index
+                self.notify_inproc(eventname, node=self, rep=rep)
+            elif eventname == "collectionfinish":
+                self.notify_inproc(eventname, node=self, ids=kwargs["ids"])
+            elif eventname == "runtest_protocol_complete":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "unscheduled":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "logwarning":
+                self.notify_inproc(
+                    eventname,
+                    message=kwargs["message"],
+                    code=kwargs["code"],
+                    nodeid=kwargs["nodeid"],
+                    fslocation=kwargs["nodeid"],
+                )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
+            else:
+                raise ValueError(f"unknown event: {eventname}")
+        except KeyboardInterrupt:
+            # should not land in receiver-thread
+            raise
+        except BaseException:
+            excinfo = pytest.ExceptionInfo.from_current()
+            print("!" * 20, excinfo)
+            self.config.notify_exception(excinfo)
+            self.shutdown()
+            self.notify_inproc("errordown", node=self, error=excinfo)
+
+
+def unserialize_warning_message(data: dict[str, Any]) -> warnings.WarningMessage:
+    import importlib
+
+    if data["message_module"]:
+        mod = importlib.import_module(data["message_module"])
+        cls = getattr(mod, data["message_class_name"])
+        message = None
+        if data["message_args"] is not None:
+            try:
+                message = cls(*data["message_args"])
+            except TypeError:
+                pass
+        if message is None:
+            # could not recreate the original warning instance;
+            # create a generic Warning instance with the original
+            # message at least
+            message_text = "{mod}.{cls}: {msg}".format(
+                mod=data["message_module"],
+                cls=data["message_class_name"],
+                msg=data["message_str"],
+            )
+            message = Warning(message_text)
+    else:
+        message = data["message_str"]
+
+    if data["category_module"]:
+        mod = importlib.import_module(data["category_module"])
+        category = getattr(mod, data["category_class_name"])
+    else:
+        category = None
+
+    kwargs = {"message": message, "category": category}
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:  # type: ignore[attr-defined]
+        if attr_name in ("message", "category"):
+            continue
+        kwargs[attr_name] = data[attr_name]
+
+    return warnings.WarningMessage(**kwargs)

testing/conftest.py

@@ -1,39 +1,68 @@
-import py
+from __future__ import annotations
+
+from collections.abc import Generator
+import shutil
+from typing import Callable
+
 import execnet
+import pytest
+
 
 pytest_plugins = "pytester"
 
-#rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
 
-def pytest_addoption(parser):
-    parser.addoption('--gx',
-       action="append", dest="gspecs", default=None,
-       help=("add a global test environment, XSpec-syntax. "))
+@pytest.fixture(autouse=True)
+def _divert_atexit(monkeypatch: pytest.MonkeyPatch) -> Generator[None]:
+    import atexit
 
-def pytest_funcarg__specssh(request):
+    finalizers = []
+
+    def fake_register(
+        func: Callable[..., object], *args: object, **kwargs: object
+    ) -> None:
+        finalizers.append((func, args, kwargs))
+
+    monkeypatch.setattr(atexit, "register", fake_register)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--gx",
+        action="append",
+        dest="gspecs",
+        help="add a global test environment, XSpec-syntax. ",
+    )
+
+
+@pytest.fixture
+def specssh(request: pytest.FixtureRequest) -> str:
     return getspecssh(request.config)
-def getgspecs(config):
-    return [execnet.XSpec(spec)
-                for spec in config.getvalueorskip("gspecs")]
+
 
 # configuration information for tests
-def getgspecs(config):
-    return [execnet.XSpec(spec)
-                for spec in config.getvalueorskip("gspecs")]
+def getgspecs(config: pytest.Config) -> list[execnet.XSpec]:
+    return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
 
-def getspecssh(config):
+
+def getspecssh(config: pytest.Config) -> str:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.ssh:
-            if not py.path.local.sysfind("ssh"):
-                py.test.skip("command not found: ssh")
+            if not shutil.which("ssh"):
+                pytest.skip("command not found: ssh")
             return str(spec)
-    py.test.skip("need '--gx ssh=...'")
+    pytest.skip("need '--gx ssh=...'")
 
-def getsocketspec(config):
+
+def getsocketspec(config: pytest.Config) -> execnet.XSpec:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.socket:
             return spec
-    py.test.skip("need '--gx socket=...'")
-
+    pytest.skip("need '--gx socket=...'")

testing/test_boxed.py

@@ -1,28 +0,0 @@
-import py
-
-@py.test.mark.skipif("not hasattr(os, 'fork')")
-def test_functional_boxed(testdir):
-    p1 = testdir.makepyfile("""
-        import os
-        def test_function():
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed")
-    result.stdout.fnmatch_lines([
-        "*CRASHED*",
-        "*1 failed*"
-    ])
-
-class TestOptionEffects:
-    def test_boxed_option_default(self, testdir):
-        tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
-        config = testdir.parseconfig()
-        assert not config.option.boxed
-        py.test.importorskip("execnet")
-        config = testdir.parseconfig('-d', tmpdir)
-        assert not config.option.boxed
-
-    def test_is_not_boxed_by_default(self, testdir):
-        config = testdir.parseconfig(testdir.tmpdir)
-        assert not config.option.boxed
-

testing/test_dsession.py

@@ -1,166 +1,634 @@
-from xdist.dsession import DSession, LoadScheduling, EachScheduling
-from _pytest import main as outcome
-import py
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+from typing import cast
+from typing import TYPE_CHECKING
+
 import execnet
+import pytest
 
-XSpec = execnet.XSpec
+from xdist.dsession import DSession
+from xdist.dsession import get_default_max_worker_restart
+from xdist.dsession import get_workers_status_line
+from xdist.dsession import WorkerStatus
+from xdist.report import report_collection_diff
+from xdist.scheduler import EachScheduling
+from xdist.scheduler import LoadScheduling
+from xdist.scheduler import WorkStealingScheduling
+from xdist.workermanage import WorkerController
 
-def run(item, node, excinfo=None):
-    runner = item.config.pluginmanager.getplugin("runner")
-    rep = runner.ItemTestReport(item=item,
-        excinfo=excinfo, when="call")
-    rep.node = node
-    return rep
 
-class MockNode:
-    def __init__(self):
-        self.sent = []
+if TYPE_CHECKING:
+    BaseOfMockGateway = execnet.Gateway
+    BaseOfMockNode = WorkerController
+else:
+    BaseOfMockGateway = object
+    BaseOfMockNode = object
 
-    def send_runtest(self, nodeid):
-        self.sent.append(nodeid)
 
-    def send_runtest_all(self):
+class MockGateway(BaseOfMockGateway):
+    def __init__(self) -> None:
+        self._count = 0
+        self.id = str(self._count)
+        self._count += 1
+
+
+class MockNode(BaseOfMockNode):
+    def __init__(self) -> None:
+        self.sent: list[int | str] = []
+        self.stolen: list[int] = []
+        self.gateway = MockGateway()
+        self._shutdown = False
+
+    def send_runtest_some(self, indices: Sequence[int]) -> None:
+        self.sent.extend(indices)
+
+    def send_runtest_all(self) -> None:
         self.sent.append("ALL")
 
-    def sendlist(self, items):
-        self.sent.extend(items)
+    def send_steal(self, indices: Sequence[int]) -> None:
+        self.stolen.extend(indices)
 
-    def shutdown(self):
-        self._shutdown=True
+    def shutdown(self) -> None:
+        self._shutdown = True
+
+    @property
+    def shutting_down(self) -> bool:
+        return self._shutdown
 
-def dumpqueue(queue):
-    while queue.qsize():
-        print(queue.get())
 
 class TestEachScheduling:
-    def test_schedule_load_simple(self):
-        node1 = MockNode()
-        node2 = MockNode()
-        sched = EachScheduling(2)
-        sched.addnode(node1)
-        sched.addnode(node2)
-        collection = ["a.py::test_1", ]
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = EachScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = ["a.py::test_1"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
-        assert sched.collection_is_completed
+        sched.add_node_collection(node2, collection)
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
-        assert not sched.tests_finished()
-        assert node1.sent == ['ALL']
-        assert node2.sent == ['ALL']
-        sched.remove_item(node1, collection[0])
-        assert not sched.tests_finished()
-        sched.remove_item(node2, collection[0])
-        assert sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
+        assert node1.sent == ["ALL"]
+        assert node2.sent == ["ALL"]
+        sched.mark_test_complete(node1, 0)
+        assert sched.tests_finished
+        sched.mark_test_complete(node2, 0)
+        assert sched.tests_finished
 
-    def test_schedule_remove_node(self):
+    def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=popen")
+        sched = EachScheduling(config)
         node1 = MockNode()
-        sched = EachScheduling(1)
-        sched.addnode(node1)
-        collection = ["a.py::test_1", ]
+        sched.add_node(node1)
+        collection = ["a.py::test_1"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
-        assert sched.collection_is_completed
+        sched.add_node_collection(node1, collection)
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
-        sched.init_distribute()
-        assert not sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
         crashitem = sched.remove_node(node1)
         assert crashitem
-        assert sched.tests_finished()
-        assert not sched.hasnodes()
+        assert sched.tests_finished
+        assert not sched.nodes
+
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self):
-        node1 = MockNode()
-        node2 = MockNode()
-        sched = LoadScheduling(2)
-        sched.addnode(node1)
-        sched.addnode(node2)
+    def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
         collection = ["a.py::test_1", "a.py::test_2"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
-        assert sched.collection_is_completed
+        sched.add_node_collection(node2, collection)
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
-        assert not sched.tests_finished()
+        sched.schedule()
+        assert not sched.pending
+        assert sched.tests_finished
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
-        x = sorted(node1.sent + node2.sent)
-        assert x == collection
-        sched.remove_item(node1, node1.sent[0])
-        sched.remove_item(node2, node2.sent[0])
-        assert sched.tests_finished()
-        assert not sched.pending
+        assert node1.sent == [0]
+        assert node2.sent == [1]
+        sent10 = node1.sent[0]
+        assert isinstance(sent10, int)
+        sched.mark_test_complete(node1, sent10)
+        assert sched.tests_finished
 
-    def test_init_distribute_chunksize(self):
-        sched = LoadScheduling(2)
-        node1 = MockNode()
-        node2 = MockNode()
-        sched.addnode(node1)
-        sched.addnode(node2)
-        sched.ITEM_CHUNKSIZE = 2
-        col = ["xyz"] * (2*sched.ITEM_CHUNKSIZE +1)
-        sched.addnode_collection(node1, col)
-        sched.addnode_collection(node2, col)
-        sched.init_distribute()
+    def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        col = ["xyz"] * 6
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
-        chunkitems = col[:sched.ITEM_CHUNKSIZE]
-        assert sent1 == chunkitems
-        assert sent2 == chunkitems
+        assert sent1 == [0, 1]
+        assert sent2 == [2, 3]
+        assert sched.pending == [4, 5]
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
-        assert len(sched.pending) == 1
-        for node in (node1, node2):
-            for i in range(sched.ITEM_CHUNKSIZE):
-                sched.remove_item(node, "xyz")
+        assert len(sched.pending) == 2
+        sched.mark_test_complete(node1, 0)
+        assert node1.sent == [0, 1, 4]
+        assert sched.pending == [5]
+        assert node2.sent == [2, 3]
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
         assert not sched.pending
 
-    def test_add_remove_node(self):
-        node = MockNode()
-        sched = LoadScheduling(1)
-        sched.addnode(node)
-        collection = ["test_file.py::test_func"]
-        sched.addnode_collection(node, collection)
+    def test_schedule_maxchunk_none(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        col = [f"test{i}" for i in range(16)]
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+        sched.mark_test_complete(node1, 0)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+
+        for i in range(7, 16):
+            sched.mark_test_complete(node1, i - 3)
+            assert node1.sent == [0, 1, *range(4, i)]
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(i, 16))
+
+    def test_schedule_maxchunk_1(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen", "--maxschedchunk=1")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        col = [f"test{i}" for i in range(16)]
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+
+        for complete_index, first_pending in enumerate(range(5, 16)):
+            sent_index = node1.sent[complete_index]
+            assert isinstance(sent_index, int)
+            sched.mark_test_complete(node1, sent_index)
+            assert node1.sent == [0, 1, *range(4, first_pending)]
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(first_pending, 16))
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = LoadScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
         assert sched.collection_is_completed
-        sched.init_distribute()
+        sched.schedule()
+        # assert not sched.tests_finished
+        assert node1.sent == [0]
+        assert node2.sent == [1]
+        assert node3.sent == []
+        assert not sched.pending
+
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = LoadScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 5
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        assert sched.collection_is_completed
+        sched.schedule()
+        # assert not sched.tests_finished
+        assert node1.sent == [0, 3]
+        assert node2.sent == [1, 4]
+        assert node3.sent == [2]
+        assert not sched.pending
+
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=popen")
+        sched = LoadScheduling(config)
+        node = MockNode()
+        sched.add_node(node)
+        collection = ["test_file.py::test_func"]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
         assert not sched.pending
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
 
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
+        """
+        Test that LoadScheduling is reporting collection errors when
+        different test ids are collected by workers.
+        """
+
+        class CollectHook:
+            """Dummy hook that stores collection reports."""
+
+            def __init__(self) -> None:
+                self.reports: list[pytest.CollectReport] = []
+
+            def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = pytester.parseconfig("--tx=2*popen")
+        config.pluginmanager.register(collect_hook, "collect_hook")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert isinstance(rep.longrepr, str)
+        assert "Different tests were collected between" in rep.longrepr
+
+
+class TestWorkStealingScheduling:
+    def test_ideal_case(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node1, collection)
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node2, collection)
+        assert bool(sched.collection_is_completed)
+        assert sched.node2collection[node1] == collection
+        assert sched.node2collection[node2] == collection
+        sched.schedule()
+        assert not sched.pending
+        assert not sched.tests_finished
+        assert node1.sent == list(range(8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sent1, sent2 = node1.sent[i], node2.sent[i]
+            assert isinstance(sent1, int) and isinstance(sent2, int)
+            sched.mark_test_complete(node1, sent1)
+            sched.mark_test_complete(node2, sent2)
+        assert bool(sched.tests_finished)
+        assert node1.stolen == []
+        assert node2.stolen == []
+
+    def test_stealing(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        sched.add_node_collection(node1, collection)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node1.sent == list(range(8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sent = node1.sent[i]
+            assert isinstance(sent, int)
+            sched.mark_test_complete(node1, sent)
+        assert node2.stolen == list(range(12, 16))
+        sched.remove_pending_tests_from_node(node2, node2.stolen)
+        for i in range(4):
+            sent = node2.sent[i]
+            assert isinstance(sent, int)
+            sched.mark_test_complete(node2, sent)
+        assert node1.stolen == [14, 15]
+        sched.remove_pending_tests_from_node(node1, node1.stolen)
+        sched.mark_test_complete(node1, 12)
+        sched.mark_test_complete(node2, 14)
+        assert node2.stolen == list(range(12, 16))
+        assert node1.stolen == [14, 15]
+        assert sched.tests_finished
+
+    def test_steal_on_add_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(5)]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        sched.mark_test_complete(node, 0)
+        node2 = MockNode()
+        sched.add_node(node2)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node.stolen == [3, 4]
+        sched.remove_pending_tests_from_node(node, node.stolen)
+        sched.mark_test_complete(node, 1)
+        sched.mark_test_complete(node2, 3)
+        assert sched.tests_finished
+        assert node2.stolen == []
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == []
+        assert node1.stolen == []
+        assert node2.sent == [0]
+        assert node2.stolen == []
+        assert node3.sent == [1]
+        assert node3.stolen == []
+        assert not sched.pending
+        assert sched.tests_finished
+
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 5
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == [0]
+        assert node2.sent == [1, 2]
+        assert node3.sent == [3, 4]
+        assert not sched.pending
+        assert not sched.tests_finished
+        sent10 = node1.sent[0]
+        assert isinstance(sent10, int)
+        sent20 = node2.sent[0]
+        assert isinstance(sent20, int)
+        sent30 = node3.sent[0]
+        assert isinstance(sent30, int)
+        sent31 = node3.sent[1]
+        assert isinstance(sent31, int)
+        sched.mark_test_complete(node1, sent10)
+        sched.mark_test_complete(node2, sent20)
+        sched.mark_test_complete(node3, sent30)
+        sched.mark_test_complete(node3, sent31)
+        assert bool(sched.tests_finished)
+        assert node1.stolen == []
+        assert node2.stolen == []
+        assert node3.stolen == []
+
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = ["test_file.py::test_func"]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        crashitem = sched.remove_node(node)
+        assert crashitem == collection[0]
+
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
+        class CollectHook:
+            def __init__(self) -> None:
+                self.reports: list[pytest.CollectReport] = []
+
+            def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = pytester.parseconfig("--tx=2*popen")
+        config.pluginmanager.register(collect_hook, "collect_hook")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert isinstance(rep.longrepr, str)
+        assert "Different tests were collected between" in rep.longrepr
+
 
 class TestDistReporter:
+    @pytest.mark.xfail
+    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp: Any) -> None:
+        config = pytester.parseconfig()
+        from _pytest.terminal import TerminalReporter
 
-    @py.test.mark.xfail
-    def test_rsync_printing(self, testdir, linecomp):
-        config = testdir.parseconfig()
-        from _pytest.pytest_terminal import TerminalReporter
         rep = TerminalReporter(config, file=linecomp.stringio)
         config.pluginmanager.register(rep, "terminalreporter")
         dsession = DSession(config)
+
         class gw1:
             id = "X1"
             spec = execnet.XSpec("popen")
+
         class gw2:
             id = "X2"
             spec = execnet.XSpec("popen")
-        #class rinfo:
+
+        # class rinfo:
         #    version_info = (2, 5, 1, 'final', 0)
         #    executable = "hello"
         #    platform = "xyz"
         #    cwd = "qwe"
 
-        #dsession.pytest_xdist_newgateway(gw1, rinfo)
-        #linecomp.assert_contains_lines([
-        #    "*X1*popen*xyz*2.5*"
-        #])
-        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
-        linecomp.assert_contains_lines([
-            "[X1,X2] rsyncing: hello",
-        ])
+        # dsession.pytest_xdist_newgateway(gw1, rinfo)
+        # linecomp.assert_contains_lines([
+        #     "*X1*popen*xyz*2.5*"
+        # ])
+        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])  # type: ignore[attr-defined]
+        linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
+
+
+def test_report_collection_diff_equal() -> None:
+    """Test reporting of equal collections."""
+    from_collection = to_collection = ["aaa", "bbb", "ccc"]
+    assert report_collection_diff(from_collection, to_collection, "1", "2") is None
+
+
+def test_default_max_worker_restart() -> None:
+    class MockConfig:
+        class option:
+            maxworkerrestart: str | None = None
+            numprocesses: int = 0
+
+    config = cast(pytest.Config, MockConfig)
+
+    assert get_default_max_worker_restart(config) is None
+
+    config.option.numprocesses = 2
+    assert get_default_max_worker_restart(config) == 8
+
+    config.option.maxworkerrestart = "1"
+    assert get_default_max_worker_restart(config) == 1
+
+    config.option.maxworkerrestart = "0"
+    assert get_default_max_worker_restart(config) == 0
+
+
+def test_report_collection_diff_different() -> None:
+    """Test reporting of different collections."""
+    from_collection = ["aaa", "bbb", "ccc", "YYY"]
+    to_collection = ["aZa", "bbb", "XXX", "ccc"]
+    error_message = (
+        "Different tests were collected between 1 and 2. The difference is:\n"
+        "--- 1\n"
+        "\n"
+        "+++ 2\n"
+        "\n"
+        "@@ -1,4 +1,4 @@\n"
+        "\n"
+        "-aaa\n"
+        "+aZa\n"
+        " bbb\n"
+        "+XXX\n"
+        " ccc\n"
+        "-YYY\n"
+        "To see why this happens see 'Known limitations' in documentation "
+        "for pytest-xdist"
+    )
+
+    msg = report_collection_diff(from_collection, to_collection, "1", "2")
+    assert msg == error_message
+
+
+@pytest.mark.xfail(reason="duplicate test ids not supported yet")
+def test_pytest_issue419(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.parametrize('birth_year', [1988, 1988, ])
+        def test_2011_table(birth_year):
+            pass
+    """
+    )
+    reprec = pytester.inline_run("-n1")
+    reprec.assertoutcome(passed=2)
+    assert 0
+
+
+Created = WorkerStatus.Created
+Initialized = WorkerStatus.Initialized
+ReadyForCollection = WorkerStatus.ReadyForCollection
+CollectionDone = WorkerStatus.CollectionDone
+
+
+@pytest.mark.parametrize(
+    "status_and_items, expected",
+    [
+        (
+            [],
+            "",
+        ),
+        (
+            [(Created, 0)],
+            "created: 1/1 worker",
+        ),
+        (
+            [(Created, 0), (Created, 0)],
+            "created: 2/2 workers",
+        ),
+        (
+            [(Initialized, 0), (Created, 0)],
+            "initialized: 1/2 workers",
+        ),
+        (
+            [(Initialized, 0), (Initialized, 0)],
+            "initialized: 2/2 workers",
+        ),
+        (
+            [(ReadyForCollection, 0), (Created, 0)],
+            "ready: 1/2 workers",
+        ),
+        (
+            [(ReadyForCollection, 0), (ReadyForCollection, 0)],
+            "ready: 2/2 workers",
+        ),
+        (
+            [(CollectionDone, 12), (Created, 0)],
+            "collecting: 1/2 workers",
+        ),
+        (
+            [(CollectionDone, 12), (CollectionDone, 12)],
+            "2 workers [12 items]",
+        ),
+        (
+            [(CollectionDone, 1), (CollectionDone, 1)],
+            "2 workers [1 item]",
+        ),
+        (
+            [(CollectionDone, 1)],
+            "1 worker [1 item]",
+        ),
+        # Different number of tests collected will raise an error and should not happen in practice,
+        # but we test for it anyway.
+        (
+            [(CollectionDone, 1), (CollectionDone, 12)],
+            "2 workers [1 item]",
+        ),
+    ],
+)
+def test_get_workers_status_line(
+    status_and_items: Sequence[tuple[WorkerStatus, int]], expected: str
+) -> None:
+    assert get_workers_status_line(status_and_items) == expected

testing/test_looponfail.py

@@ -1,255 +1,402 @@
-import py
+from __future__ import annotations
+
+import pathlib
+from pathlib import Path
+import shutil
+import tempfile
+import textwrap
+import unittest.mock
+
+import pytest
+
 from xdist.looponfail import RemoteControl
 from xdist.looponfail import StatRecorder
 
+
 class TestStatRecorder:
-    def test_filechange(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
+    def test_filechange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        hello.write("world")
+        hello.write_text("world")
         changed = sd.check()
         assert changed
 
-        p = tmp.ensure("new.py")
+        hello.with_suffix(".pyc").write_text("hello")
+        changed = sd.check()
+        assert not changed
+
+        p = tmp / "new.py"
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "b", "c.py").ensure()
+        tmp.joinpath("a", "b").mkdir(parents=True)
+        tmp.joinpath("a", "b", "c.py").touch()
         changed = sd.check()
         assert changed
 
-        tmp.join("a", "c.txt").ensure()
+        tmp.joinpath("a", "c.txt").touch()
         changed = sd.check()
         assert changed
         changed = sd.check()
         assert not changed
 
-        tmp.join("a").remove()
+        shutil.rmtree(str(tmp.joinpath("a")))
         changed = sd.check()
         assert changed
 
-    def test_filechange_deletion_race(self, tmpdir, monkeypatch):
-        tmp = tmpdir
+    def test_dirchange(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        tmp.joinpath("dir").mkdir()
+        tmp.joinpath("dir", "hello.py").touch()
+        sd = StatRecorder([tmp])
+        assert not sd.fil(tmp / "dir")
+
+    def test_filechange_deletion_race(self, tmp_path: Path) -> None:
+        tmp = tmp_path
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        p = tmp.ensure("new.py")
+        p = tmp.joinpath("new.py")
+        p.touch()
         changed = sd.check()
         assert changed
 
-        p.remove()
+        p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(tmp, 'visit', lambda *args: [p])
-
-        changed = sd.check()
+        dirname = str(tmp)
+        dirnames: list[str] = []
+        filenames = [str(p)]
+        with unittest.mock.patch(
+            "os.walk", return_value=[(dirname, dirnames, filenames)], autospec=True
+        ):
+            changed = sd.check()
         assert changed
 
-    def test_pycremoval(self, tmpdir):
-        tmp = tmpdir
-        hello = tmp.ensure("hello.py")
+    def test_pycremoval(self, tmp_path: Path) -> None:
+        tmp = tmp_path
+        hello = tmp / "hello.py"
+        hello.touch()
         sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
-        pycfile = hello + "c"
-        pycfile.ensure()
+        pycfile = hello.with_suffix(".pyc")
+        pycfile.touch()
+        hello.write_text("world")
         changed = sd.check()
-        assert not changed
+        assert changed
+        assert not pycfile.exists()
 
-        hello.write("world")
-        changed = sd.check()
-        assert not pycfile.check()
-
-    def test_waitonchange(self, tmpdir, monkeypatch):
-        tmp = tmpdir
+    def test_waitonchange(
+        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        tmp = tmp_path
         sd = StatRecorder([tmp])
 
-        l = [True, False]
-        monkeypatch.setattr(StatRecorder, 'check', lambda self: l.pop())
+        ret_values = [True, False]
+        monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
         sd.waitonchange(checkinterval=0.2)
-        assert not l
+        assert not ret_values
+
 
 class TestRemoteControl:
-    def test_nofailures(self, testdir):
-        item = testdir.getitem("def test_func(): pass\n")
+    def test_nofailures(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func(): pass\n")
         control = RemoteControl(item.config)
         control.setup()
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failures_somewhere(self, testdir):
-        item = testdir.getitem("def test_func(): assert 0\n")
+    def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
+        item = pytester.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
-        failures = control.runsession()
+        failures = control.runsession()[0]
         assert failures
         control.setup()
-        item.fspath.write("def test_func(): assert 1\n")
-        pyc = item.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        item.path.write_text("def test_func():\n assert 1\n")
+        removepyc(item.path)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
-    def test_failure_change(self, testdir):
-        modcol = testdir.getitem("""
-            def test_func():
-                assert 0
-        """)
+    def test_failure_change(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
+        )
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol.fspath.write(py.code.Source("""
-            def test_func():
-                assert 1
-            def test_new():
-                assert 0
-        """))
-        pyc = modcol.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        modcol_path = modcol.path
+
+        modcol_path.write_text(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 1
+                def test_new():
+                    assert 0
+                """
+            )
+        )
+        removepyc(modcol_path)
         control.loop_once()
         assert not control.failures
         control.loop_once()
         assert control.failures
         assert str(control.failures).find("test_new") != -1
 
-    def test_failure_subdir_no_init(self, testdir):
-        modcol = testdir.getitem("""
-            def test_func():
-                assert 0
-        """)
-        parent = modcol.fspath.dirpath().dirpath()
-        parent.chdir()
-        modcol.config.args = [py.path.local(x).relto(parent)
-                                for x in modcol.config.args]
+    def test_failure_subdir_no_init(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    assert 0
+                """
+            )
+        )
+        parent = modcol.path.parent.parent
+        monkeypatch.chdir(parent)
+        modcol.config.args = [
+            str(Path(x).relative_to(parent)) for x in modcol.config.args
+        ]
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
         control.loop_once()
         assert control.failures
 
+    def test_ignore_sys_path_hook_entry(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Modifying sys.path as seen by the worker process is a bit tricky,
+        # because any changes made in the current process do not carry over.
+        # However, we can leverage the `sitecustomize` behavior to run arbitrary
+        # code when the subprocess interpreter is starting up. We just need to
+        # install our module in the search path, which we can accomplish by
+        # adding a temporary directory to PYTHONPATH.
+        tmpdir = tempfile.TemporaryDirectory()
+        with open(pathlib.Path(tmpdir.name) / "sitecustomize.py", "w") as custom:
+            print(
+                textwrap.dedent(
+                    """
+                    import sys
+                    sys.path.append('dummy.__path_hook__')
+                    """
+                ),
+                file=custom,
+            )
+
+        monkeypatch.setenv("PYTHONPATH", tmpdir.name, prepend=":")
+
+        item = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    import sys
+                    assert "dummy.__path_hook__" in sys.path
+                """
+            )
+        )
+        control = RemoteControl(item.config)
+        control.setup()
+        topdir, failures = control.runsession()[:2]
+        assert not failures
+
+
 class TestLooponFailing:
-    def test_looponfail_from_fail_to_ok(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                x = 0
-                assert x == 1
-            def test_two():
-                assert 1
-        """)
+    def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                def test_two():
+                    assert 1
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-        modcol.fspath.write(py.code.Source("""
-            def test_one():
-                assert 1
-            def test_two():
-                assert 1
-        """))
-        removepyc(modcol.fspath)
+        modcol.path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1
+                def test_two():
+                    assert 1
+                """
+            )
+        )
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert not remotecontrol.failures
 
-    def test_looponfail_from_one_to_two_tests(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                assert 0
-        """)
+    def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
-        assert 'test_one' in remotecontrol.failures[0]
+        assert "test_one" in remotecontrol.failures[0]
 
-        modcol.fspath.write(py.code.Source("""
-            def test_one():
-                assert 1 # passes now
-            def test_two():
-                assert 0 # new and fails
-        """))
-        removepyc(modcol.fspath)
+        modcol.path.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 1 # passes now
+                def test_two():
+                    assert 0 # new and fails
+                """
+            )
+        )
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
-        assert 'test_one' not in remotecontrol.failures[0]
-        assert 'test_two' in remotecontrol.failures[0]
+        assert "test_one" not in remotecontrol.failures[0]
+        assert "test_two" in remotecontrol.failures[0]
 
-    def test_looponfail_removed_test(self, testdir):
-        modcol = testdir.getmodulecol("""
-            def test_one():
-                assert 0
-            def test_two():
-                assert 0
-        """)
+    @pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
+    def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                def test_two():
+                    assert 0
+                """
+            )
+        )
         remotecontrol = RemoteControl(modcol.config)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 2
 
-        modcol.fspath.write(py.code.Source("""
-            def test_xxx(): # renamed test
-                assert 0
-            def test_two():
-                assert 1 # pass now
-        """))
-        removepyc(modcol.fspath)
+        modcol.path.write_text(
+            textwrap.dedent(
+                """
+                def test_xxx(): # renamed test
+                    assert 0
+                def test_two():
+                    assert 1 # pass now
+                """
+            )
+        )
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
 
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
+    def test_looponfail_multiple_errors(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        modcol = pytester.getmodulecol(
+            textwrap.dedent(
+                """
+                def test_one():
+                    assert 0
+                """
+            )
+        )
+        remotecontrol = RemoteControl(modcol.config)
+        orig_runsession = remotecontrol.runsession
+
+        def runsession_dups() -> tuple[list[str], list[str], bool]:
+            # twisted.trial test cases may report multiple errors.
+            failures, reports, collection_failed = orig_runsession()
+            print(failures)
+            return failures * 2, reports, collection_failed
+
+        monkeypatch.setattr(remotecontrol, "runsession", runsession_dups)
+        remotecontrol.loop_once()
+        assert len(remotecontrol.failures) == 1
+
 
 class TestFunctional:
-    def test_fail_to_ok(self, testdir):
-        p = testdir.makepyfile("""
-            def test_one():
-                x = 0
-                assert x == 1
-        """)
-        #p = testdir.mkdir("sub").join(p1.basename)
-        #p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+    def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 0
+                    assert x == 1
+                """
+            )
+        )
+        # p = pytester.mkdir("sub").join(p1.basename)
+        # p1.move(p)
+        child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
         child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
-        p.write(py.code.Source("""
-            def test_one():
-                x = 1
-                assert x == 1
-        """))
+        p.write_text(
+            textwrap.dedent(
+                """
+                def test_one():
+                    x = 1
+                    assert x == 1
+                """
+            ),
+        )
         child.expect(".*1 passed.*")
         child.kill(15)
 
-    def test_xfail_passes(self, testdir):
-        p = testdir.makepyfile("""
-            import py
-            @py.test.mark.xfail
-            def test_one():
-                pass
-        """)
-        child = testdir.spawn_pytest("-f %s" % p)
+    def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
+        p = pytester.makepyfile(
+            textwrap.dedent(
+                """
+                import pytest
+                @pytest.mark.xfail
+                def test_one():
+                    pass
+                """
+            )
+        )
+        child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
-        child.expect("### LOOPONFAILING ####")
+        # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
         child.kill(15)
 
-def removepyc(path):
-    # XXX damn those pyc files
-    pyc = path + "c"
-    if pyc.check():
-        pyc.remove()
 
+def removepyc(path: Path) -> None:
+    # XXX damn those pyc files
+    pyc = path.with_suffix(".pyc")
+    if pyc.exists():
+        pyc.unlink()
+    c = path.parent / "__pycache__"
+    if c.exists():
+        shutil.rmtree(c)

testing/test_newhooks.py

@@ -0,0 +1,121 @@
+import pytest
+
+
+class TestHooks:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): pass
+            def test_c(): pass
+        """
+        )
+
+    def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
+        """Test that log reports from pytest_runtest_logreport when running with
+        xdist contain "node", "nodeid", "worker_id", and "testrun_uid"
+        attributes (#8)."""
+        pytester.makeconftest(
+            """
+            def pytest_runtest_logreport(report):
+                if hasattr(report, 'node'):
+                    if report.when == "call":
+                        workerid = report.node.workerinput['workerid']
+                        testrunuid = report.node.workerinput['testrunuid']
+                        if workerid != report.worker_id:
+                            print("HOOK: Worker id mismatch: %s %s"
+                                   % (workerid, report.worker_id))
+                        elif testrunuid != report.testrun_uid:
+                            print("HOOK: Testrun uid mismatch: %s %s"
+                                   % (testrunuid, report.testrun_uid))
+                        else:
+                            print("HOOK: %s %s %s"
+                                   % (report.nodeid, report.worker_id, report.testrun_uid))
+        """
+        )
+        res = pytester.runpytest("-n1", "-s")
+        res.stdout.fnmatch_lines(
+            [
+                "*HOOK: test_runtest_logreport.py::test_a gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_b gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_c gw0 *",
+                "*3 passed*",
+            ]
+        )
+
+    def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_xdist_node_collection_finished hook (#8)."""
+        pytester.makeconftest(
+            """
+            def pytest_xdist_node_collection_finished(node, ids):
+                workerid = node.workerinput['workerid']
+                stripped_ids = [x.split('::')[1] for x in ids]
+                print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(
+            ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
+        )
+        res.stdout.fnmatch_lines(["*3 passed*"])
+
+
+class TestCrashItem:
+    @pytest.fixture(autouse=True)
+    def create_test_file(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            """
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+            def test_d(): pass
+        """
+        )
+
+    def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n2", "-s")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(["*3 passed*"])
+
+    def test_handlecrashitem_one(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook with just one test."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n1", "-s", "-k", "test_b")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(
+            [
+                "FAILED test_handlecrashitem_one.py::test_b",
+                "FAILED test_handlecrashitem_one.py::test_b",
+            ]
+        )

testing/test_plugin.py

@@ -1,65 +1,339 @@
-import py
+from contextlib import suppress
+import os
+from pathlib import Path
+import sys
+
 import execnet
-from xdist.slavemanage import NodeManager
+import pytest
 
-def test_dist_incompatibility_messages(testdir):
-    result = testdir.runpytest("--pdb", "--looponfail")
+from xdist.workermanage import NodeManager
+
+
+@pytest.fixture
+def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Make pytest-xdist believe the system has 3 CPUs."""
+    # block import
+    monkeypatch.setitem(sys.modules, "psutil", None)
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setattr(os, "cpu_count", lambda: 3)
+
+
+def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
+    result = pytester.runpytest("--pdb", "--looponfail")
     assert result.ret != 0
-    result = testdir.runpytest("--pdb", "-n", "3")
+    result = pytester.runpytest("--pdb", "-n", "3")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
-    result = testdir.runpytest("--pdb", "-d", "--tx", "popen")
+    result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
     assert result.ret != 0
     assert "incompatible" in result.stderr.str()
 
-def test_dist_options(testdir):
-    from xdist.plugin import check_options
-    config = testdir.parseconfigure("-n 2")
+
+def test_dist_options(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    config = pytester.parseconfigure("-n 2")
     check_options(config)
     assert config.option.dist == "load"
-    assert config.option.tx == ['popen'] * 2
-    config = testdir.parseconfigure("-d")
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("--numprocesses", "2")
     check_options(config)
     assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
+    config = pytester.parseconfigure("-d")
+    check_options(config)
+    assert config.option.dist == "load"
+
+    config = pytester.parseconfigure("--numprocesses", "0")
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
+    config = pytester.parseconfigure("--numprocesses", "0", "-d")
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
+    config = pytester.parseconfigure(
+        "--numprocesses", "0", "--dist", "each", "--tx", "2*popen"
+    )
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
+
+def test_auto_detect_cpus(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
+
+    if hasattr(os, "sched_getaffinity"):
+        monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
+    elif hasattr(os, "cpu_count"):
+        monkeypatch.setattr(os, "cpu_count", lambda: 99)
+    else:
+        import multiprocessing
+
+        monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
+
+    config = pytester.parseconfigure("-n2")
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 99
+
+    for numprocesses in (0, "auto", "logical"):
+        config = pytester.parseconfigure(f"-n{numprocesses}", "--pdb")
+        check_options(config)
+        assert config.getoption("usepdb")
+        assert config.getoption("numprocesses") == 0
+        assert config.getoption("dist") == "no"
+
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setenv("TRAVIS", "true")
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+
+def test_auto_detect_cpus_psutil(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 84
+
+
+def test_auto_detect_cpus_os(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_hook_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
+def test_hook_auto_num_workers_arg(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # config.option.numprocesses is a pytest feature,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers(config):
+            if config.option.numprocesses == 'auto':
+                return 42
+            if config.option.numprocesses == 'logical':
+                return 8
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 8
+
+
+def test_hook_auto_num_workers_none(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    # Returning None from a hook to skip it is pytest behavior,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return None
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "5")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 5
+
+
+def test_envvar_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "7")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+
+def test_envvar_auto_num_workers_warn(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "fourscore")
+
+    config = pytester.parseconfigure("-nauto")
+    with pytest.warns(UserWarning):
+        check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_auto_num_workers_hook_overrides_envvar(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "987")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 2
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+
+def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+    from xdist.plugin import pytest_configure as configure
+
+    config = pytester.parseconfigure("-n1")
+    check_options(config)
+    configure(config)
+    assert config.pluginmanager.hasplugin("dsession")
+
+    config = pytester.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    configure(config)
+    assert not config.pluginmanager.hasplugin("dsession")
+
+
+def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    nm = NodeManager(config)
+    assert nm.testrunuid == "test123"
+
+
+def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
+    config = pytester.parseconfigure("--tx=popen")
+    nm = NodeManager(config)
+    assert len(nm.testrunuid) == 32
+
 
 class TestDistOptions:
-    def test_getxspecs(self, testdir):
-        config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
+    def test_getxspecs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
         nodemanager = NodeManager(config)
-        xspecs = nodemanager._getxspecs()
+        xspecs = nodemanager._gettxspecs()
         assert len(xspecs) == 2
         print(xspecs)
         assert xspecs[0].popen
         assert xspecs[1].ssh == "xyz"
 
-    def test_xspecs_multiplied(self, testdir):
-        config = testdir.parseconfigure("--tx=3*popen",)
-        xspecs = NodeManager(config)._getxspecs()
+    def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--tx=3*popen")
+        xspecs = NodeManager(config)._gettxspecs()
         assert len(xspecs) == 3
         assert xspecs[1].popen
 
-    def test_getrsyncdirs(self, testdir):
-        config = testdir.parseconfigure('--rsyncdir=' + str(testdir.tmpdir))
+    def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
         nm = NodeManager(config, specs=[execnet.XSpec("popen")])
         assert not nm._getrsyncdirs()
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
         assert nm.roots
-        assert testdir.tmpdir in nm.roots
+        assert pytester.path in nm.roots
 
-    def test_getrsyncdirs_with_conftest(self, testdir):
-        p = py.path.local()
-        for bn in 'x y z'.split():
-            p.mkdir(bn)
-        testdir.makeini("""
+    def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfigure("--rsyncignore=fo*")
+        nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
+        assert "fo*" in nm.rsyncoptions["ignores"]
+
+    def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
+        p = Path.cwd()
+        for bn in ("x", "y", "z"):
+            p.joinpath(bn).mkdir()
+        pytester.makeini(
+            """
             [pytest]
             rsyncdirs= x
-        """)
-        config = testdir.parseconfigure(
-              testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z')
+        """
+        )
+        config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
         nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
         roots = nm._getrsyncdirs()
-        #assert len(roots) == 3 + 1 # pylib
-        assert py.path.local('y') in roots
-        assert py.path.local('z') in roots
-        assert testdir.tmpdir.join('x') in roots
-
+        # assert len(roots) == 3 + 1 # pylib
+        assert Path("y").resolve() in roots
+        assert Path("z").resolve() in roots
+        assert pytester.path.joinpath("x") in roots

testing/test_remote.py

@@ -1,46 +1,73 @@
-import py
-from xdist.slavemanage import SlaveController, unserialize_report
-from xdist.remote import serialize_report
-import execnet
-queue = py.builtin._tryimport("queue", "Queue")
-from py.builtin import print_
+from __future__ import annotations
+
 import marshal
+import pprint
+from queue import Queue
+import sys
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Union
+import uuid
+
+import execnet
+import pytest
+
+from xdist.workermanage import NodeManager
+from xdist.workermanage import WorkerController
+
 
 WAIT_TIMEOUT = 10.0
 
-def check_marshallable(d):
+
+def check_marshallable(d: object) -> None:
     try:
-        marshal.dumps(d)
-    except ValueError:
-        py.std.pprint.pprint(d)
-        raise ValueError("not marshallable")
+        marshal.dumps(d)  # type: ignore[arg-type]
+    except ValueError as e:
+        pprint.pprint(d)
+        raise ValueError("not marshallable") from e
+
 
 class EventCall:
-    def __init__(self, eventcall):
+    def __init__(self, eventcall: tuple[str, dict[str, Any]]) -> None:
         self.name, self.kwargs = eventcall
 
-    def __str__(self):
-        return "<EventCall %s(**%s)>" %(self.name, self.kwargs)
+    def __str__(self) -> str:
+        return f"<EventCall {self.name}(**{self.kwargs})>"
 
-class SlaveSetup:
-    use_callback = False
 
-    def __init__(self, request):
-        self.testdir = testdir = request.getfuncargvalue("testdir")
+class WorkerSetup:
+    def __init__(
+        self, request: pytest.FixtureRequest, pytester: pytest.Pytester
+    ) -> None:
         self.request = request
-        self.events = queue.Queue()
+        self.pytester = pytester
+        self.use_callback = False
+        self.events = Queue()  # type: ignore[var-annotated]
 
-    def setup(self, ):
-        self.testdir.chdir()
-        #import os ; os.environ['EXECNET_DEBUG'] = "2"
-        self.gateway = execnet.makegateway()
-        self.config = config = self.testdir.parseconfigure()
-        putevent = self.use_callback and self.events.put or None
-        self.slp = SlaveController(None, self.gateway, config, putevent)
+    def setup(self) -> None:
+        self.pytester.chdir()
+        # import os ; os.environ['EXECNET_DEBUG'] = "2"
+        self.gateway = execnet.makegateway("execmodel=main_thread_only//popen")
+        self.config = config = self.pytester.parseconfigure()
+        putevent = self.events.put if self.use_callback else None
+
+        class DummyMananger:
+            testrunuid = uuid.uuid4().hex
+            specs = [0, 1]
+
+        nodemanager = cast(NodeManager, DummyMananger)
+
+        self.slp = WorkerController(
+            nodemanager=nodemanager,
+            gateway=self.gateway,
+            config=config,
+            putevent=putevent,  # type: ignore[arg-type]
+        )
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
-    def popevent(self, name=None):
+    def popevent(self, name: str | None = None) -> EventCall:
         while 1:
             if self.use_callback:
                 data = self.events.get(timeout=WAIT_TIMEOUT)
@@ -49,181 +76,346 @@ class SlaveSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping %s" % (ev,))
+            print(f"skipping {ev}")
 
-    def sendcommand(self, name, **kwargs):
+    def sendcommand(self, name: str, **kwargs: Any) -> None:
         self.slp.sendcommand(name, **kwargs)
 
-def pytest_funcarg__slave(request):
-    return SlaveSetup(request)
 
-def test_remoteinitconfig(testdir):
-    from xdist.remote import remote_initconfig
-    config1 = testdir.parseconfig()
-    config2 = remote_initconfig(config1.option.__dict__, config1.args)
-    assert config2.option.__dict__ == config1.option.__dict__
-    assert config2.pluginmanager.getplugin("terminal") in (-1, None)
+@pytest.fixture
+def worker(request: pytest.FixtureRequest, pytester: pytest.Pytester) -> WorkerSetup:
+    return WorkerSetup(request, pytester)
 
-class TestReportSerialization:
-    def test_itemreport_outcomes(self, testdir):
-        reprec = testdir.inline_runsource("""
-            import py
-            def test_pass(): pass
-            def test_fail(): 0/0
-            @py.test.mark.skipif("True")
-            def test_skip(): pass
-            def test_skip_imperative():
-                py.test.skip("hello")
-            @py.test.mark.xfail("True")
-            def test_xfail(): 0/0
-            def test_xfail_imperative():
-                py.test.xfail("hello")
-        """)
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 17 # with setup/teardown "passed" reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("testreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if newrep.skipped and 'xfail' not in newrep.keywords:
-                assert len(newrep.longrepr) == 3
-            assert newrep.outcome == rep.outcome
-            assert newrep.when == rep.when
-            assert newrep.keywords == rep.keywords
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
 
-    def test_collectreport_passed(self, testdir):
-        reprec = testdir.inline_runsource("def test_func(): pass")
-        reports = reprec.getreports("pytest_collectreport")
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
+class TestWorkerInteractor:
+    UnserializerReport = Callable[
+        [dict[str, Any]], Union[pytest.CollectReport, pytest.TestReport]
+    ]
 
-    def test_collectreport_fail(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
+    @pytest.fixture
+    def unserialize_report(self, pytestconfig: pytest.Config) -> UnserializerReport:
+        def unserialize(
+            data: dict[str, Any],
+        ) -> pytest.CollectReport | pytest.TestReport:
+            return pytestconfig.hook.pytest_report_from_serializable(  # type: ignore[no-any-return]
+                config=pytestconfig, data=data
+            )
 
-class TestSlaveInteractor:
-    def test_basic_collect_and_runtests(self, slave):
-        p = slave.testdir.makepyfile("""
+        return unserialize
+
+    def test_basic_collect_and_runtests(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
             def test_func():
                 pass
-        """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        """
+        )
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
-        assert ev.kwargs['topdir'] == slave.testdir.tmpdir
-        ids = ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        assert ev.kwargs["topdir"] == str(worker.pytester.path)
+        ids = ev.kwargs["ids"]
         assert len(ids) == 1
-        slave.sendcommand("runtests", ids=ids)
-        ev = slave.popevent("testreport") # setup
-        ev = slave.popevent("testreport")
+        worker.sendcommand("runtests", indices=list(range(len(ids))))
+        worker.sendcommand("shutdown")
+        ev = worker.popevent("logstart")
+        assert ev.kwargs["nodeid"].endswith("test_func")
+        assert len(ev.kwargs["location"]) == 3
+        ev = worker.popevent("testreport")  # setup
+        ev = worker.popevent("testreport")
         assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        slave.sendcommand("shutdown")
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, slave):
-        p = slave.testdir.makepyfile("""
-            import py
-            py.test.skip("hello")
-        """)
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+    def test_remote_collect_skip(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
+            import pytest
+            pytest.skip("hello", allow_module_level=True)
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
-        assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
-        ev = slave.popevent("collectionfinish")
-        assert not ev.kwargs['ids']
+        assert isinstance(rep.longrepr, tuple)
+        assert rep.longrepr[2] == "Skipped: hello"
+        ev = worker.popevent("collectionfinish")
+        assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, slave):
-        p = slave.testdir.makepyfile("""aasd qwe""")
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+    def test_remote_collect_fail(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile("""aasd qwe""")
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
-        assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.failed
-        ev = slave.popevent("collectionfinish")
-        assert not ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, slave):
-        p = slave.testdir.makepyfile("""
+    def test_runtests_all(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
             def test_func(): pass
             def test_func2(): pass
-        """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        """
+        )
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
-        ids = ev.kwargs['ids']
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
         assert len(ids) == 2
-        slave.sendcommand("runtests_all", )
-        slave.sendcommand("shutdown", )
+        worker.sendcommand("runtests_all")
+        worker.sendcommand("shutdown")
         for func in "::test_func", "::test_func2":
-            for i in range(3):  # setup/call/teardown
-                ev = slave.popevent("testreport")
+            for _ in range(3):  # setup/call/teardown
+                ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
-                rep = unserialize_report(ev.name, ev.kwargs['data'])
+                rep = unserialize_report(ev.kwargs["data"])
                 assert rep.nodeid.endswith(func)
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, slave):
-        py.test.xfail("implement a simple test for event production")
-        assert not slave.use_callback
-        p = slave.testdir.makepyfile("""
+    def test_happy_run_events_converted(
+        self, pytester: pytest.Pytester, worker: WorkerSetup
+    ) -> None:
+        pytest.xfail("implement a simple test for event production")
+        assert not worker.use_callback  # type: ignore[unreachable]
+        worker.pytester.makepyfile(
+            """
             def test_func():
                 pass
-        """)
-        slave.setup()
-        hookrec = testdir.getreportrecorder(slave.config)
-        for data in slave.slp.channel:
-            slave.slp.process_from_remote(data)
-        slave.slp.process_from_remote(slave.slp.ENDMARK)
-        py.std.pprint.pprint(hookrec.hookrecorder.calls)
-        hookrec.hookrecorder.contains([
-            ("pytest_collectstart", "collector.fspath == aaa"),
-            ("pytest_pycollect_makeitem", "name == 'test_func'"),
-            ("pytest_collectreport", "report.collector.fspath == aaa"),
-            ("pytest_collectstart", "collector.fspath == bbb"),
-            ("pytest_pycollect_makeitem", "name == 'test_func'"),
-            ("pytest_collectreport", "report.collector.fspath == bbb"),
-        ])
+        """
+        )
+        worker.setup()
+        hookrec = pytester.getreportrecorder(worker.config)
+        for data in worker.slp.channel:
+            worker.slp.process_from_remote(data)
+        worker.slp.process_from_remote(worker.slp.ENDMARK)
+        pprint.pprint(hookrec.hookrecorder.calls)
+        hookrec.hookrecorder.contains(
+            [
+                ("pytest_collectstart", "collector.fspath == aaa"),
+                ("pytest_pycollect_makeitem", "name == 'test_func'"),
+                ("pytest_collectreport", "report.collector.fspath == aaa"),
+                ("pytest_collectstart", "collector.fspath == bbb"),
+                ("pytest_pycollect_makeitem", "name == 'test_func'"),
+                ("pytest_collectreport", "report.collector.fspath == bbb"),
+            ]
+        )
 
+    def test_process_from_remote_error_handling(
+        self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
+    ) -> None:
+        worker.use_callback = True
+        worker.setup()
+        worker.slp.process_from_remote(("<nonono>", {}))
+        out, err = capsys.readouterr()
+        assert "INTERNALERROR> ValueError: unknown event: <nonono>" in out
+        ev = worker.popevent()
+        assert ev.name == "errordown"
+
+    def test_steal_work(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
+            import time
+            def test_func(): time.sleep(1)
+            def test_func2(): pass
+            def test_func3(): pass
+            def test_func4(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 4
+        worker.sendcommand("runtests_all")
+
+        # wait for test_func setup
+        ev = worker.popevent("testreport")
+        rep = unserialize_report(ev.kwargs["data"])
+        assert rep.nodeid.endswith("::test_func")
+        assert rep.when == "setup"
+
+        worker.sendcommand("steal", indices=[1, 2])
+        ev = worker.popevent("unscheduled")
+        # Cannot steal index 1 because it is completed already, so do not steal any.
+        assert ev.kwargs["indices"] == []
+
+        # Index 2 can be stolen, as it is still pending.
+        worker.sendcommand("steal", indices=[2])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == [2]
+
+        reports = [
+            ("test_func", "call"),
+            ("test_func", "teardown"),
+            ("test_func2", "setup"),
+            ("test_func2", "call"),
+            ("test_func2", "teardown"),
+        ]
+
+        for func, when in reports:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith(f"::{func}")
+            assert rep.when == when
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func4")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
+    def test_steal_empty_queue(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
+            def test_func(): pass
+            def test_func2(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 2
+        worker.sendcommand("runtests_all")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func")
+            assert rep.when == when
+
+        worker.sendcommand("steal", indices=[0, 1])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == []
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func2")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
+
+def test_remote_env_vars(pytester: pytest.Pytester) -> None:
+    pytester.makepyfile(
+        """
+        import os
+        def test():
+            assert len(os.environ['PYTEST_XDIST_TESTRUNUID']) == 32
+            assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
+            assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
+    """
+    )
+    result = pytester.runpytest("-n2", "--max-worker-restart=0")
+    assert result.ret == 0
+
+
+def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
+    """Test/document the behavior due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_argv():
+            assert sys.argv == ["-c"]
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0
+
+
+def test_remote_mainargv(pytester: pytest.Pytester) -> None:
+    outer_argv = sys.argv
+
+    pytester.makepyfile(
+        f"""
+        def test_mainargv(request):
+            assert request.config.workerinput["mainargv"] == {outer_argv!r}
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0
+
+
+def test_remote_usage_prog(pytester: pytest.Pytester) -> None:
+    pytester.makeconftest(
+        """
+        import pytest
+
+        config_parser = None
+
+        @pytest.fixture
+        def get_config_parser():
+            return config_parser
+
+        def pytest_configure(config):
+            global config_parser
+            config_parser = config._parser
+    """
+    )
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test(get_config_parser, request):
+            get_config_parser._getparser().error("my_usage_error")
+    """
+    )
+
+    result = pytester.runpytest_subprocess("-n1")
+    assert result.ret == 1
+    result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
+
+
+def test_remote_sys_path(pytester: pytest.Pytester) -> None:
+    """Work around sys.path differences due to execnet using `python -c`."""
+    pytester.makepyfile(
+        """
+        import sys
+
+        def test_sys_path():
+            assert "" not in sys.path
+        """
+    )
+    result = pytester.runpytest("-n1")
+    assert result.ret == 0

testing/test_slavemanage.py

@@ -1,233 +0,0 @@
-import py
-import os
-import execnet
-from xdist.slavemanage import HostRSync, NodeManager
-
-pytest_plugins = "pytester",
-
-def pytest_funcarg__hookrecorder(request):
-    _pytest = request.getfuncargvalue('_pytest')
-    config = request.getfuncargvalue('config')
-    return _pytest.gethookrecorder(config.hook)
-
-def pytest_funcarg__config(request):
-    testdir = request.getfuncargvalue("testdir")
-    config = testdir.parseconfig()
-    return config
-
-class pytest_funcarg__mysetup:
-    def __init__(self, request):
-        temp = request.getfuncargvalue("tmpdir")
-        self.source = temp.mkdir("source")
-        self.dest = temp.mkdir("dest")
-        request.getfuncargvalue("_pytest")
-
-class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, config):
-        gm = NodeManager(config, ["popen"])
-        assert gm.specs[0].chdir is None
-
-    def test_default_chdir(self, config):
-        l = ["ssh=noco", "socket=xyz"]
-        for spec in NodeManager(config, l).specs:
-            assert spec.chdir == "pyexecnetcache"
-        for spec in NodeManager(config, l, defaultchdir="abc").specs:
-            assert spec.chdir == "abc"
-
-    def test_popen_makegateway_events(self, config, hookrecorder, _pytest):
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.makegateways()
-        call = hookrecorder.popcall("pytest_xdist_setupnodes")
-        assert len(call.specs) == 2
-
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.spec == execnet.XSpec("popen")
-        assert call.gateway.id == "gw0"
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.id == "gw1"
-        assert len(hm.group) == 2
-        hm.teardown_nodes()
-        assert not len(hm.group)
-
-    def test_popens_rsync(self, config, mysetup):
-        source = mysetup.source
-        hm = NodeManager(config, ["popen"] * 2)
-        hm.makegateways()
-        assert len(hm.group) == 2
-        for gw in hm.group:
-            class pseudoexec:
-                args = []
-                def __init__(self, *args):
-                    self.args.extend(args)
-                def waitclose(self):
-                    pass
-            gw.remote_exec = pseudoexec
-        l = []
-        hm.rsync(source, notify=lambda *args: l.append(args))
-        assert not l
-        hm.teardown_nodes()
-        assert not len(hm.group)
-        assert "sys.path.insert" in gw.remote_exec.args[0]
-
-    def test_rsync_popen_with_path(self, config, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 1)
-        hm.makegateways()
-        source.ensure("dir1", "dir2", "hello")
-        l = []
-        hm.rsync(source, notify=lambda *args: l.append(args))
-        assert len(l) == 1
-        assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
-        hm.teardown_nodes()
-        dest = dest.join(source.basename)
-        assert dest.join("dir1").check()
-        assert dest.join("dir1", "dir2").check()
-        assert dest.join("dir1", "dir2", 'hello').check()
-
-    def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder):
-        source, dest = mysetup.source, mysetup.dest
-        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 2)
-        hm.makegateways()
-        source.ensure("dir1", "dir2", "hello")
-        hm.rsync(source)
-        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
-        assert call.source == source
-        assert len(call.gateways) == 1
-        assert call.gateways[0] in hm.group
-        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
-
-class TestHRSync:
-    class pytest_funcarg__mysetup:
-        def __init__(self, request):
-            tmp = request.getfuncargvalue('tmpdir')
-            self.source = tmp.mkdir("source")
-            self.dest = tmp.mkdir("dest")
-
-    def test_hrsync_filter(self, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        source.ensure("dir", "file.txt")
-        source.ensure(".svn", "entries")
-        source.ensure(".somedotfile", "moreentries")
-        source.ensure("somedir", "editfile~")
-        syncer = HostRSync(source)
-        l = list(source.visit(rec=syncer.filter,
-                                   fil=syncer.filter))
-        assert len(l) == 3
-        basenames = [x.basename for x in l]
-        assert 'dir' in basenames
-        assert 'file.txt' in basenames
-        assert 'somedir' in basenames
-
-    def test_hrsync_one_host(self, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        gw = execnet.makegateway("popen//chdir=%s" % dest)
-        finished = []
-        rsync = HostRSync(source)
-        rsync.add_target_host(gw, finished=lambda: finished.append(1))
-        source.join("hello.py").write("world")
-        rsync.send()
-        gw.exit()
-        assert dest.join(source.basename, "hello.py").check()
-        assert len(finished) == 1
-
-
-class TestNodeManager:
-    @py.test.mark.xfail
-    def test_rsync_roots_no_roots(self, testdir, mysetup):
-        mysetup.source.ensure("dir1", "file1").write("hello")
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
-        #assert nodemanager.config.topdir == source == config.topdir
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        p, = nodemanager.gwmanager.multi_exec(
-            "import os ; channel.send(os.getcwd())").receive_each()
-        p = py.path.local(p)
-        py.builtin.print_("remote curdir", p)
-        assert p == mysetup.dest.join(config.topdir.basename)
-        assert p.join("dir1").check()
-        assert p.join("dir1", "file1").check()
-
-    def test_popen_rsync_subdir(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir1 = mysetup.source.mkdir("dir1")
-        dir2 = dir1.mkdir("dir2")
-        dir2.ensure("hello")
-        for rsyncroot in (dir1, source):
-            dest.remove()
-            nodemanager = NodeManager(testdir.parseconfig(
-                "--tx", "popen//chdir=%s" % dest,
-                "--rsyncdir", rsyncroot,
-                source,
-            ))
-            nodemanager.makegateways()
-            nodemanager.rsync_roots()
-            if rsyncroot == source:
-                dest = dest.join("source")
-            assert dest.join("dir1").check()
-            assert dest.join("dir1", "dir2").check()
-            assert dest.join("dir1", "dir2", 'hello').check()
-            nodemanager.teardown_nodes()
-
-    def test_init_rsync_roots(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        source.ensure("dir1", "somefile", dir=1)
-        dir2.ensure("hello")
-        source.ensure("bogusdir", "file")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
-            [pytest]
-            rsyncdirs=dir1/dir2
-        """))
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        assert dest.join("dir2").check()
-        assert not dest.join("dir1").check()
-        assert not dest.join("bogus").check()
-
-    def test_rsyncignore(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        dir2 = source.ensure("dir1", "dir2", dir=1)
-        dir5 = source.ensure("dir5", "dir6", "bogus")
-        dirf = source.ensure("dir5", "file")
-        dir2.ensure("hello")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
-            [pytest]
-            rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6
-        """))
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        assert dest.join("dir1").check()
-        assert not dest.join("dir1", "dir2").check()
-        assert dest.join("dir5","file").check()
-        assert not dest.join("dir6").check()
-
-    def test_optimise_popen(self, testdir, mysetup):
-        source, dest = mysetup.source, mysetup.dest
-        specs = ["popen"] * 3
-        source.join("conftest.py").write("rsyncdirs = ['a']")
-        source.ensure('a', dir=1)
-        config = testdir.parseconfig(source)
-        nodemanager = NodeManager(config, specs)
-        nodemanager.makegateways()
-        nodemanager.rsync_roots()
-        for gwspec in nodemanager.specs:
-            assert gwspec._samefilesystem()
-            assert not gwspec.chdir
-
-    def test_ssh_setup_nodes(self, specssh, testdir):
-        testdir.makepyfile(__init__="", test_x="""
-            def test_one():
-                pass
-        """)
-        reprec = testdir.inline_run("-d", "--rsyncdir=%s" % testdir.tmpdir,
-                "--tx", specssh, testdir.tmpdir)
-        rep, = reprec.getreports("pytest_runtest_logreport")
-        assert rep.passed
-
-

testing/test_workermanage.py

@@ -0,0 +1,480 @@
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+import textwrap
+import warnings
+
+import execnet
+import pytest
+from util import generate_warning
+
+from xdist import workermanage
+from xdist._path import visit_path
+from xdist.remote import serialize_warning_message
+from xdist.workermanage import HostRSync
+from xdist.workermanage import NodeManager
+from xdist.workermanage import unserialize_warning_message
+
+
+pytest_plugins = "pytester"
+
+
+@pytest.fixture
+def hookrecorder(
+    config: pytest.Config, pytester: pytest.Pytester
+) -> pytest.HookRecorder:
+    hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
+    return hookrecorder
+
+
+@pytest.fixture
+def config(pytester: pytest.Pytester) -> pytest.Config:
+    return pytester.parseconfig()
+
+
+@pytest.fixture
+def source(tmp_path: Path) -> Path:
+    source = tmp_path / "source"
+    source.mkdir()
+    return source
+
+
+@pytest.fixture
+def dest(tmp_path: Path) -> Path:
+    dest = tmp_path / "dest"
+    dest.mkdir()
+    return dest
+
+
+@pytest.fixture
+def workercontroller(monkeypatch: pytest.MonkeyPatch) -> None:
+    class MockController:
+        def __init__(self, *args: object) -> None:
+            pass
+
+        def setup(self) -> None:
+            pass
+
+    monkeypatch.setattr(workermanage, "WorkerController", MockController)
+
+
+class TestNodeManagerPopen:
+    def test_popen_no_default_chdir(self, config: pytest.Config) -> None:
+        gm = NodeManager(config, ["popen"])
+        assert gm.specs[0].chdir is None
+
+    def test_default_chdir(self, config: pytest.Config) -> None:
+        specs = ["ssh=noco", "socket=xyz"]
+        for spec in NodeManager(config, specs).specs:
+            assert spec.chdir == "pyexecnetcache"
+        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
+            assert spec.chdir == "abc"
+
+    def test_popen_makegateway_events(
+        self,
+        config: pytest.Config,
+        hookrecorder: pytest.HookRecorder,
+        workercontroller: None,
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
+        call = hookrecorder.popcall("pytest_xdist_setupnodes")
+        assert len(call.specs) == 2
+
+        # check expected gateways
+        gw_calls = [
+            hookrecorder.popcall("pytest_xdist_newgateway"),
+            hookrecorder.popcall("pytest_xdist_newgateway"),
+        ]
+        assert {c.gateway.id for c in gw_calls} == {"gw0", "gw1"}
+
+        for c in gw_calls:
+            expected_spec = execnet.XSpec(
+                f"execmodel=main_thread_only//popen//id={c.gateway.id}"
+            )
+            assert c.gateway.spec == expected_spec
+
+        assert len(hm.group) == 2
+        hm.teardown_nodes()
+        assert not len(hm.group)
+
+    def test_popens_rsync(
+        self,
+        config: pytest.Config,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+    ) -> None:
+        hm = NodeManager(config, ["popen"] * 2)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
+        assert len(hm.group) == 2
+        for gw in hm.group:
+
+            class pseudoexec:
+                args = []  # type: ignore[var-annotated]
+
+                def __init__(self, *args: object) -> None:
+                    self.args.extend(args)
+
+                def waitclose(self) -> None:
+                    pass
+
+            gw.remote_exec = pseudoexec  # type: ignore[assignment]
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert not notifications
+        hm.teardown_nodes()
+        assert not len(hm.group)
+        assert "sys.path.insert" in gw.remote_exec.args[0]  # type: ignore[attr-defined]
+
+    def test_rsync_popen_with_path(
+        self, config: pytest.Config, source: Path, dest: Path, workercontroller: None
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        notifications = []
+        for gw in hm.group:
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert len(notifications) == 1
+        assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
+        hm.teardown_nodes()
+        dest = dest.joinpath(source.name)
+        assert dest.joinpath("dir1").exists()
+        assert dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir1", "dir2", "hello").exists()
+
+    def test_rsync_same_popen_twice(
+        self,
+        config: pytest.Config,
+        source: Path,
+        dest: Path,
+        hookrecorder: pytest.HookRecorder,
+        workercontroller: None,
+    ) -> None:
+        hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
+        hm.roots = []
+        hm.setup_nodes(None)  # type: ignore[arg-type]
+        source.joinpath("dir1", "dir2").mkdir(parents=True)
+        source.joinpath("dir1", "dir2", "hello").touch()
+        gw = hm.group[0]
+        hm.rsync(gw, source)
+        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
+        assert call.source == source
+        assert len(call.gateways) == 1
+        assert call.gateways[0] in hm.group
+        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
+
+
+class TestHRSync:
+    def test_hrsync_filter(self, source: Path, dest: Path) -> None:
+        source.joinpath("dir").mkdir()
+        source.joinpath("dir", "file.txt").touch()
+        source.joinpath(".svn").mkdir()
+        source.joinpath(".svn", "entries").touch()
+        source.joinpath(".somedotfile").mkdir()
+        source.joinpath(".somedotfile", "moreentries").touch()
+        source.joinpath("somedir").mkdir()
+        source.joinpath("somedir", "editfile~").touch()
+        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
+        files = list(visit_path(source, recurse=syncer.filter, filter=syncer.filter))
+        names = {x.name for x in files}
+        assert names == {"dir", "file.txt", "somedir"}
+
+    def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
+        gw = execnet.makegateway("execmodel=main_thread_only//popen//chdir=%s" % dest)
+        finished = []
+        rsync = HostRSync(source)
+        rsync.add_target_host(gw, finished=lambda: finished.append(1))
+        source.joinpath("hello.py").write_text("world")
+        rsync.send()
+        gw.exit()
+        assert dest.joinpath(source.name, "hello.py").exists()
+        assert len(finished) == 1
+
+
+class TestNodeManager:
+    @pytest.mark.xfail(run=False)
+    def test_rsync_roots_no_roots(
+        self, pytester: pytest.Pytester, source: Path, dest: Path
+    ) -> None:
+        source.joinpath("dir1").mkdir()
+        source.joinpath("dir1", "file1").write_text("hello")
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        # assert nodemanager.config.topdir == source == config.topdir
+        nodemanager.makegateways()  # type: ignore[attr-defined]
+        nodemanager.rsync_roots()  # type: ignore[call-arg]
+        (p,) = nodemanager.gwmanager.multi_exec(  # type: ignore[attr-defined]
+            "import os ; channel.send(os.getcwd())"
+        ).receive_each()
+        p = Path(p)
+        print("remote curdir", p)
+        assert p == dest.joinpath(config.rootpath.name)
+        assert p.joinpath("dir1").is_dir()
+        assert p.joinpath("dir1", "file1").is_file()
+
+    def test_popen_rsync_subdir(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        dir2 = dir1 / "dir2"
+        dir2.mkdir()
+        dir2.joinpath("hello").touch()
+        for rsyncroot in (dir1, source):
+            shutil.rmtree(str(dest), ignore_errors=True)
+            nodemanager = NodeManager(
+                pytester.parseconfig(
+                    "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
+                )
+            )
+            # calls .rsync_roots()
+            nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+            if rsyncroot == source:
+                dest = dest.joinpath("source")
+            assert dest.joinpath("dir1").exists()
+            assert dest.joinpath("dir1", "dir2").exists()
+            assert dest.joinpath("dir1", "dir2", "hello").exists()
+            nodemanager.teardown_nodes()
+
+    @pytest.mark.parametrize(
+        ["flag", "expects_report"],
+        [
+            ("-q", False),
+            ("", False),
+            ("-v", True),
+        ],
+    )
+    def test_rsync_report(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+        capsys: pytest.CaptureFixture[str],
+        flag: str,
+        expects_report: bool,
+    ) -> None:
+        dir1 = source / "dir1"
+        dir1.mkdir()
+        args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
+        if flag:
+            args.append(flag)
+        nodemanager = NodeManager(pytester.parseconfig(*args))
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
+    def test_init_rsync_roots(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir1", "somefile").mkdir()
+        dir2.joinpath("hello").touch()
+        source.joinpath("bogusdir").mkdir()
+        source.joinpath("bogusdir", "file").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs=dir1/dir2
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+        assert dest.joinpath("dir2").exists()
+        assert not dest.joinpath("dir1").exists()
+        assert not dest.joinpath("bogus").exists()
+
+    def test_rsyncignore(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+    ) -> None:
+        dir2 = source.joinpath("dir1", "dir2")
+        dir2.mkdir(parents=True)
+        source.joinpath("dir5", "dir6").mkdir(parents=True)
+        source.joinpath("dir5", "dir6", "bogus").touch()
+        source.joinpath("dir5", "file").touch()
+        dir2.joinpath("hello").touch()
+        source.joinpath("foo").mkdir()
+        source.joinpath("foo", "bar").touch()
+        source.joinpath("bar").mkdir()
+        source.joinpath("bar", "foo").touch()
+        source.joinpath("tox.ini").write_text(
+            textwrap.dedent(
+                """
+                [pytest]
+                rsyncdirs = dir1 dir5
+                rsyncignore = dir1/dir2 dir5/dir6 foo*
+                """
+            )
+        )
+        config = pytester.parseconfig(source)
+        config.option.rsyncignore = ["bar"]
+        nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+        assert dest.joinpath("dir1").exists()
+        assert not dest.joinpath("dir1", "dir2").exists()
+        assert dest.joinpath("dir5", "file").exists()
+        assert not dest.joinpath("dir6").exists()
+        assert not dest.joinpath("foo").exists()
+        assert not dest.joinpath("bar").exists()
+
+    def test_optimise_popen(
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
+    ) -> None:
+        specs = ["popen"] * 3
+        source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
+        source.joinpath("a").mkdir()
+        config = pytester.parseconfig(source)
+        nodemanager = NodeManager(config, specs)
+        # calls .rysnc_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+        for gwspec in nodemanager.specs:
+            assert gwspec._samefilesystem()
+            assert not gwspec.chdir
+
+    def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            __init__="",
+            test_x="""
+            def test_one():
+                pass
+        """,
+        )
+        reprec = pytester.inline_run(
+            "-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
+        )
+        (rep,) = reprec.getreports("pytest_runtest_logreport")
+        assert rep.passed
+
+    def test_proxy_gateway_setup_nodes(self, pytester: pytest.Pytester) -> None:
+        nodemanager = NodeManager(
+            pytester.parseconfig(
+                "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy"
+            )
+        )
+        nodes = nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+
+        # Proxy gateways are considered as an execnet gateway
+        assert len(nodemanager.group) == 2
+
+        # Proxy gateways do not run workers
+        assert len(nodes) == 1
+
+    def test_proxy_gateway(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            __init__="",
+            test_x="""
+            def test_one():
+                pass
+        """,
+        )
+        reprec = pytester.inline_run(
+            "-d", "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy"
+        )
+        rep = reprec.getreports("pytest_runtest_logreport")
+        assert rep[1].passed
+
+
+class MyWarning(UserWarning):
+    pass
+
+
+@pytest.mark.parametrize(
+    "w_cls",
+    [
+        UserWarning,
+        MyWarning,
+        "Imported",
+        pytest.param(
+            "Nested",
+            marks=pytest.mark.xfail(reason="Nested warning classes are not supported."),
+        ),
+    ],
+)
+def test_unserialize_warning_msg(w_cls: type[Warning] | str) -> None:
+    """Test that warning serialization process works well."""
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        if not isinstance(w_cls, str):
+            warnings.warn("hello", w_cls)
+        elif w_cls == "Imported":
+            generate_warning()
+        elif w_cls == "Nested":
+            # dynamic creation
+            class MyWarning2(UserWarning):
+                pass
+
+            warnings.warn("hello", MyWarning2)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+    w_msg2 = unserialize_warning_message(data)
+
+    # Compare the two objects
+    all_keys = set(vars(w_msg).keys()).union(set(vars(w_msg2).keys()))
+    for k in all_keys:
+        v1 = getattr(w_msg, k)
+        v2 = getattr(w_msg2, k)
+        if k == "message":
+            assert type(v1) is type(v2)
+            assert v1.args == v2.args
+        else:
+            assert v1 == v2
+
+
+class MyWarningUnknown(UserWarning):
+    # Changing the __module__ attribute is only safe if class can be imported
+    # from there
+    __module__ = "unknown"
+
+
+def test_warning_serialization_tweaked_module() -> None:
+    """Test for GH#404."""
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        warnings.warn("hello", MyWarningUnknown)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+
+    # __module__ cannot be found!
+    with pytest.raises(ModuleNotFoundError):
+        unserialize_warning_message(data)

testing/util.py

@@ -0,0 +1,9 @@
+import warnings
+
+
+class MyWarning2(UserWarning):
+    pass
+
+
+def generate_warning() -> None:
+    warnings.warn(MyWarning2("hello"))

tox.ini

@@ -1,23 +1,50 @@
 [tox]
-envlist=py26,py31,py27,py25,py24
-indexserver=
-    testrun = http://pypi.testrun.org
-    pypi = http://pypi.python.org/simple
+envlist =
+  linting
+  py{39,310,311,312,313}-pytestlatest
+  py310-pytestmain
+  py310-psutil
+  py310-setproctitle
+isolated_build = true
 
 [testenv]
-changedir=testing
-deps=:testrun:pytest>=2.2.0.dev2
-commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
+extras =
+  testing
+  psutil: psutil
+  setproctitle: setproctitle
+deps =
+  pytestmin: pytest==7.0.0
+  pytestlatest: pytest
+  pytestmain: git+https://github.com/pytest-dev/pytest.git
+commands =
+  pytest {posargs:{env:_XDIST_TOX_DEFAULT_POSARGS:}}
+setenv =
+  _XDIST_TOX_DEFAULT_POSARGS={env:_XDIST_TOX_POSARGS_PSUTIL:}
+  psutil: _XDIST_TOX_POSARGS_PSUTIL=-k psutil
 
-[testenv:py31]
-deps=:pypi:pytest  # XXX needed because ClueRelease/pip broken
-[testenv:py32]
-deps=:pypi:pytest  # XXX needed because ClueRelease/pip broken
+[testenv:linting]
+skip_install = True
+usedevelop = True
+passenv = PRE_COMMIT_HOME
+deps =
+  pre-commit
+commands = pre-commit run --all-files --show-diff-on-failure
 
-[testenv:py26]
-deps=
-    :testrun:pytest
-    :pypi:pexpect
+[testenv:release]
+changedir =
+description = do a release, required posarg of the version number
+skipsdist = True
+usedevelop = True
+passenv = *
+deps =
+  towncrier
+commands =
+  towncrier build --version {posargs} --yes
 
-[pytest]
-addopts = -rsfxX
+[testenv:docs]
+usedevelop = True
+deps =
+    sphinx
+    sphinx_rtd_theme
+commands =
+    sphinx-build -W --keep-going -b html docs docs/_build/html {posargs:}

xdist/__init__.py

@@ -1,2 +0,0 @@
-#
-__version__ = '1.7.dev1'

xdist/dsession.py

@@ -1,392 +0,0 @@
-import pytest, py
-import sys
-from xdist.slavemanage import NodeManager
-queue = py.builtin._tryimport('queue', 'Queue')
-
-class EachScheduling:
-
-    def __init__(self, numnodes, log=None):
-        self.numnodes = numnodes
-        self.node2collection = {}
-        self.node2pending = {}
-        if log is None:
-            self.log = py.log.Producer("eachsched")
-        else:
-            self.log = log.loadsched
-        self.collection_is_completed = False
-
-    def hasnodes(self):
-        return bool(self.node2pending)
-
-    def addnode(self, node):
-        self.node2collection[node] = None
-
-    def tests_finished(self):
-        if not self.collection_is_completed:
-            return False
-        for items in self.node2pending.values():
-            if items:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        assert not self.collection_is_completed
-        assert self.node2collection[node] is None
-        self.node2collection[node] = list(collection)
-        self.node2pending[node] = []
-        if len(self.node2pending) >= self.numnodes:
-            self.collection_is_completed = True
-
-    def remove_item(self, node, item):
-        self.node2pending[node].remove(item)
-
-    def remove_node(self, node):
-        # KeyError if we didn't get an addnode() yet
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-        crashitem = pending.pop(0)
-        # XXX what about the rest of pending?
-        return crashitem
-
-    def init_distribute(self):
-        assert self.collection_is_completed
-        for node, pending in self.node2pending.items():
-            node.send_runtest_all()
-            pending[:] = self.node2collection[node]
-
-class LoadScheduling:
-    LOAD_THRESHOLD_NEWITEMS = 5
-    ITEM_CHUNKSIZE = 10
-
-    def __init__(self, numnodes, log=None):
-        self.numnodes = numnodes
-        self.node2pending = {}
-        self.node2collection = {}
-        self.pending = []
-        if log is None:
-            self.log = py.log.Producer("loadsched")
-        else:
-            self.log = log.loadsched
-        self.collection_is_completed = False
-
-    def hasnodes(self):
-        return bool(self.node2pending)
-
-    def addnode(self, node):
-        self.node2pending[node] = []
-
-    def tests_finished(self):
-        if not self.collection_is_completed or self.pending:
-            return False
-        for items in self.node2pending.values():
-            if items:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        assert not self.collection_is_completed
-        assert node in self.node2pending
-        self.node2collection[node] = list(collection)
-        if len(self.node2collection) >= self.numnodes:
-            self.collection_is_completed = True
-
-    def remove_item(self, node, item):
-        if item not in self.item2nodes:
-            raise AssertionError(item, self.item2nodes)
-        nodes = self.item2nodes[item]
-        if node in nodes: # the node might have gone down already
-            nodes.remove(node)
-        #if not nodes:
-        #    del self.item2nodes[item]
-        pending = self.node2pending[node]
-        pending.remove(item)
-        # pre-load items-to-test if the node may become ready
-        if self.pending and len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
-            item = self.pending.pop(0)
-            pending.append(item)
-            self.item2nodes.setdefault(item, []).append(node)
-            node.send_runtest(item)
-        #self.log("items waiting for node: %d" %(len(self.pending)))
-        #self.log("item2pending still executing: %s" %(self.item2nodes,))
-        #self.log("node2pending: %s" %(self.node2pending,))
-
-    def remove_node(self, node):
-        pending = self.node2pending.pop(node)
-        # KeyError if we didn't get an addnode() yet
-        for item in pending:
-            l = self.item2nodes[item]
-            l.remove(node)
-            if not l:
-                del self.item2nodes[item]
-        if not pending:
-            return
-        crashitem = pending.pop(0)
-        self.pending.extend(pending)
-        return crashitem
-
-    def init_distribute(self):
-        assert self.collection_is_completed
-        assert not hasattr(self, 'item2nodes')
-        self.item2nodes = {}
-        # XXX allow nodes to have different collections
-        col = list(self.node2collection.values())[0]
-        for node, collection in self.node2collection.items():
-            assert collection == col
-        self.pending = col
-        if not col:
-            return
-        available = list(self.node2pending.items())
-        num_available = self.numnodes
-        max_one_round = num_available * self.ITEM_CHUNKSIZE -1
-        for i, item in enumerate(self.pending):
-            nodeindex = i % num_available
-            node, pending = available[nodeindex]
-            node.send_runtest(item)
-            self.item2nodes.setdefault(item, []).append(node)
-            pending.append(item)
-            if i >= max_one_round:
-                break
-        del self.pending[:i+1]
-
-class Interrupted(KeyboardInterrupt):
-    """ signals an immediate interruption. """
-
-class DSession:
-    def __init__(self, config):
-        self.config = config
-        self.log = py.log.Producer("dsession")
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-        self.shuttingdown = False
-        self.countfailures = 0
-        self.maxfail = config.getvalue("maxfail")
-        self.queue = queue.Queue()
-        try:
-            self.terminal = config.pluginmanager.getplugin("terminalreporter")
-        except KeyError:
-            self.terminal = None
-        else:
-            self.trdist = TerminalDistReporter(config)
-            config.pluginmanager.register(self.trdist, "terminaldistreporter")
-
-    def report_line(self, line):
-        if self.terminal and self.config.option.verbose >= 0:
-            self.terminal.write_line(line)
-
-    @pytest.mark.trylast
-    def pytest_sessionstart(self, session):
-        self.nodemanager = NodeManager(self.config)
-        self.nodemanager.setup_nodes(putevent=self.queue.put)
-
-    def pytest_sessionfinish(self, session):
-        """ teardown any resources after a test run. """
-        nm = getattr(self, 'nodemanager', None) # if not fully initialized
-        if nm is not None:
-            nm.teardown_nodes()
-
-    def pytest_collection(self):
-        # prohibit collection of test items in master process
-        return True
-
-    def pytest_runtestloop(self):
-        numnodes = len(self.nodemanager.specs)
-        dist = self.config.getvalue("dist")
-        if dist == "load":
-            self.sched = LoadScheduling(numnodes, log=self.log)
-        elif dist == "each":
-            self.sched = EachScheduling(numnodes, log=self.log)
-        else:
-            assert 0, dist
-        self.shouldstop = False
-        self.session_finished = False
-        while not self.session_finished:
-            self.loop_once()
-            if self.shouldstop:
-                raise Interrupted(str(self.shouldstop))
-        return True
-
-    def loop_once(self):
-        """ process one callback from one of the slaves. """
-        while 1:
-            try:
-                eventcall = self.queue.get(timeout=2.0)
-                break
-            except queue.Empty:
-                continue
-        callname, kwargs = eventcall
-        assert callname, kwargs
-        method = "slave_" + callname
-        call = getattr(self, method)
-        self.log("calling method: %s(**%s)" % (method, kwargs))
-        call(**kwargs)
-        if self.sched.tests_finished():
-            self.triggershutdown()
-
-    #
-    # callbacks for processing events from slaves
-    #
-
-    def slave_slaveready(self, node, slaveinfo):
-        node.slaveinfo = slaveinfo
-        node.slaveinfo['id'] = node.gateway.id
-        node.slaveinfo['spec'] = node.gateway.spec
-        self.config.hook.pytest_testnodeready(node=node)
-        self.sched.addnode(node)
-        if self.shuttingdown:
-            node.shutdown()
-
-    def slave_slavefinished(self, node):
-        self.config.hook.pytest_testnodedown(node=node, error=None)
-        if node.slaveoutput['exitstatus'] == 2: # keyboard-interrupt
-            self.shouldstop = "%s received keyboard-interrupt" % (node,)
-            self.slave_errordown(node, "keyboard-interrupt")
-            return
-        crashitem = self.sched.remove_node(node)
-        #assert not crashitem, (crashitem, node)
-        if self.shuttingdown and not self.sched.hasnodes():
-            self.session_finished = True
-
-    def slave_errordown(self, node, error):
-        self.config.hook.pytest_testnodedown(node=node, error=error)
-        try:
-            crashitem = self.sched.remove_node(node)
-        except KeyError:
-            pass
-        else:
-            if crashitem:
-                self.handle_crashitem(crashitem, node)
-                #self.report_line("item crashed on node: %s" % crashitem)
-        if not self.sched.hasnodes():
-            self.session_finished = True
-
-    def slave_collectionfinish(self, node, ids):
-        self.sched.addnode_collection(node, ids)
-        if self.terminal:
-            self.trdist.setstatus(node.gateway.spec, "[%d]" %(len(ids)))
-
-        if self.sched.collection_is_completed:
-            if self.terminal:
-                self.trdist.ensure_show_status()
-                self.terminal.write_line("")
-                self.terminal.write_line("scheduling tests via %s" %(
-                    self.sched.__class__.__name__))
-
-            self.sched.init_distribute()
-
-    def slave_logstart(self, node, nodeid, location):
-        self.config.hook.pytest_runtest_logstart(
-            nodeid=nodeid, location=location)
-
-    def slave_testreport(self, node, rep):
-        if not (rep.passed and rep.when != "call"):
-            if rep.when in ("setup", "call"):
-                self.sched.remove_item(node, rep.nodeid)
-        #self.report_line("testreport %s: %s" %(rep.id, rep.status))
-        rep.node = node
-        self.config.hook.pytest_runtest_logreport(report=rep)
-        self._handlefailures(rep)
-
-    def slave_teardownreport(self, node, rep):
-        rep.node = node
-        self.config.hook.pytest__teardown_final_logerror(report=rep)
-
-    def slave_collectreport(self, node, rep):
-        #self.report_line("collectreport %s: %s" %(rep.id, rep.status))
-        #rep.node = node
-        self._handlefailures(rep)
-
-    def _handlefailures(self, rep):
-        if rep.failed:
-            self.countfailures += 1
-            if self.maxfail and self.countfailures >= self.maxfail:
-                self.shouldstop = "stopping after %d failures" % (
-                    self.countfailures)
-
-    def triggershutdown(self):
-        self.log("triggering shutdown")
-        self.shuttingdown = True
-        for node in self.sched.node2pending:
-            node.shutdown()
-
-    def handle_crashitem(self, nodeid, slave):
-        # XXX get more reporting info by recording pytest_runtest_logstart?
-        runner = self.config.pluginmanager.getplugin("runner")
-        fspath = nodeid.split("::")[0]
-        msg = "Slave %r crashed while running %r" %(slave.gateway.id, nodeid)
-        rep = runner.TestReport(nodeid, (fspath, None, fspath), (),
-            "failed", msg, "???")
-        rep.node = slave
-        self.config.hook.pytest_runtest_logreport(report=rep)
-
-class TerminalDistReporter:
-    def __init__(self, config):
-        self.config = config
-        self.tr = config.pluginmanager.getplugin("terminalreporter")
-        self._status = {}
-        self._lastlen = 0
-
-    def write_line(self, msg):
-        self.tr.write_line(msg)
-
-    def ensure_show_status(self):
-        if not self.tr.hasmarkup:
-            self.write_line(self.getstatus())
-
-    def setstatus(self, spec, status, show=True):
-        self._status[spec.id] = status
-        if show and self.tr.hasmarkup:
-            self.rewrite(self.getstatus())
-
-    def getstatus(self):
-        parts = ["%s %s" %(spec.id, self._status[spec.id])
-                   for spec in self._specs]
-        return " / ".join(parts)
-
-    def rewrite(self, line, newline=False):
-        pline = line + " " * max(self._lastlen-len(line), 0)
-        if newline:
-            self._lastlen = 0
-            pline += "\n"
-        else:
-            self._lastlen = len(line)
-        self.tr.rewrite(pline, bold=True)
-
-    def pytest_xdist_setupnodes(self, specs):
-        self._specs = specs
-        for spec in specs:
-            self.setstatus(spec, "I", show=False)
-        self.setstatus(spec, "I", show=True)
-        self.ensure_show_status()
-
-    def pytest_xdist_newgateway(self, gateway):
-        if self.config.option.verbose > 0:
-            rinfo = gateway._rinfo()
-            version = "%s.%s.%s" % rinfo.version_info[:3]
-            self.rewrite("[%s] %s Python %s cwd: %s" % (
-                gateway.id, rinfo.platform, version, rinfo.cwd),
-                newline=True)
-        self.setstatus(gateway.spec, "C")
-
-    def pytest_testnodeready(self, node):
-        if self.config.option.verbose > 0:
-            d = node.slaveinfo
-            infoline = "[%s] Python %s" %(
-                d['id'],
-                d['version'].replace('\n', ' -- '),)
-            self.rewrite(infoline, newline=True)
-        self.setstatus(node.gateway.spec, "ok")
-
-    def pytest_testnodedown(self, node, error):
-        if not error:
-            return
-        self.write_line("[%s] node down: %s" %(node.gateway.id, error))
-
-    #def pytest_xdist_rsyncstart(self, source, gateways):
-    #    targets = ",".join([gw.id for gw in gateways])
-    #    msg = "[%s] rsyncing: %s" %(targets, source)
-    #    self.write_line(msg)
-    #def pytest_xdist_rsyncfinish(self, source, gateways):
-    #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
-    #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
-

xdist/looponfail.py

@@ -1,227 +0,0 @@
-"""
-    Implement -f aka looponfailing for py.test.
-
-    NOTE that we try to avoid loading and depending on application modules
-    within the controlling process (the one that starts repeatedly test
-    processes) otherwise changes to source code can crash
-    the controlling process which should best never happen.
-"""
-
-import py, pytest
-import sys
-import execnet
-
-def looponfail_main(config):
-    remotecontrol = RemoteControl(config)
-    rootdirs = config.getini("looponfailroots")
-    statrecorder = StatRecorder(rootdirs)
-    try:
-        while 1:
-            remotecontrol.loop_once()
-            if not remotecontrol.failures and remotecontrol.wasfailing:
-                continue # the last failures passed, let's immediately rerun all
-            repr_pytest_looponfailinfo(
-                failreports=remotecontrol.failures,
-                rootdirs=rootdirs)
-            statrecorder.waitonchange(checkinterval=2.0)
-    except KeyboardInterrupt:
-        print()
-
-class RemoteControl(object):
-    def __init__(self, config):
-        self.config = config
-        self.failures = []
-
-    def trace(self, *args):
-        if self.config.option.debug:
-            msg = " ".join([str(x) for x in args])
-            py.builtin.print_("RemoteControl:", msg)
-
-    def initgateway(self):
-        return execnet.makegateway("popen")
-
-    def setup(self, out=None):
-        if out is None:
-            out = py.io.TerminalWriter()
-        if hasattr(self, 'gateway'):
-            raise ValueError("already have gateway %r" % self.gateway)
-        self.trace("setting up slave session")
-        self.gateway = self.initgateway()
-        self.channel = channel = self.gateway.remote_exec(init_slave_session,
-            args=self.config.args,
-            option_dict=vars(self.config.option),
-        )
-        remote_outchannel = channel.receive()
-        def write(s):
-            out._file.write(s)
-            out._file.flush()
-        remote_outchannel.setcallback(write)
-
-    def ensure_teardown(self):
-        if hasattr(self, 'channel'):
-            if not self.channel.isclosed():
-                self.trace("closing", self.channel)
-                self.channel.close()
-            del self.channel
-        if hasattr(self, 'gateway'):
-            self.trace("exiting", self.gateway)
-            self.gateway.exit()
-            del self.gateway
-
-    def runsession(self):
-        try:
-            self.trace("sending", self.failures)
-            self.channel.send(self.failures)
-            try:
-                return self.channel.receive()
-            except self.channel.RemoteError:
-                e = sys.exc_info()[1]
-                self.trace("ERROR", e)
-                raise
-        finally:
-            self.ensure_teardown()
-
-    def loop_once(self):
-        self.setup()
-        self.wasfailing = self.failures and len(self.failures)
-        result = self.runsession()
-        failures, reports, collection_failed = result
-        if collection_failed:
-            reports = ["Collection failed, keeping previous failure set"]
-        else:
-            self.failures = failures
-
-def repr_pytest_looponfailinfo(failreports, rootdirs):
-    tr = py.io.TerminalWriter()
-    if failreports:
-        tr.sep("#", "LOOPONFAILING", bold=True)
-        for report in failreports:
-            if report:
-                tr.line(report, red=True)
-    tr.sep("#", "waiting for changes", bold=True)
-    for rootdir in rootdirs:
-        tr.line("### Watching:   %s" %(rootdir,), bold=True)
-
-
-def init_slave_session(channel, args, option_dict):
-    import os, sys
-    import py
-    outchannel = channel.gateway.newchannel()
-    sys.stdout = sys.stderr = outchannel.makefile('w')
-    channel.send(outchannel)
-    # prune sys.path to not contain relative paths
-    newpaths = []
-    for p in sys.path:
-        if p:
-            if not os.path.isabs(p):
-                p = os.path.abspath(p)
-            newpaths.append(p)
-    sys.path[:] = newpaths
-
-    #fullwidth, hasmarkup = channel.receive()
-    from _pytest.config import Config
-    config = Config.fromdictargs(option_dict, list(args))
-    config.args = args
-    from xdist.looponfail import SlaveFailSession
-    SlaveFailSession(config, channel).main()
-
-class SlaveFailSession:
-    def __init__(self, config, channel):
-        self.config = config
-        self.channel = channel
-        self.recorded_failures = []
-        self.collection_failed = False
-        config.pluginmanager.register(self)
-        config.option.looponfail = False
-        config.option.usepdb = False
-
-    def DEBUG(self, *args):
-        if self.config.option.debug:
-            print(" ".join(map(str, args)))
-
-    def pytest_collection(self, session):
-        self.session = session
-        self.trails = self.current_command
-        hook = self.session.ihook
-        try:
-            items = session.perform_collect(self.trails or None)
-        except pytest.UsageError:
-            items = session.perform_collect(None)
-        hook.pytest_collection_modifyitems(session=session, config=session.config, items=items)
-        hook.pytest_collection_finish(session=session)
-        return True
-
-    def pytest_runtest_logreport(self, report):
-        if report.failed:
-            self.recorded_failures.append(report)
-
-    def pytest_collectreport(self, report):
-        if report.failed:
-            self.recorded_failures.append(report)
-            self.collection_failed = True
-
-    def main(self):
-        self.DEBUG("SLAVE: received configuration, waiting for command trails")
-        try:
-            command = self.channel.receive()
-        except KeyboardInterrupt:
-            return # in the slave we can't do much about this
-        self.DEBUG("received", command)
-        self.current_command = command
-        self.config.hook.pytest_cmdline_main(config=self.config)
-        trails, failreports = [], []
-        for rep in self.recorded_failures:
-            trails.append(rep.nodeid)
-            loc = rep.longrepr
-            loc = str(getattr(loc, 'reprcrash', loc))
-            failreports.append(loc)
-        self.channel.send((trails, failreports, self.collection_failed))
-
-class StatRecorder:
-    def __init__(self, rootdirlist):
-        self.rootdirlist = rootdirlist
-        self.statcache = {}
-        self.check() # snapshot state
-
-    def fil(self, p):
-        return p.ext in ('.py', '.txt', '.c', '.h')
-    def rec(self, p):
-        return p.check(dotfile=0)
-
-    def waitonchange(self, checkinterval=1.0):
-        while 1:
-            changed = self.check()
-            if changed:
-                return
-            py.std.time.sleep(checkinterval)
-
-    def check(self, removepycfiles=True):
-        changed = False
-        statcache = self.statcache
-        newstat = {}
-        for rootdir in self.rootdirlist:
-            for path in rootdir.visit(self.fil, self.rec):
-                oldstat = statcache.pop(path, None)
-                try:
-                    newstat[path] = curstat = path.stat()
-                except py.error.ENOENT:
-                    if oldstat:
-                        changed = True
-                else:
-                    if oldstat:
-                       if oldstat.mtime != curstat.mtime or \
-                          oldstat.size != curstat.size:
-                            changed = True
-                            py.builtin.print_("# MODIFIED", path)
-                            if removepycfiles and path.ext == ".py":
-                                pycfile = path + "c"
-                                if pycfile.check():
-                                    pycfile.remove()
-
-                    else:
-                        changed = True
-        if statcache:
-            changed = True
-        self.statcache = newstat
-        return changed
-

xdist/newhooks.py

@@ -1,21 +0,0 @@
-
-def pytest_xdist_setupnodes(config, specs):
-    """ called before any remote node is set up. """
-
-def pytest_xdist_newgateway(gateway):
-    """ called on new raw gateway creation. """
-
-def pytest_xdist_rsyncstart(source, gateways):
-    """ called before rsyncing a directory to remote gateways takes place. """
-
-def pytest_xdist_rsyncfinish(source, gateways):
-    """ called after rsyncing a directory to remote gateways takes place. """
-
-def pytest_configure_node(node):
-    """ configure node information before it gets instantiated. """
-
-def pytest_testnodeready(node):
-    """ Test Node is ready to operate. """
-
-def pytest_testnodedown(node, error):
-    """ Test Node is down. """

xdist/plugin.py

@@ -1,123 +0,0 @@
-import sys
-import py, pytest
-
-def pytest_addoption(parser):
-    group = parser.getgroup("xdist", "distributed and subprocess testing")
-    group._addoption('-f', '--looponfail',
-           action="store_true", dest="looponfail", default=False,
-           help="run tests in subprocess, wait for modified files "
-                "and re-run failing test set until all pass.")
-    group._addoption('-n', dest="numprocesses", metavar="numprocesses",
-           action="store", type="int",
-           help="shortcut for '--dist=load --tx=NUM*popen'")
-    group.addoption('--boxed',
-           action="store_true", dest="boxed", default=False,
-           help="box each test run in a separate process (unix)")
-    group._addoption('--dist', metavar="distmode",
-           action="store", choices=['load', 'each', 'no'],
-           type="choice", dest="dist", default="no",
-           help=("set mode for distributing tests to exec environments.\n\n"
-                 "each: send each test to each available environment.\n\n"
-                 "load: send each test to available environment.\n\n"
-                 "(default) no: run tests inprocess, don't distribute."))
-    group._addoption('--tx', dest="tx", action="append", default=[],
-           metavar="xspec",
-           help=("add a test execution environment. some examples: "
-                 "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
-                 "--tx ssh=user@codespeak.net//chdir=testcache"))
-    group._addoption('-d',
-           action="store_true", dest="distload", default=False,
-           help="load-balance tests.  shortcut for '--dist=load'")
-    group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
-           help="add directory for rsyncing to remote tx nodes.")
-
-    parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
-         ' remote distributed testing.', type="pathlist")
-    parser.addini('rsyncignore', 'list of (relative) paths to be ignored '
-         'for rsyncing.', type="pathlist")
-    parser.addini("looponfailroots", type="pathlist",
-        help="directories to check for changes", default=[py.path.local()])
-
-# -------------------------------------------------------------------------
-# distributed testing hooks
-# -------------------------------------------------------------------------
-def pytest_addhooks(pluginmanager):
-    from xdist import newhooks
-    pluginmanager.addhooks(newhooks)
-
-# -------------------------------------------------------------------------
-# distributed testing initialization
-# -------------------------------------------------------------------------
-
-def pytest_cmdline_main(config):
-    check_options(config)
-    if config.getvalue("looponfail"):
-        from xdist.looponfail import looponfail_main
-        looponfail_main(config)
-        return 2 # looponfail only can get stop with ctrl-C anyway
-
-def pytest_configure(config, __multicall__):
-    __multicall__.execute()
-    if config.getvalue("dist") != "no":
-        from xdist.dsession import DSession
-        session = DSession(config)
-        config.pluginmanager.register(session, "dsession")
-
-def check_options(config):
-    if config.option.numprocesses:
-        config.option.dist = "load"
-        config.option.tx = ['popen'] * int(config.option.numprocesses)
-    if config.option.distload:
-        config.option.dist = "load"
-    val = config.getvalue
-    if not val("collectonly"):
-        usepdb = config.option.usepdb  # a core option
-        if val("looponfail"):
-            if usepdb:
-                raise pytest.UsageError("--pdb incompatible with --looponfail.")
-        elif val("dist") != "no":
-            if usepdb:
-                raise pytest.UsageError("--pdb incompatible with distributing tests.")
-
-
-def pytest_runtest_protocol(item):
-    if item.config.getvalue("boxed"):
-        reports = forked_run_report(item)
-        for rep in reports:
-            item.ihook.pytest_runtest_logreport(report=rep)
-        return True
-
-def forked_run_report(item):
-    # for now, we run setup/teardown in the subprocess
-    # XXX optionally allow sharing of setup/teardown
-    from _pytest.runner import runtestprotocol
-    EXITSTATUS_TESTEXIT = 4
-    import marshal
-    from xdist.remote import serialize_report
-    from xdist.slavemanage import unserialize_report
-    def runforked():
-        try:
-            reports = runtestprotocol(item, log=False)
-        except KeyboardInterrupt:
-            py.std.os._exit(EXITSTATUS_TESTEXIT)
-        return marshal.dumps([serialize_report(x) for x in reports])
-
-    ff = py.process.ForkedFunc(runforked)
-    result = ff.waitfinish()
-    if result.retval is not None:
-        report_dumps = marshal.loads(result.retval)
-        return [unserialize_report("testreport", x) for x in report_dumps]
-    else:
-        if result.exitstatus == EXITSTATUS_TESTEXIT:
-            py.test.exit("forked test item %s raised Exit" %(item,))
-        return [report_process_crash(item, result)]
-
-def report_process_crash(item, result):
-    path, lineno = item._getfslineno()
-    info = "%s:%s: running the test CRASHED with signal %d" %(
-            path, lineno, result.signal)
-    from _pytest import runner
-    call = runner.CallInfo(lambda: 0/0, "???")
-    call.excinfo = info
-    rep = runner.pytest_runtest_makereport(item, call)
-    return rep

xdist/remote.py

@@ -1,135 +0,0 @@
-"""
-    This module is executed in remote subprocesses and helps to
-    control a remote testing session and relay back information.
-    It assumes that 'py' is importable and does not have dependencies
-    on the rest of the xdist code.  This means that the xdist-plugin
-    needs not to be installed in remote environments.
-"""
-
-import sys, os
-
-class SlaveInteractor:
-    def __init__(self, config, channel):
-        self.config = config
-        self.slaveid = config.slaveinput.get('slaveid', "?")
-        self.log = py.log.Producer("slave-%s" % self.slaveid)
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-        self.channel = channel
-        config.pluginmanager.register(self)
-
-    def sendevent(self, name, **kwargs):
-        self.log("sending", name, kwargs)
-        self.channel.send((name, kwargs))
-
-    def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
-            self.log("IERROR> " + line)
-
-    def pytest__teardown_final_logerror(self, report):
-        rep = serialize_report(report)
-        self.sendevent("teardownreport", data=rep)
-
-    def pytest_sessionstart(self, session):
-        self.session = session
-        slaveinfo = getinfodict()
-        self.sendevent("slaveready", slaveinfo=slaveinfo)
-
-    def pytest_sessionfinish(self, __multicall__, exitstatus):
-        self.config.slaveoutput['exitstatus'] = exitstatus
-        res = __multicall__.execute()
-        self.sendevent("slavefinished", slaveoutput=self.config.slaveoutput)
-        return res
-
-    def pytest_collection(self, session):
-        self.sendevent("collectionstart")
-
-    def pytest_runtestloop(self, session):
-        self.log("entering main loop")
-        while 1:
-            name, kwargs = self.channel.receive()
-            self.log("received command %s(**%s)" % (name, kwargs))
-            if name == "runtests":
-                ids = kwargs['ids']
-                for nodeid in ids:
-                    item = self._id2item[nodeid]
-                    self.config.hook.pytest_runtest_protocol(item=item)
-            elif name == "runtests_all":
-                for item in session.items:
-                    self.config.hook.pytest_runtest_protocol(item=item)
-            elif name == "shutdown":
-                break
-        return True
-
-    def pytest_collection_finish(self, session):
-        self._id2item = {}
-        ids = []
-        for item in session.items:
-            self._id2item[item.nodeid] = item
-            ids.append(item.nodeid)
-        self.sendevent("collectionfinish",
-            topdir=str(session.fspath),
-            ids=ids)
-
-    #def pytest_runtest_logstart(self, nodeid, location, fspath):
-    #    self.sendevent("logstart", nodeid=nodeid, location=location)
-
-    def pytest_runtest_logreport(self, report):
-        data = serialize_report(report)
-        self.sendevent("testreport", data=data)
-
-    def pytest_collectreport(self, report):
-        data = serialize_report(report)
-        self.sendevent("collectreport", data=data)
-
-def serialize_report(rep):
-    import py
-    d = rep.__dict__.copy()
-    if hasattr(rep.longrepr, 'toterminal'):
-        d['longrepr'] = str(rep.longrepr)
-    else:
-        d['longrepr'] = rep.longrepr
-    for name in d:
-        if isinstance(d[name], py.path.local):
-            d[name] = str(d[name])
-        elif name == "result":
-            d[name] = None # for now
-    return d
-
-def getinfodict():
-    import platform
-    return dict(
-        version = sys.version,
-        version_info = tuple(sys.version_info),
-        sysplatform = sys.platform,
-        platform = platform.platform(),
-        executable = sys.executable,
-        cwd = os.getcwd(),
-    )
-
-def remote_initconfig(option_dict, args):
-    from _pytest.config import Config
-    option_dict['plugins'].append("no:terminal")
-    config = Config.fromdictargs(option_dict, args)
-    config.option.looponfail = False
-    config.option.usepdb = False
-    config.option.dist = "no"
-    config.option.distload = False
-    config.option.numprocesses = None
-    config.args = args
-    return config
-    
-
-if __name__ == '__channelexec__':
-    slaveinput,args,option_dict = channel.receive()
-    importpath = os.getcwd()
-    sys.path.insert(0, importpath) # XXX only for remote situations
-    os.environ['PYTHONPATH'] = (importpath + os.pathsep +
-        os.environ.get('PYTHONPATH', ''))
-    #os.environ['PYTHONPATH'] = importpath
-    import py
-    config = remote_initconfig(option_dict, args)
-    config.slaveinput = slaveinput
-    config.slaveoutput = {}
-    interactor = SlaveInteractor(config, channel)
-    config.hook.pytest_cmdline_main(config=config)

xdist/slavemanage.py

@@ -1,302 +0,0 @@
-import py, pytest
-import sys, os
-import execnet
-import xdist.remote
-
-from _pytest import runner # XXX load dynamically
-
-class NodeManager(object):
-    EXIT_TIMEOUT = 10
-    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
-        self.config = config
-        self._nodesready = py.std.threading.Event()
-        self.trace = self.config.trace.get("nodemanager")
-        self.group = execnet.Group()
-        if specs is None:
-            specs = self._getxspecs()
-        self.specs = []
-        for spec in specs:
-            if not isinstance(spec, execnet.XSpec):
-                spec = execnet.XSpec(spec)
-            if not spec.chdir and not spec.popen:
-                spec.chdir = defaultchdir
-            self.group.allocate_id(spec)
-            self.specs.append(spec)
-        self.roots = self._getrsyncdirs()
-
-    def rsync_roots(self):
-        """ make sure that all remote gateways
-            have the same set of roots in their
-            current directory.
-        """
-        options = {
-            'ignores': self.config.getini("rsyncignore"),
-            'verbose': self.config.option.verbose,
-        }
-        if self.roots:
-            # send each rsync root
-            for root in self.roots:
-                self.rsync(root, **options)
-
-    def makegateways(self):
-        assert not list(self.group)
-        self.config.hook.pytest_xdist_setupnodes(config=self.config,
-            specs=self.specs)
-        for spec in self.specs:
-            gw = self.group.makegateway(spec)
-            self.config.hook.pytest_xdist_newgateway(gateway=gw)
-
-    def setup_nodes(self, putevent):
-        self.makegateways()
-        self.rsync_roots()
-        self.trace("setting up nodes")
-        for gateway in self.group:
-            node = SlaveController(self, gateway, self.config, putevent)
-            gateway.node = node  # to keep node alive
-            node.setup()
-            self.trace("started node %r" % node)
-
-    def teardown_nodes(self):
-        self.group.terminate(self.EXIT_TIMEOUT)
-
-    def _getxspecs(self):
-        xspeclist = []
-        for xspec in self.config.getvalue("tx"):
-            i = xspec.find("*")
-            try:
-                num = int(xspec[:i])
-            except ValueError:
-                xspeclist.append(xspec)
-            else:
-                xspeclist.extend([xspec[i+1:]] * num)
-        if not xspeclist:
-            raise pytest.UsageError(
-                "MISSING test execution (tx) nodes: please specify --tx")
-        return [execnet.XSpec(x) for x in xspeclist]
-
-    def _getrsyncdirs(self):
-        for spec in self.specs:
-            if not spec.popen or spec.chdir:
-                break
-        else:
-            return []
-        import pytest, _pytest
-        pytestpath = pytest.__file__.rstrip("co")
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
-        config = self.config
-        candidates = [py._pydir,pytestpath,pytestdir]
-        candidates += config.option.rsyncdir
-        rsyncroots = config.getini("rsyncdirs")
-        if rsyncroots:
-            candidates.extend(rsyncroots)
-        roots = []
-        for root in candidates:
-            root = py.path.local(root).realpath()
-            if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: %r" %(root,))
-            if root not in roots:
-                roots.append(root)
-        return roots
-
-    def rsync(self, source, notify=None, verbose=False, ignores=None):
-        """ perform rsync to all remote hosts.
-        """
-        rsync = HostRSync(source, verbose=verbose, ignores=ignores)
-        seen = py.builtin.set()
-        gateways = []
-        for gateway in self.group:
-            spec = gateway.spec
-            if spec.popen and not spec.chdir:
-                # XXX this assumes that sources are python-packages
-                # and that adding the basedir does not hurt
-                gateway.remote_exec("""
-                    import sys ; sys.path.insert(0, %r)
-                """ % os.path.dirname(str(source))).waitclose()
-                continue
-            if spec not in seen:
-                def finished():
-                    if notify:
-                        notify("rsyncrootready", spec, source)
-                rsync.add_target_host(gateway, finished=finished)
-                seen.add(spec)
-                gateways.append(gateway)
-        if seen:
-            self.config.hook.pytest_xdist_rsyncstart(
-                source=source,
-                gateways=gateways,
-            )
-            rsync.send()
-            self.config.hook.pytest_xdist_rsyncfinish(
-                source=source,
-                gateways=gateways,
-            )
-
-class HostRSync(execnet.RSync):
-    """ RSyncer that filters out common files
-    """
-    def __init__(self, sourcedir, *args, **kwargs):
-        self._synced = {}
-        ignores= None
-        if 'ignores' in kwargs:
-            ignores = kwargs.pop('ignores')
-        self._ignores = ignores or []
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
-
-    def filter(self, path):
-        path = py.path.local(path)
-        if not path.ext in ('.pyc', '.pyo'):
-            if not path.basename.endswith('~'):
-                if path.check(dotfile=0):
-                    for x in self._ignores:
-                        if path == x:
-                            break
-                    else:
-                        return True
-
-    def add_target_host(self, gateway, finished=None):
-        remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(gateway, remotepath,
-                                          finishedcallback=finished,
-                                          delete=True,)
-
-    def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
-            path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
-            remotepath = gateway.spec.chdir
-            py.builtin.print_('%s:%s <= %s' %
-                              (gateway.spec, remotepath, path))
-
-
-def make_reltoroot(roots, args):
-    # XXX introduce/use public API for splitting py.test args
-    splitcode = "::"
-    l = []
-    for arg in args:
-        parts = arg.split(splitcode)
-        fspath = py.path.local(parts[0])
-        for root in roots:
-            x = fspath.relto(root)
-            if x or fspath == root:
-                parts[0] = root.basename + "/" + x
-                break
-        else:
-            raise ValueError("arg %s not relative to an rsync root" % (arg,))
-        l.append(splitcode.join(parts))
-    return l
-    
-class SlaveController(object):
-    ENDMARK = -1
-
-    def __init__(self, nodemanager, gateway, config, putevent):
-        self.nodemanager = nodemanager
-        self.putevent = putevent
-        self.gateway = gateway
-        self.config = config
-        self.slaveinput = {'slaveid': gateway.id}
-        self._down = False
-        self.log = py.log.Producer("slavectl-%s" % gateway.id)
-        if not self.config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-
-    def __repr__(self):
-        return "<%s %s>" %(self.__class__.__name__, self.gateway.id,)
-
-    def setup(self):
-        self.log("setting up slave session")
-        spec = self.gateway.spec
-        args = self.config.args
-        if not spec.popen or spec.chdir:
-            args = make_reltoroot(self.nodemanager.roots, args)
-        option_dict = vars(self.config.option)
-        if spec.popen:
-            name = "popen-%s" % self.gateway.id
-            basetemp = self.config._tmpdirhandler.getbasetemp()
-            option_dict['basetemp'] = str(basetemp.join(name))
-        self.config.hook.pytest_configure_node(node=self)
-        self.channel = self.gateway.remote_exec(xdist.remote)
-        self.channel.send((self.slaveinput, args, option_dict))
-        if self.putevent:
-            self.channel.setcallback(self.process_from_remote,
-                endmarker=self.ENDMARK)
-
-    def ensure_teardown(self):
-        if hasattr(self, 'channel'):
-            if not self.channel.isclosed():
-                self.log("closing", self.channel)
-                self.channel.close()
-            #del self.channel
-        if hasattr(self, 'gateway'):
-            self.log("exiting", self.gateway)
-            self.gateway.exit()
-            #del self.gateway
-
-    def send_runtest(self, nodeid):
-        self.sendcommand("runtests", ids=[nodeid])
-
-    def send_runtest_all(self):
-        self.sendcommand("runtests_all",)
-
-    def shutdown(self):
-        if not self._down and not self.channel.isclosed():
-            self.sendcommand("shutdown")
-
-    def sendcommand(self, name, **kwargs):
-        """ send a named parametrized command to the other side. """
-        self.log("sending command %s(**%s)" % (name, kwargs))
-        self.channel.send((name, kwargs))
-
-    def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing %s(**%s)" % (eventname, kwargs))
-        self.putevent((eventname, kwargs))
-
-    def process_from_remote(self, eventcall):
-        """ this gets called for each object we receive from
-            the other side and if the channel closes.
-
-            Note that channel callbacks run in the receiver
-            thread of execnet gateways - we need to
-            avoid raising exceptions or doing heavy work.
-        """
-        try:
-            if eventcall == self.ENDMARK:
-                err = self.channel._getremoteerror()
-                if not self._down:
-                    if not err or isinstance(err, EOFError):
-                        err = "Not properly terminated" # lost connection?
-                    self.notify_inproc("errordown", node=self, error=err)
-                    self._down = True
-                return
-            eventname, kwargs = eventcall
-            if eventname in ("collectionstart"):
-                self.log("ignoring %s(%s)" %(eventname, kwargs))
-            elif eventname == "slaveready":
-                self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname == "slavefinished":
-                self._down = True
-                self.slaveoutput = kwargs['slaveoutput']
-                self.notify_inproc("slavefinished", node=self)
-            #elif eventname == "logstart":
-            #    self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname in ("testreport", "collectreport", "teardownreport"):
-                rep = unserialize_report(eventname, kwargs['data'])
-                self.notify_inproc(eventname, node=self, rep=rep)
-            elif eventname == "collectionfinish":
-                self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
-            else:
-                raise ValueError("unknown event: %s" %(eventname,))
-        except KeyboardInterrupt:
-            # should not land in receiver-thread
-            raise
-        except:
-            excinfo = py.code.ExceptionInfo()
-            py.builtin.print_("!" * 20, excinfo)
-            self.config.pluginmanager.notify_exception(excinfo)
-
-def unserialize_report(name, reportdict):
-    d = reportdict
-    if name == "testreport":
-        return runner.TestReport(**d)
-    elif name == "collectreport":
-        return runner.CollectReport(**d)
-    elif name == "teardownreport":
-        return runner.TeardownErrorReport(**d)