Preserve mirrored pytest-xdist version

for more information, see https://pre-commit.ci

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/deploy.yml

@@ -0,0 +1,57 @@
+name: deploy
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Release version'
+        required: true
+        default: '1.2.3'
+
+jobs:
+
+  package:
+    runs-on: ubuntu-latest
+    env:
+      SETUPTOOLS_SCM_PRETEND_VERSION: ${{ github.event.inputs.version }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build and Check Package
+      uses: hynek/build-and-inspect-python-package@v2.13
+
+  deploy:
+    needs: package
+    runs-on: ubuntu-latest
+    environment: deploy
+    permissions:
+      id-token: write  # For PyPI trusted publishers.
+      contents: write  # For tag.
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download Package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@v1.12.4
+      with:
+        attestations: true
+
+    - name: Push tag
+      run: |
+        git config user.name "pytest bot"
+        git config user.email "pytestbot@gmail.com"
+        git tag --annotate --message=v${{ github.event.inputs.version }} v${{ github.event.inputs.version }} ${{ github.sha }}
+        git push origin v${{ github.event.inputs.version }}
+
+    - name: GitHub Release
+      uses: softprops/action-gh-release@v2
+      with:
+        files: dist/*
+        tag_name: v${{ github.event.inputs.version }}

.github/workflows/main.yml

@@ -1,81 +0,0 @@
-name: build
-
-on: [push, pull_request]
-
-jobs:
-  build:
-
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        tox_env:
-          - "py36-pytestlatest"
-          - "py37-pytestlatest"
-          - "py38-pytestlatest"
-          - "py39-pytestlatest"
-          - "py310-pytestlatest"
-          - "py38-pytestmain"
-          - "py38-psutil"
-          - "py38-setproctitle"
-
-        os: [ubuntu-latest, windows-latest]
-        include:
-          - tox_env: "py36-pytestlatest"
-            python: "3.6"
-          - tox_env: "py37-pytestlatest"
-            python: "3.7"
-          - tox_env: "py38-pytestlatest"
-            python: "3.8"
-          - tox_env: "py39-pytestlatest"
-            python: "3.9"
-          - tox_env: "py310-pytestlatest"
-            python: "3.10-dev"
-          - tox_env: "py38-pytestmain"
-            python: "3.8"
-          - tox_env: "py38-psutil"
-            python: "3.8"
-          - tox_env: "py38-setproctitle"
-            python: "3.8"
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-    - name: Install tox
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox
-    - name: Test
-      run: |
-        tox -e ${{ matrix.tox_env }}
-
-  deploy:
-
-    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
-
-    runs-on: ubuntu-latest
-
-    needs: build
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: "3.7"
-    - name: Install wheel
-      run: |
-        python -m pip install --upgrade pip
-        pip install wheel
-    - name: Build package
-      run: |
-        python setup.py sdist bdist_wheel
-    - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@master
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_token }}

.github/workflows/test.yml

@@ -0,0 +1,93 @@
+name: test
+
+on:
+  push:
+    branches:
+      - master
+      - "test-me-*"
+
+  pull_request:
+    branches:
+      - "*"
+
+# Cancel running jobs for the same workflow and branch.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  package:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Build and Check Package
+      uses: hynek/build-and-inspect-python-package@v2.13
+
+  test:
+
+    needs: [package]
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py39-pytestmin"
+          - "py39-pytestlatest"
+          - "py310-pytestlatest"
+          - "py311-pytestlatest"
+          - "py311-pytestmain"
+          - "py312-pytestlatest"
+          - "py313-pytestlatest"
+          - "py310-psutil"
+          - "py310-setproctitle"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py39-pytestmin"
+            python: "3.9"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py310-pytestlatest"
+            python: "3.10"
+          - tox_env: "py311-pytestlatest"
+            python: "3.11"
+          - tox_env: "py311-pytestmain"
+            python: "3.11"
+          - tox_env: "py312-pytestlatest"
+            python: "3.12"
+          - tox_env: "py313-pytestlatest"
+            python: "3.13"
+          - tox_env: "py310-psutil"
+            python: "3.10"
+          - tox_env: "py310-setproctitle"
+            python: "3.10"
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
+
+    - name: Download Package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+
+    - name: Test
+      shell: bash
+      run: |
+        tox run -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz`

.pre-commit-config.yaml

@@ -1,25 +1,19 @@
 repos:
--   repo: https://github.com/psf/black
-    rev: 21.12b0
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: "v0.12.3"
   hooks:
-    -   id: black
-        args: [--safe, --quiet, --target-version, py35]
+    - id: ruff
+      args: ["--fix"]
+    - id: ruff-format
+-   repo: https://github.com/asottile/blacken-docs
+    rev: 1.19.1
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black==23.1.0]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v5.0.0
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
     -   id: check-yaml
-    -   id: debug-statements
--   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-    -   id: flake8
--   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.1
-    hooks:
-    -   id: pyupgrade
-        args: [--py3-plus]
 -   repo: local
     hooks:
     -   id: rst
@@ -29,11 +23,13 @@ repos:
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v1.16.1
     hooks:
     -   id: mypy
         files: ^(src/|testing/)
         args: []
         additional_dependencies:
-          - pytest>=6.2.0
-          - py>=1.10.0
+          - pytest>=7.0.0
+          - execnet>=2.1.0
+          - types-psutil
+          - setproctitle

.readthedocs.yaml

@@ -0,0 +1,14 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - path: .
+    - requirements: docs/requirements.txt

CHANGELOG.rst

@@ -1,6 +1,292 @@
+pytest-xdist 3.8.0 (2025-06-30)
+===============================
+
+Features
+--------
+
+- `#1083 <https://github.com/pytest-dev/pytest-xdist/issues/1083>`_: Add ``--no-loadscope-reorder`` and ``--loadscope-reorder`` option to control whether to automatically reorder tests in loadscope for tests where relative ordering matters. This only applies when using ``loadscope``. 
+
+  For example, [test_file_1, test_file_2, ..., test_file_n] are given as input test files, if ``--no-loadscope-reorder`` is used, for either worker, the ``test_file_a`` will be executed before ``test_file_b`` only if ``a < b``.
+
+  The default behavior is to reorder the tests to maximize the number of tests that can be executed in parallel.
+
+pytest-xdist 3.7.0 (2025-05-26)
+===============================
+
+Features
+--------
+
+- `#1142 <https://github.com/pytest-dev/pytest-xdist/issues/1142>`_: Added support for Python 3.13.
+
+- `#1144 <https://github.com/pytest-dev/pytest-xdist/issues/1144>`_: The internal `steal` command is now atomic - it unschedules either all requested tests or none.
+
+  This is a prerequisite for group/scope support in the `worksteal` scheduler, so test groups won't be broken up incorrectly.
+
+- `#1170 <https://github.com/pytest-dev/pytest-xdist/issues/1170>`_: Add the `--px` arg to create proxy gateways.
+
+  Proxy gateways are passed to additional gateways using the `via` keyword.
+  They can serve as a way to run multiple workers on remote machines.
+
+- `#1200 <https://github.com/pytest-dev/pytest-xdist/issues/1200>`_: Now multiple ``xdist_group`` markers are considered when assigning tests to groups (order does not matter).
+
+  Previously, only the last marker would assign a test to a group, but now if a test has multiple ``xdist_group`` marks applied (for example via parametrization or via fixtures), they are merged to make a new group.
+
+
+Removals
+--------
+
+- `#1162 <https://github.com/pytest-dev/pytest-xdist/issues/1162>`_: Dropped support for EOL Python 3.8.
+
+
+Trivial Changes
+---------------
+
+- `#1092 <https://github.com/pytest-dev/pytest-xdist/issues/1092>`_: Update an error message to better indicate where users should go for more information.
+
+- `#1190 <https://github.com/pytest-dev/pytest-xdist/issues/1190>`_: Switched to using a SPDX license identifier introduced in PEP 639.
+
+pytest-xdist 3.6.1 (2024-04-28)
+===============================
+
+Bug Fixes
+---------
+
+- `#1071 <https://github.com/pytest-dev/pytest-xdist/issues/1071>`_: Add backward compatibility for deadlock issue with the ``execnet`` new ``main_thread_only`` "execmodel" triggered when pytest-cov accesses rinfo.
+
+pytest-xdist 3.6.0 (2024-04-19)
+===============================
+
+This release was YANKED due to a regression fixed in 3.6.1.
+
+Features
+--------
+
+- `#1027 <https://github.com/pytest-dev/pytest-xdist/pull/1027>`_:``pytest-xdist`` workers now always execute the tests in the main thread.
+  Previously some tests might end up executing in a separate thread other than ``main`` in the workers, due to some internal `execnet`` details. This can cause problems specially with async frameworks where the event loop is running in the ``main`` thread (for example `#620 <https://github.com/pytest-dev/pytest-xdist/issues/620>`__).
+
+Bug Fixes
+---------
+
+- `#1024 <https://github.com/pytest-dev/pytest-xdist/issues/1024>`_: Added proper handling of ``shouldstop`` (such as set by ``--max-fail``) and ``shouldfail`` conditions in workers.
+  Previously, a worker might have continued executing further tests before the controller could terminate the session.
+
+- `#1028 <https://github.com/pytest-dev/pytest-xdist/issues/1028>`_: Fixed compatibility issue between `looponfail` and editable installs.
+
+- `#620 <https://github.com/pytest-dev/pytest-xdist/issues/620>`_: Use the new ``main_thread_only`` ``execnet`` "execmodel" so that code which expects to only run in the main thread will now work as expected.
+
+- `#937 <https://github.com/pytest-dev/pytest-xdist/issues/937>`_: Fixed a bug where plugin would raise an incompatibility error with ``--pdb`` despite using ``-n0``.
+
+
+Removals
+--------
+
+- `#1053 <https://github.com/pytest-dev/pytest-xdist/issues/1053>`_: Dropped support for Python 3.7.
+
+- `#1057 <https://github.com/pytest-dev/pytest-xdist/issues/1057>`_: pytest>=7.0.0 is now required.
+
+  execnet>=2.1.0 is now required.
+
+
+Trivial Changes
+---------------
+
+- `#1020 <https://github.com/pytest-dev/pytest-xdist/issues/1020>`_: pytest-xdist's ``setup.py`` file is removed.
+
+  If you relied on this file, e.g. to install pytest using ``setup.py install``,
+  please see `Why you shouldn't invoke setup.py directly <https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html#summary>`_ for alternatives.
+
+- `#1057 <https://github.com/pytest-dev/pytest-xdist/issues/1057>`_: The internals of pytest-xdist are now fully typed. The typing is not exposed yet.
+
+- `#996 <https://github.com/pytest-dev/pytest-xdist/issues/996>`_: Adjusted license file format and content to ensure security scanners will identity the license.
+
+pytest-xdist 3.5.0 (2023-11-21)
+===============================
+
+Features
+--------
+
+- `#632 <https://github.com/pytest-dev/pytest-xdist/issues/632>`_: ``--dist=loadscope`` now sorts scopes by number of tests to assign largest scopes early -- in many cases this should improve overall test session running time, as there is less chance of a large scope being left to be processed near the end of the session, leaving other workers idle.
+
+
+pytest-xdist 3.4.0 (2023-11-11)
+===============================
+
+Features
+--------
+
+- `#963 <https://github.com/pytest-dev/pytest-xdist/issues/963>`_: Wait for workers to finish reporting when test run stops early.
+
+  This makes sure that the results of in-progress tests are displayed.
+  Previously these reports were being discarded, losing information about the
+  test run.
+
+- `#965 <https://github.com/pytest-dev/pytest-xdist/issues/965>`_: Added support for Python 3.12.
+
+
+pytest-xdist 3.3.1 (2023-05-19)
+===============================
+
+Bug Fixes
+---------
+
+- `#907 <https://github.com/pytest-dev/pytest-xdist/issues/907>`_: Avoid remote calls during startup as ``execnet`` by default does not ensure remote affinity with the
+  main thread and might accidentally schedule the pytest worker into a non-main thread, which breaks numerous frameworks,
+  for example ``asyncio``, ``anyio``, ``PyQt/PySide``, etc.
+
+  A more safe correction will require thread affinity in ``execnet`` (`pytest-dev/execnet#96 <https://github.com/pytest-dev/execnet/issues/96>`__).
+
+
+pytest-xdist 3.3.0 (2023-05-12)
+===============================
+
+Features
+--------
+
+- `#555 <https://github.com/pytest-dev/pytest-xdist/issues/555>`_: Improved progress output when collecting nodes to be less verbose.
+
+
+pytest-xdist 3.2.1 (2023-03-12)
+===============================
+
+Bug Fixes
+---------
+
+- `#884 <https://github.com/pytest-dev/pytest-xdist/issues/884>`_: Fixed hang in ``worksteal`` scheduler.
+
+
+pytest-xdist 3.2.0 (2023-02-07)
+===============================
+
+Improved Documentation
+----------------------
+
+- `#863 <https://github.com/pytest-dev/pytest-xdist/issues/863>`_: Document limitations for debugging due to standard I/O of workers not being forwarded. Also, mention remote debugging as a possible workaround.
+
+
+Features
+--------
+
+- `#855 <https://github.com/pytest-dev/pytest-xdist/issues/855>`_: Users can now configure ``load`` scheduling precision using ``--maxschedchunk`` command
+  line option.
+
+- `#858 <https://github.com/pytest-dev/pytest-xdist/issues/858>`_: New ``worksteal`` scheduler, based on the idea of `work stealing <https://en.wikipedia.org/wiki/Work_stealing>`_. It's similar to ``load`` scheduler, but it should handle tests with significantly differing duration better, and, at the same time, it should provide similar or better reuse of fixtures.
+
+
+Trivial Changes
+---------------
+
+- `#870 <https://github.com/pytest-dev/pytest-xdist/issues/870>`_: Make the tests pass even when ``$PYTEST_XDIST_AUTO_NUM_WORKERS`` is set.
+
+
+pytest-xdist 3.1.0 (2022-12-01)
+===============================
+
+Features
+--------
+
+- `#789 <https://github.com/pytest-dev/pytest-xdist/issues/789>`_: Users can now set a default distribution mode in their configuration file:
+
+  .. code-block:: ini
+
+      [pytest]
+      addopts = --dist loadscope
+
+- `#842 <https://github.com/pytest-dev/pytest-xdist/issues/842>`_: Python 3.11 is now officially supported.
+
+
+Removals
+--------
+
+- `#842 <https://github.com/pytest-dev/pytest-xdist/issues/842>`_: Python 3.6 is no longer supported.
+
+
+pytest-xdist 3.0.2 (2022-10-25)
+===============================
+
+Bug Fixes
+---------
+
+- `#813 <https://github.com/pytest-dev/pytest-xdist/issues/813>`_: Cancel shutdown when a crashed worker is restarted.
+
+
+Deprecations
+------------
+
+- `#825 <https://github.com/pytest-dev/pytest-xdist/issues/825>`_: The ``--rsyncdir`` command line argument and ``rsyncdirs`` config variable are deprecated.
+
+  The rsync feature will be removed in pytest-xdist 4.0.
+
+- `#826 <https://github.com/pytest-dev/pytest-xdist/issues/826>`_: The ``--looponfail`` command line argument and ``looponfailroots`` config variable are deprecated.
+
+  The loop-on-fail feature will be removed in pytest-xdist 4.0.
+
+
+Improved Documentation
+----------------------
+
+- `#791 <https://github.com/pytest-dev/pytest-xdist/issues/791>`_: Document the ``pytest_xdist_auto_num_workers`` hook.
+
+- `#796 <https://github.com/pytest-dev/pytest-xdist/issues/796>`_: Added known limitations section to documentation.
+
+- `#829 <https://github.com/pytest-dev/pytest-xdist/issues/829>`_: Document the ``-n logical`` option.
+
+
+Features
+--------
+
+- `#792 <https://github.com/pytest-dev/pytest-xdist/issues/792>`_: The environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` can now be used to
+  specify the default for ``-n auto`` and ``-n logical``.
+
+- `#812 <https://github.com/pytest-dev/pytest-xdist/issues/812>`_: Partially restore old initial batch distribution algorithm in ``LoadScheduling``.
+
+  pytest orders tests for optimal sequential execution - i. e. avoiding
+  unnecessary setup and teardown of fixtures. So executing tests in consecutive
+  chunks is important for optimal performance.
+
+  In v1.14, initial test distribution in ``LoadScheduling`` was changed to
+  round-robin, optimized for the corner case, when the number of tests is less
+  than ``2 * number of nodes``. At the same time, it became worse for all other
+  cases.
+
+  For example: if some tests use some "heavy" fixture, and these tests fit into
+  the initial batch, with round-robin distribution the fixture will be created
+  ``min(n_tests, n_workers)`` times, no matter how many other tests there are.
+
+  With the old algorithm (before v1.14), if there are enough tests not using
+  the fixture, the fixture was created only once.
+
+  So restore the old behavior for typical cases where the number of tests is
+  much greater than the number of workers (or, strictly speaking, when there
+  are at least 2 tests for every node).
+
+
+Removals
+--------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command-line option has been removed. If you still need this functionality, install `pytest-forked <https://pypi.org/project/pytest-forked>`__ separately.
+
+
+Trivial Changes
+---------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``py`` dependency has been dropped.
+
+- `#822 <https://github.com/pytest-dev/pytest-xdist/issues/822>`_: Replace internal usage of ``py.log`` with a custom solution (but with the same interface).
+
+- `#823 <https://github.com/pytest-dev/pytest-xdist/issues/823>`_: Remove usage of ``py._pydir`` as an rsync candidate.
+
+- `#824 <https://github.com/pytest-dev/pytest-xdist/issues/824>`_: Replace internal usages of ``py.path.local`` by ``pathlib.Path``.
+
+
 pytest-xdist 2.5.0 (2021-12-10)
 ===============================
 
+Deprecations and Removals
+-------------------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command line argument is deprecated. Install `pytest-forked <https://pypi.org/project/pytest-forked>`__ and use ``--forked`` instead. pytest-xdist 3.0.0 will remove the ``--boxed`` argument and ``pytest-forked`` dependency.
+
+
 Features
 --------
 

LICENSE

@@ -1,3 +1,6 @@
+MIT License
+
+Copyright (c) 2010 Holger Krekel and contributors.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

MANIFEST.in

@@ -1,5 +0,0 @@
-exclude .appveyor.yml
-exclude .gitignore
-exclude .pre-commit-config.yaml
-exclude .travis.yml
-prune .github

README.rst

@@ -13,7 +13,7 @@ pytest-xdist
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/test/badge.svg
     :target: https://github.com/pytest-dev/pytest-xdist/actions
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
@@ -25,520 +25,9 @@ tests across multiple CPUs to speed up test execution::
     pytest -n auto
 
 With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
-the tests randomly across them. There is also a number of `distribution modes`_ to choose from.
+the tests randomly across them.
 
-**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+Documentation
+=============
 
-.. contents:: **Table of Contents**
-
-Installation
-------------
-
-Install the plugin with::
-
-    pip install pytest-xdist
-
-
-To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
-
-    pip install pytest-xdist[psutil]
-
-
-Features
---------
-
-* Test run parallelization_: tests can be executed across  multiple CPUs or hosts.
-  This allows to speed up development or to use special resources of `remote machines`_.
-
-* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
-  pytest waits until a file in your project changes and then re-runs
-  the previously failing tests.  This is repeated until all tests pass
-  after which again a full run is performed.
-
-* `Multi-Platform`_ coverage: you can specify different Python interpreters
-  or different platforms and run tests in parallel on all of them.
-
-  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
-  program source code to the remote place.
-  You may specify different Python versions and interpreters. It does not
-  installs/synchronize dependencies however.
-
-  **Note**: this mode exists mostly for backward compatibility, as modern development
-  relies on continuous integration for multi-platform testing.
-
-.. _parallelization:
-
-Running tests across multiple CPUs
-----------------------------------
-
-To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
-
-    pytest -n 8
-
-Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
-can lead to considerable speed ups, especially if your test suite takes a
-noticeable amount of time.
-
-The test distribution algorithm is configured with the ``--dist`` command-line option:
-
-.. _distribution modes:
-
-* ``--dist load`` **(default)**: Sends pending tests to any worker that is
-  available, without any guaranteed order.
-
-* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
-  and by **class** for *test methods*. Groups are distributed to available
-  workers as whole units. This guarantees that all tests in a group run in the
-  same process. This can be useful if you have expensive module-level or
-  class-level fixtures. Grouping by class takes priority over grouping by
-  module.
-
-* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
-  distributed to available workers as whole units. This guarantees that all
-  tests in a file run in the same worker.
-
-* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
-  distributed to available workers as whole units. This guarantees that all
-  tests with same ``xdist_group`` name run in the same worker.
-
-  .. code-block:: python
-
-      @pytest.mark.xdist_group(name="group1")
-      def test1():
-          pass
-
-      class TestA:
-          @pytest.mark.xdist_group("group1")
-          def test2():
-              pass
-
-  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
-  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
-
-* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).
-
-
-Running tests in a Python subprocess
-------------------------------------
-
-To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
-
-    pytest -d --tx popen//python=python3.9
-
-This will start a subprocess which is run with the ``python3.9``
-Python interpreter, found in your system binary lookup path.
-
-If you prefix the --tx option value like this::
-
-    --tx 3*popen//python=python3.9
-
-then three subprocesses would be created and tests
-will be load-balanced across these three processes.
-
-.. _boxed:
-
-Running tests in a boxed subprocess
------------------------------------
-
-This functionality has been moved to the
-`pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ plugin, but the ``--boxed`` option
-is still kept for backward compatibility.
-
-.. _`remote machines`:
-
-Sending tests to remote SSH accounts
-------------------------------------
-
-Suppose you have a package ``mypkg`` which contains some
-tests that you can successfully run locally. And you
-have a ssh-reachable machine ``myhost``.  Then
-you can ad-hoc distribute your tests by typing::
-
-    pytest -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
-
-This will synchronize your :code:`mypkg` package directory
-to a remote ssh account and then locally collect tests
-and send them to remote places for execution.
-
-You can specify multiple :code:`--rsyncdir` directories
-to be sent to the remote side.
-
-.. note::
-
-  For pytest to collect and send tests correctly
-  you not only need to make sure all code and tests
-  directories are rsynced, but that any test (sub) directory
-  also has an :code:`__init__.py` file because internally
-  pytest references tests as a fully qualified python
-  module path.  **You will otherwise get strange errors**
-  during setup of the remote side.
-
-
-You can specify multiple :code:`--rsyncignore` glob patterns
-to be ignored when file are sent to the remote side.
-There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
-Those you cannot override using rsyncignore command-line or
-ini-file option(s).
-
-
-Sending tests to remote Socket Servers
---------------------------------------
-
-Download the single-module `socketserver.py`_ Python program
-and run it like this::
-
-    python socketserver.py
-
-It will tell you that it starts listening on the default
-port.  You can now on your home machine specify this
-new socket host with something like this::
-
-    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
-
-
-.. _`atonce`:
-.. _`Multi-Platform`:
-
-
-Running tests on many platforms at once
----------------------------------------
-
-The basic command to run tests on multiple platforms is::
-
-    pytest --dist=each --tx=spec1 --tx=spec2
-
-If you specify a windows host, an OSX host and a Linux
-environment this command will send each tests to all
-platforms - and report back failures from all platforms
-at once. The specifications strings use the `xspec syntax`_.
-
-.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
-
-.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
-
-.. _`execnet`: https://codespeak.net/execnet
-
-
-When tests crash
-----------------
-
-If a test crashes a worker, pytest-xdist will automatically restart that worker
-and report the test’s failure. You can use the ``--max-worker-restart`` option
-to limit the number of worker restarts that are allowed, or disable restarting
-altogether using ``--max-worker-restart 0``.
-
-
-How-tos
--------
-
-Identifying the worker process during a test
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 1.15.*
-
-If you need to determine the identity of a worker process in
-a test or fixture, you may use the ``worker_id`` fixture to do so:
-
-.. code-block:: python
-
-    @pytest.fixture()
-    def user_account(worker_id):
-        """ use a different account in each xdist worker """
-        return "account_%s" % worker_id
-
-When ``xdist`` is disabled (running with ``-n0`` for example), then
-``worker_id`` will return ``"master"``.
-
-Worker processes also have the following environment variables
-defined:
-
-* ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
-* ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session,
-  e.g., ``"4"`` when ``-n 4`` is given in the command-line.
-
-The information about the worker_id in a test is stored in the ``TestReport`` as
-well, under the ``worker_id`` attribute.
-
-Since version 2.0, the following functions are also available in the ``xdist`` module:
-
-.. code-block:: python
-
-    def is_xdist_worker(request_or_session) -> bool:
-        """Return `True` if this is an xdist worker, `False` otherwise
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-     def is_xdist_controller(request_or_session) -> bool:
-        """Return `True` if this is the xdist controller, `False` otherwise
-
-        Note: this method also returns `False` when distribution has not been
-        activated at all.
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-    def is_xdist_master(request_or_session) -> bool:
-        """Deprecated alias for is_xdist_controller."""
-
-    def get_xdist_worker_id(request_or_session) -> str:
-        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-        if running on the controller node.
-
-        If not distributing tests (for example passing `-n0` or not passing `-n` at all)
-        also return 'master'.
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-
-Identifying workers from the system environment
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 2.4*
-
-If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
-update the process title (command line) on its workers to show their current
-state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
-``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
-Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
-pointer regarding the Process Explorer tool.
-
-This is intended purely as an UX enhancement, e.g. to track down issues with
-long-running or CPU intensive tests.  Errors in changing the title are ignored
-silently.  Please try not to rely on the title format or title changes in
-external scripts.
-
-.. _`setproctitle`: https://pypi.org/project/setproctitle/
-
-
-Uniquely identifying the current test run
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 1.32.*
-
-If you need to globally distinguish one test run from others in your
-workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
-wanted to create a separate database for each test run:
-
-.. code-block:: python
-
-    import pytest
-    from posix_ipc import Semaphore, O_CREAT
-
-    @pytest.fixture(scope="session", autouse=True)
-    def create_unique_database(testrun_uid):
-        """ create a unique database for this particular test run """
-        database_url = f"psql://myapp-{testrun_uid}"
-
-        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
-            if not database_exists(database_url):
-                create_database(database_url)
-
-    @pytest.fixture()
-    def db(testrun_uid):
-        """ retrieve unique database """
-        database_url = f"psql://myapp-{testrun_uid}"
-        return database_get_instance(database_url)
-
-
-Additionally, during a test run, the following environment variable is defined:
-
-* ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
-
-Accessing ``sys.argv`` from the controller node in workers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To access the ``sys.argv`` passed to the command-line of the controller node, use
-``request.config.workerinput["mainargv"]``.
-
-
-Specifying test exec environments in an ini file
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-You can use pytest's ini file configuration to avoid typing common options.
-You can for example make running with three subprocesses your default like this:
-
-.. code-block:: ini
-
-    [pytest]
-    addopts = -n3
-
-You can also add default environments like this:
-
-.. code-block:: ini
-
-    [pytest]
-    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
-
-and then just type::
-
-    pytest --dist=each
-
-to run tests in each of the environments.
-
-
-Specifying "rsync" dirs in an ini-file
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
-you may specify directories to include or to exclude in synchronisation:
-
-.. code-block:: ini
-
-    [pytest]
-    rsyncdirs = . mypkg helperpkg
-    rsyncignore = .hg
-
-These directory specifications are relative to the directory
-where the configuration file was found.
-
-.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
-.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
-.. _`pytest`: http://pytest.org
-
-
-Making session-scoped fixtures execute only once
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
-a subset of all tests. This means that tests in different processes requesting a high-level
-scoped fixture (for example ``session``) will execute the fixture code more than once, which
-breaks expectations and might be undesired in certain situations.
-
-While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
-executed exactly once, this can be achieved by using a lock file for inter-process communication.
-
-The example below needs to execute the fixture ``session_data`` only once (because it is
-resource intensive, or needs to execute only once to define configuration options, etc), so it makes
-use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
-when the first process requests the fixture, while the other processes will then read
-the data from a file.
-
-Here is the code:
-
-.. code-block:: python
-
-    import json
-
-    import pytest
-    from filelock import FileLock
-
-
-    @pytest.fixture(scope="session")
-    def session_data(tmp_path_factory, worker_id):
-        if worker_id == "master":
-            # not executing in with multiple workers, just produce the data and let
-            # pytest's fixture caching do its job
-            return produce_expensive_data()
-
-        # get the temp directory shared by all workers
-        root_tmp_dir = tmp_path_factory.getbasetemp().parent
-
-        fn = root_tmp_dir / "data.json"
-        with FileLock(str(fn) + ".lock"):
-            if fn.is_file():
-                data = json.loads(fn.read_text())
-            else:
-                data = produce_expensive_data()
-                fn.write_text(json.dumps(data))
-        return data
-
-
-The example above can also be use in cases a fixture needs to execute exactly once per test session, like
-initializing a database service and populating initial tables.
-
-This technique might not work for every case, but should be a starting point for many situations
-where executing a high-scope fixture exactly once is important.
-
-
-How does xdist work?
---------------------
-
-``xdist`` works by spawning one or more **workers**, which are
-controlled by the **controller**. Each **worker** is responsible for
-performing a full test collection and afterwards running tests as
-dictated by the **controller**.
-
-The execution flow is:
-
-1. **controller** spawns one or more **workers** at the beginning of the
-   test session. The communication between **controller** and **worker**
-   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
-   its
-   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
-   The actual interpreters executing the code for the **workers** might
-   be remote or local.
-
-2. Each **worker** itself is a mini pytest runner. **workers** at this
-   point perform a full test collection, sending back the collected
-   test-ids back to the **controller** which does not perform any
-   collection itself.
-
-3. The **controller** receives the result of the collection from all
-   nodes. At this point the **controller** performs some sanity check to
-   ensure that all **workers** collected the same tests (including
-   order), bailing out otherwise. If all is well, it converts the list
-   of test-ids into a list of simple indexes, where each index
-   corresponds to the position of that test in the original collection
-   list. This works because all nodes have the same collection list, and
-   saves bandwidth because the **controller** can now tell one of the
-   workers to just *execute test index 3* index of passing the full test
-   id.
-
-4. If **dist-mode** is **each**: the **controller** just sends the full
-   list of test indexes to each node at this moment.
-
-5. If **dist-mode** is **load**: the **controller** takes around 25% of
-   the tests and sends them one by one to each **worker** in a round
-   robin fashion. The rest of the tests will be distributed later as
-   **workers** finish tests (see below).
-
-6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
-   implement custom tests distribution logic.
-
-7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
-   implementation basically loops over all collected items in the
-   ``session`` object and executes the ``pytest_runtest_protocol`` for
-   each test item, but in xdist **workers** sit idly waiting for
-   **controller** to send tests for execution. As tests are received by
-   **workers**, ``pytest_runtest_protocol`` is executed for each test.
-   Here it worth noting an implementation detail: **workers** always
-   must keep at least one test item on their queue due to how the
-   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
-   to pass the ``nextitem`` to the hook, the worker must wait for more
-   instructions from controller before executing that remaining test. If
-   it receives more tests, then it can safely call
-   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
-   parameter will be. If it receives a “shutdown” signal, then it can
-   execute the hook passing ``nextitem`` as ``None``.
-
-8. As tests are started and completed at the **workers**, the results
-   are sent back to the **controller**, which then just forwards the
-   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
-   and ``pytest_runtest_logreport``. This way other plugins (for example
-   ``junitxml``) can work normally. The **controller** (when in
-   dist-mode **load**) decides to send more tests to a node when a test
-   completes, using some heuristics such as test durations and how many
-   tests each **worker** still has to run.
-
-9. When the **controller** has no more pending tests it will send a
-   “shutdown” signal to all **workers**, which will then run their
-   remaining tests to completion and shut down. At this point the
-   **controller** will sit waiting for **workers** to shut down, still
-   processing events such as ``pytest_runtest_logreport``.
-
-FAQ
----
-
-**Question**: Why does each worker do its own collection, as opposed to having the
-controller collect once and distribute from that collection to the
-workers?
-
-If collection was performed by controller then it would have to
-serialize collected items to send them through the wire, as workers live
-in another process. The problem is that test items are not easily
-(impossible?) to serialize, as they contain references to the test
-functions, fixture managers, config objects, etc. Even if one manages to
-serialize it, it seems it would be very hard to get it right and easy to
-break by any small change in pytest.
+Documentation is available at `Read The Docs <https://pytest-xdist.readthedocs.io>`__.

RELEASING.rst

@@ -32,14 +32,10 @@ To publish a new release ``X.Y.Z``, the steps are as follows:
 
     $ tox -e release -- X.Y.Z
 
-#. Commit and push the branch for review.
+#. Commit and push the branch to ``upstream`` and open a PR.
 
-#. Once PR is **green** and **approved**, create and push a tag::
+#. Once the PR is **green** and **approved**, start the ``deploy`` workflow manually from the branch ``release-VERSION``, passing ``VERSION`` as parameter, or execute::
 
-    $ export VERSION=X.Y.Z
-    $ git tag v$VERSION release-$VERSION
-    $ git push git@github.com:pytest-dev/pytest-xdist.git v$VERSION
+   gh workflow run deploy.yml -R pytest-dev/pytest-xdist --ref release-X.Y.Z --field version=X.Y.Z
 
-That will build the package and publish it on ``PyPI`` automatically.
-
-#. Merge the release PR to `master`.
+#. Merge the release PR to ``master``.

changelog/468.deprecation.rst

@@ -1,3 +0,0 @@
-The ``--boxed`` commmand line argument is deprecated.
-Install pytest-forked and use ``--forked`` instead.
-pytest-xdist 3.0.0 will remove the ``--boxed`` argument and pytest-forked dependency.

docs/.gitignore

@@ -0,0 +1 @@
+_build/

docs/changelog.rst

@@ -0,0 +1,5 @@
+=========
+Changelog
+=========
+
+.. include:: ../CHANGELOG.rst

docs/conf.py

@@ -0,0 +1,55 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "pytest-xdist"
+copyright = "2010, holger krekel and contributors"
+author = "holger krekel and contributors"
+
+master_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx_rtd_theme",
+    "sphinx.ext.autodoc",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']

docs/crash.rst

@@ -0,0 +1,7 @@
+When tests crash
+================
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.

docs/distribution.rst

@@ -0,0 +1,130 @@
+.. _parallelization:
+
+Running tests across multiple CPUs
+==================================
+
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
+
+    pytest -n auto
+
+This can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
+
+With ``-n auto``, pytest-xdist will use as many processes as your computer
+has physical CPU cores.
+
+Use ``-n logical`` to use the number of *logical* CPU cores rather than
+physical ones. This currently requires the `psutil <https://pypi.org/project/psutil/>`__ package to be installed;
+if it is not or if it fails to determine the number of logical CPUs, fall back to ``-n auto`` behavior.
+
+Pass a number, e.g. ``-n 8``, to specify the number of processes explicitly.
+
+Use ``-n 0`` to disable xdist and run all tests in the main process.
+
+To specify a different meaning for ``-n auto`` and ``-n logical`` for your
+tests, you can:
+
+* Set the environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` to the
+  desired number of processes.
+
+* Implement the ``pytest_xdist_auto_num_workers``
+  `pytest hook <https://docs.pytest.org/en/latest/how-to/writing_plugins.html>`__
+  (a ``pytest_xdist_auto_num_workers(config)`` function in e.g. ``conftest.py``)
+  that returns the number of processes to use.
+  The hook can use ``config.option.numprocesses`` to determine if the user
+  asked for ``"auto"`` or ``"logical"``, and it can return ``None`` to fall
+  back to the default.
+
+If both the hook and environment variable are specified, the hook takes
+priority.
+
+
+Parallelization can be configured further with these options:
+
+* ``--maxprocesses=maxprocesses``: limit the maximum number of workers to
+  process the tests.
+
+* ``--max-worker-restart``: maximum number of workers that can be restarted
+  when crashed (set to zero to disable this feature).
+
+The test distribution algorithm is configured with the ``--dist`` command-line option:
+
+.. _distribution modes:
+
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order. Scheduling can be fine-tuned with
+  the `--maxschedchunk` option, see output of `pytest --help`.
+
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
+
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker. If a test has
+  multiple groups, they will be joined together into a new group,
+  the order of the marks doesn't matter. This works along with marks from fixtures
+  and from the pytestmark global variable.
+
+  .. code-block:: python
+
+      @pytest.mark.xdist_group(name="group1")
+      def test1():
+          pass
+
+
+      class TestA:
+          @pytest.mark.xdist_group("group1")
+          def test2():
+              pass
+
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+
+  .. code-block:: python
+
+      @pytest.fixture(
+          scope="session",
+          params=[
+              pytest.param(
+                  "chrome",
+                  marks=pytest.mark.xdist_group("chrome"),
+              ),
+              pytest.param(
+                  "firefox",
+                  marks=pytest.mark.xdist_group("firefox"),
+              ),
+              pytest.param(
+                  "edge",
+                  marks=pytest.mark.xdist_group("edge"),
+              ),
+          ],
+      )
+      def setup_container():
+          pass
+
+
+      @pytest.mark.xdist_group(name="data-store")
+      def test_data_store(setup_container):
+          ...
+
+  This will generate 3 new groups: ``chrome_data-store``, ``data-store_firefox`` and ``data-store_edge`` (the markers are lexically sorted before being merged together).
+
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
+
+* ``--dist worksteal``: Initially, tests are distributed evenly among all
+  available workers. When a worker completes most of its assigned tests and
+  doesn't have enough tests to continue (currently, every worker needs at least
+  two tests in its queue), an attempt is made to reassign ("steal") a portion
+  of tests from some other worker's queue. The results should be similar to
+  the ``load`` method, but ``worksteal`` should handle tests with significantly
+  differing duration better, and, at the same time, it should provide similar
+  or better reuse of fixtures.
+
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).

docs/how-it-works.rst

@@ -0,0 +1,90 @@
+How it works?
+=============
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* instead of passing the full
+   test id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

docs/how-to.rst

@@ -0,0 +1,232 @@
+How-tos
+-------
+
+This section show cases how to accomplish some specialized tasks with ``pytest-xdist``.
+
+Identifying the worker process during a test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.15.*
+
+If you need to determine the identity of a worker process in
+a test or fixture, you may use the ``worker_id`` fixture to do so:
+
+.. code-block:: python
+
+    @pytest.fixture()
+    def user_account(worker_id):
+        """use a different account in each xdist worker"""
+        return "account_%s" % worker_id
+
+When ``xdist`` is disabled (running with ``-n0`` for example), then
+``worker_id`` will return ``"master"``.
+
+Worker processes also have the following environment variables
+defined:
+
+.. envvar:: PYTEST_XDIST_WORKER
+
+The name of the worker, e.g., ``"gw2"``.
+
+.. envvar:: PYTEST_XDIST_WORKER_COUNT
+
+The total number of workers in this session, e.g., ``"4"`` when ``-n 4`` is given in the command-line.
+
+The information about the worker_id in a test is stored in the ``TestReport`` as
+well, under the ``worker_id`` attribute.
+
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+
+.. autofunction:: xdist.is_xdist_worker
+.. autofunction:: xdist.is_xdist_controller
+.. autofunction:: xdist.is_xdist_master
+.. autofunction:: xdist.get_xdist_worker_id
+
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
+Uniquely identifying the current test run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """create a unique database for this particular test run"""
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """retrieve unique database"""
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+.. envvar:: PYTEST_XDIST_TESTRUNUID
+
+The unique id of the test run.
+
+Accessing ``sys.argv`` from the controller node in workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the ``sys.argv`` passed to the command-line of the controller node, use
+``request.config.workerinput["mainargv"]``.
+
+
+Specifying test exec environments in an ini file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use pytest's ini file configuration to avoid typing common options.
+You can for example make running with three subprocesses your default like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = -n3
+
+You can also add default environments like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
+
+and then just type::
+
+    pytest --dist=each
+
+to run tests in each of the environments.
+
+
+Specifying "rsync" dirs in an ini-file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
+you may specify directories to include or to exclude in synchronisation:
+
+.. code-block:: ini
+
+    [pytest]
+    rsyncdirs = . mypkg helperpkg
+    rsyncignore = .hg
+
+These directory specifications are relative to the directory
+where the configuration file was found.
+
+.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
+.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
+.. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+Creating one log file for each worker
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create one log file for each worker with ``pytest-xdist``, you can leverage :envvar:`PYTEST_XDIST_WORKER`
+to generate a unique filename for each worker.
+
+Example:
+
+.. code-block:: python
+
+    # content of conftest.py
+    def pytest_configure(config):
+        worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+        if worker_id is not None:
+            logging.basicConfig(
+                format=config.getini("log_file_format"),
+                filename=f"tests_{worker_id}.log",
+                level=config.getini("log_file_level"),
+            )
+
+
+When running the tests with ``-n3``, for example, three files will be created in the current directory:
+``tests_gw0.log``, ``tests_gw1.log`` and ``tests_gw2.log``.

docs/index.rst

@@ -0,0 +1,63 @@
+pytest-xdist
+============
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+.. note::
+    Due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+Features
+--------
+
+* Test run :ref:`parallelization`: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of :ref:`remote machines`.
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
+  pytest waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed (DEPRECATED).
+
+* :ref:`Multi-Platform` coverage: you can specify different Python interpreters
+  or different platforms and run tests in parallel on all of them.
+
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  program source code to the remote place.
+  You may specify different Python versions and interpreters. It does not
+  installs/synchronize dependencies however.
+
+  **Note**: this mode exists mostly for backward compatibility, as modern development
+  relies on continuous integration for multi-platform testing.
+
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Contents:
+
+    distribution
+    subprocess
+    remote
+    crash
+    how-to
+    how-it-works
+    known-limitations
+    changelog

docs/known-limitations.rst

@@ -0,0 +1,71 @@
+Known limitations
+=================
+
+pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist.
+
+Order and amount of test must be consistent
+-------------------------------------------
+
+It is not possible to have tests that differ in order or their amount across workers.
+
+This is especially true with ``pytest.mark.parametrize``, when values are produced with sets or other unordered iterables/generators.
+
+
+Example:
+
+.. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", {"a", "b"})
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+In the example above, the fact that ``set`` are not necessarily ordered can cause different workers
+to collect tests in different order, which will throw an error.
+
+Workarounds
+~~~~~~~~~~~
+
+A solution to this is to guarantee that the parametrized values have the same order.
+
+Some solutions:
+
+* Convert your sequence to a ``list``.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", ["a", "b"])
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+* Sort your sequence, guaranteeing order.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", sorted({"a", "b"}))
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+Output (stdout and stderr) from workers
+---------------------------------------
+
+The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time.
+
+However, this option does not work with ``pytest-xdist`` because `execnet <https://github.com/pytest-dev/execnet>`__ the underlying library used for communication between master and workers, does not support transferring stdout/stderr from workers.
+
+Currently, there are no plans to support this in ``pytest-xdist``.
+
+Debugging
+~~~~~~~~~
+
+This also means that debugging using PDB (or any other debugger that wants to use standard I/O) will not work. The ``--pdb`` option is disabled when distributing tests with ``pytest-xdist`` for this reason.
+
+It is generally likely best to use ``pytest-xdist`` to find failing tests and then debug them without distribution; however, if you need to debug from within a worker process (for example, to address failures that only happen when running tests concurrently), remote debuggers (for example, `python-remote-pdb <https://github.com/ionelmc/python-remote-pdb>`__ or `python-web-pdb <https://github.com/romanvm/python-web-pdb>`__) have been reported to work for this purpose.

docs/remote.rst

@@ -0,0 +1,100 @@
+
+.. _`Multi-Platform`:
+.. _`remote machines`:
+
+Sending tests to remote SSH accounts
+====================================
+
+.. deprecated:: 3.0
+
+.. warning::
+
+    The ``rsync`` feature is deprecated because its implementation is faulty
+    in terms of reproducing the development environment in the remote
+    worker, and there is no clear solution moving forward.
+
+    For that reason, ``rsync`` is scheduled to be removed in release 4.0, to let the team
+    focus on a smaller set of features.
+
+    Note that SSH and socket server are not planned for removal, as they are part
+    of the ``execnet`` feature set.
+
+Suppose you have a package ``mypkg`` which contains some
+tests that you can successfully run locally. And you
+have a ssh-reachable machine ``myhost``.  Then
+you can ad-hoc distribute your tests by typing::
+
+    pytest -d  --rsyncdir mypkg --tx ssh=myhostpopen mypkg/tests/unit/test_something.py
+
+This will synchronize your :code:`mypkg` package directory
+to a remote ssh account and then locally collect tests
+and send them to remote places for execution.
+
+You can specify multiple :code:`--rsyncdir` directories
+to be sent to the remote side.
+
+.. note::
+
+  For pytest to collect and send tests correctly
+  you not only need to make sure all code and tests
+  directories are rsynced, but that any test (sub) directory
+  also has an :code:`__init__.py` file because internally
+  pytest references tests as a fully qualified python
+  module path.  **You will otherwise get strange errors**
+  during setup of the remote side.
+
+
+You can specify multiple :code:`--rsyncignore` glob patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
+Sending tests to remote Socket Servers
+--------------------------------------
+
+Download the single-module `socketserver.py`_ Python program
+and run it like this::
+
+    python socketserver.py
+
+It will tell you that it starts listening on the default
+port.  You can now on your home machine specify this
+new socket host with something like this::
+
+    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg
+
+
+Using proxies to run multiple workers on remote machines
+---------------------------------------
+
+In case you want to run multiple workers on a remote machine,
+you can create a proxy gateway for the machine, and run multiple
+workers using the `via` attribute.::
+
+    pytest -d --px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy
+
+Here we declare a proxy gateway using the `--px` arg, and
+create 5 workers that run on the remote server using the proxy.
+Note that the proxy gateway does not run a worker, thus only 5
+workers are created.
+
+
+Running tests on many platforms at once
+---------------------------------------
+
+The basic command to run tests on multiple platforms is::
+
+    pytest --dist=each --tx=spec1 --tx=spec2
+
+If you specify a windows host, an OSX host and a Linux
+environment this command will send each tests to all
+platforms - and report back failures from all platforms
+at once. The specifications strings use the `xspec syntax`_.
+
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
+
+.. _`execnet`: https://codespeak.net/execnet
+
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/src/execnet/script/socketserver.py

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx
+sphinx-rtd-theme

docs/subprocess.rst

@@ -0,0 +1,16 @@
+Running tests in a Python subprocess
+====================================
+
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
+
+    pytest -d --tx popen//python=python3.9
+
+This will start a subprocess which is run with the ``python3.9``
+Python interpreter, found in your system binary lookup path.
+
+If you prefix the --tx option value like this::
+
+    --tx 3*popen//python=python3.9
+
+then three subprocesses would be created and tests
+will be load-balanced across these three processes.

pyproject.toml

@@ -1,14 +1,141 @@
 [build-system]
 requires = [
-  # sync with setup.py until we discard non-pep-517/518
-  "setuptools>=45.0",
-  "setuptools-scm[toml]>=6.2.3",
-  "wheel",
+  "setuptools>=77.0",
 ]
 build-backend = "setuptools.build_meta"
 
-[tool.setuptools_scm]
-write_to = "src/xdist/_version.py"
+[project]
+name = "pytest-xdist"
+version = "3.7.1.dev24+g2b4372bd6"
+description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs"
+readme = "README.rst"
+license = "MIT"
+authors = [{name = "holger krekel and contributors", email = "pytest-dev@python.org"}, {email = "holger@merlinux.eu"}]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Framework :: Pytest",
+    "Intended Audience :: Developers",
+    "Operating System :: POSIX",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: MacOS :: MacOS X",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Utilities",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+]
+requires-python = ">=3.9"
+dependencies = [
+    "execnet>=2.1",
+    "pytest>=7.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/pytest-dev/pytest-xdist"
+Documentation = "https://pytest-xdist.readthedocs.io/en/latest"
+Changelog = "https://pytest-xdist.readthedocs.io/en/latest/changelog.html"
+Source = "https://github.com/pytest-dev/pytest-xdist"
+Tracker = "https://github.com/pytest-dev/pytest-xdist/issues"
+
+[project.entry-points.pytest11]
+xdist = "xdist.plugin"
+"xdist.looponfail" = "xdist.looponfail"
+
+[project.optional-dependencies]
+testing = ["filelock"]
+psutil = ["psutil>=3.0"]
+setproctitle = ["setproctitle"]
+
+[tool.pytest.ini_options]
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = "-ra -p no:pytest-services"
+testpaths = ["testing"]
+
+[tool.ruff]
+src = ["src"]
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    "B",        # bugbear
+    "D",        # pydocstyle
+    "E",        # pycodestyle
+    "F",        # pyflakes
+    "I",        # isort
+    "PYI",      # flake8-pyi
+    "UP",       # pyupgrade
+    "RUF",      # ruff
+    "W",        # pycodestyle
+    "T10",      # flake8-debugger
+    "PIE",      # flake8-pie
+    "FA",       # flake8-future-annotations
+    "PGH",      # pygrep-hooks
+    "PLE",      # pylint error
+    "PLW",      # pylint warning
+    "PLR1714",  # Consider merging multiple comparisons
+]
+ignore = [
+    # bugbear ignore
+    "B011",     # Do not `assert False` (`python -O` removes these calls)
+    "B028",     # No explicit `stacklevel` keyword argument found
+    # pydocstyle ignore
+    "D100",     # Missing docstring in public module
+    "D101",     # Missing docstring in public class
+    "D102",     # Missing docstring in public method
+    "D103",     # Missing docstring in public function
+    "D104",     # Missing docstring in public package
+    "D105",     # Missing docstring in magic method
+    "D106",     # Missing docstring in public nested class
+    "D107",     # Missing docstring in `__init__`
+    "D209",     # Multi-line docstring closing quotes should be on a separate line
+    "D205",     # 1 blank line required between summary line and description
+    "D400",     # First line should end with a period
+    "D401",     # First line of docstring should be in imperative mood
+    # ruff ignore
+    "RUF012",   # Mutable class attributes should be annotated with `typing.ClassVar`
+    # pylint ignore
+    "PLW0603",  # Using the global statement
+    "PLW0120",  # remove the else and dedent its contents
+    "PLW2901",  # for loop variable overwritten by assignment target
+    "PLR5501",  # Use `elif` instead of `else` then `if`
+    "UP031",  # Use format specifiers instead of percent format
+]
+
+[tool.ruff.lint.pycodestyle]
+# In order to be able to format for 88 char in ruff format
+max-line-length = 120
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"
+
+[tool.ruff.lint.isort]
+force-single-line = true
+combine-as-imports = true
+force-sort-within-sections = true
+order-by-type = false
+lines-after-imports = 2
+
+[tool.ruff.lint.per-file-ignores]
+"src/xdist/_version.py" = ["I001"]
+
+[tool.mypy]
+mypy_path = ["src"]
+files = ["src", "testing"]
+strict = true
+warn_unreachable = true
+[[tool.mypy.overrides]]
+module = ["xdist._version"]
+ignore_missing_imports = true
+
 
 [tool.towncrier]
 package = "xdist"
@@ -17,32 +144,23 @@ directory = "changelog/"
 title_format = "pytest-xdist {version} ({project_date})"
 template = "changelog/_template.rst"
 
-  [[tool.towncrier.type]]
-  directory = "removal"
-  name = "Deprecations and Removals"
-  showcontent = true
+[tool.towncrier.fragment.removal]
+name = "Removals"
 
-  [[tool.towncrier.type]]
-  directory = "feature"
+[tool.towncrier.fragment.deprecation]
+name = "Deprecations"
+
+[tool.towncrier.fragment.feature]
 name = "Features"
-  showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "bugfix"
+[tool.towncrier.fragment.bugfix]
 name = "Bug Fixes"
-  showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "vendor"
+[tool.towncrier.fragment.vendor]
 name = "Vendored Libraries"
-  showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "doc"
+[tool.towncrier.fragment.doc]
 name = "Improved Documentation"
-  showcontent = true
 
-  [[tool.towncrier.type]]
-  directory = "trivial"
+[tool.towncrier.fragment.trivial]
 name = "Trivial Changes"
-  showcontent = true

setup.cfg

@@ -1,76 +0,0 @@
-[metadata]
-name = pytest-xdist
-description = pytest xdist plugin for distributed testing and loop-on-failing modes
-long_description = file: README.rst
-license = MIT
-author = holger krekel and contributors
-author_email = pytest-dev@python.org,holger@merlinux.eu
-url = https://github.com/pytest-dev/pytest-xdist
-platforms =
-    linux
-    osx
-    win32
-classifiers =
-    Development Status :: 5 - Production/Stable
-    Framework :: Pytest
-    Intended Audience :: Developers
-    License :: OSI Approved :: MIT License
-    Operating System :: POSIX
-    Operating System :: Microsoft :: Windows
-    Operating System :: MacOS :: MacOS X
-    Topic :: Software Development :: Testing
-    Topic :: Software Development :: Quality Assurance
-    Topic :: Utilities
-    Programming Language :: Python
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.6
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-license_file = LICENSE
-
-[options]
-packages = find:
-package_dir = =src
-zip_safe = False
-python_requires = >=3.6
-install_requires =
-    execnet>=1.1
-    pytest>=6.2.0
-    pytest-forked
-setup_requires = setuptools_scm>=6.0
-
-[options.packages.find]
-where = src
-
-[options.entry_points]
-pytest11 =
-    xdist = xdist.plugin
-    xdist.looponfail = xdist.looponfail
-
-[options.extras_require]
-testing =
-    filelock
-psutil = psutil>=3.0
-setproctitle = setproctitle
-
-[flake8]
-max-line-length = 100
-
-[mypy]
-mypy_path = src
-# TODO: Enable this & fix errors.
-# check_untyped_defs = True
-disallow_any_generics = True
-ignore_missing_imports = True
-no_implicit_optional = True
-show_error_codes = True
-strict_equality = True
-warn_redundant_casts = True
-warn_return_any = True
-warn_unreachable = True
-warn_unused_configs = True
-# TODO: Enable this & fix errors.
-# no_implicit_reexport = True

setup.py

@@ -1,4 +0,0 @@
-from setuptools import setup
-
-if __name__ == "__main__":
-    setup()

src/xdist/__init__.py

@@ -1,15 +1,16 @@
-from xdist.plugin import (
-    is_xdist_worker,
-    is_xdist_master,
-    get_xdist_worker_id,
-    is_xdist_controller,
-)
-from xdist._version import version as __version__
+from xdist.plugin import get_xdist_worker_id
+from xdist.plugin import is_xdist_controller
+from xdist.plugin import is_xdist_master
+from xdist.plugin import is_xdist_worker
+
+
+__version__ = "3.7.1.dev24+g2b4372bd6"
+
 
 __all__ = [
     "__version__",
-    "is_xdist_worker",
-    "is_xdist_master",
-    "is_xdist_controller",
     "get_xdist_worker_id",
+    "is_xdist_controller",
+    "is_xdist_master",
+    "is_xdist_worker",
 ]

src/xdist/_path.py

@@ -0,0 +1,20 @@
+from collections.abc import Iterator
+from itertools import chain
+import os
+from pathlib import Path
+from typing import Callable
+
+
+def visit_path(
+    path: Path, *, filter: Callable[[Path], bool], recurse: Callable[[Path], bool]
+) -> Iterator[Path]:
+    """
+    Implements the interface of ``py.path.local.visit()`` for Path objects,
+    to simplify porting the code over from ``py.path.local``.
+    """
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames[:] = [x for x in dirnames if recurse(Path(dirpath, x))]
+        for name in chain(dirnames, filenames):
+            p = Path(dirpath, name)
+            if filter(p):
+                yield p

src/xdist/dsession.py

@@ -1,17 +1,28 @@
-import py
+from __future__ import annotations
+
+from collections.abc import Sequence
+from enum import auto
+from enum import Enum
+from queue import Empty
+from queue import Queue
+import sys
+from typing import Any
+import warnings
+
+import execnet
 import pytest
 
+from xdist.remote import Producer
+from xdist.remote import WorkerInfo
+from xdist.scheduler import EachScheduling
+from xdist.scheduler import LoadFileScheduling
+from xdist.scheduler import LoadGroupScheduling
+from xdist.scheduler import LoadScheduling
+from xdist.scheduler import LoadScopeScheduling
+from xdist.scheduler import Scheduling
+from xdist.scheduler import WorkStealingScheduling
 from xdist.workermanage import NodeManager
-from xdist.scheduler import (
-    EachScheduling,
-    LoadScheduling,
-    LoadScopeScheduling,
-    LoadFileScheduling,
-    LoadGroupScheduling,
-)
-
-
-from queue import Empty, Queue
+from xdist.workermanage import WorkerController
 
 
 class Interrupted(KeyboardInterrupt):
@@ -19,7 +30,7 @@ class Interrupted(KeyboardInterrupt):
 
 
 class DSession:
-    """A pytest plugin which runs a distributed test session
+    """A pytest plugin which runs a distributed test session.
 
     At the beginning of the test session this creates a NodeManager
     instance which creates and starts all nodes.  Nodes then emit
@@ -32,44 +43,44 @@ class DSession:
     it will wait for instructions.
     """
 
-    def __init__(self, config):
+    shouldstop: bool | str
+
+    def __init__(self, config: pytest.Config) -> None:
         self.config = config
-        self.log = py.log.Producer("dsession")
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
-        self.nodemanager = None
-        self.sched = None
+        self.log = Producer("dsession", enabled=config.option.debug)
+        self.nodemanager: NodeManager | None = None
+        self.sched: Scheduling | None = None
         self.shuttingdown = False
         self.countfailures = 0
-        self.maxfail = config.getvalue("maxfail")
-        self.queue = Queue()
-        self._session = None
-        self._failed_collection_errors = {}
-        self._active_nodes = set()
+        self.maxfail: int = config.getvalue("maxfail")
+        self.queue: Queue[tuple[str, dict[str, Any]]] = Queue()
+        self._session: pytest.Session | None = None
+        self._failed_collection_errors: dict[object, bool] = {}
+        self._active_nodes: set[WorkerController] = set()
         self._failed_nodes_count = 0
         self._max_worker_restart = get_default_max_worker_restart(self.config)
         # summary message to print at the end of the session
-        self._summary_report = None
+        self._summary_report: str | None = None
         self.terminal = config.pluginmanager.getplugin("terminalreporter")
         if self.terminal:
             self.trdist = TerminalDistReporter(config)
             config.pluginmanager.register(self.trdist, "terminaldistreporter")
 
     @property
-    def session_finished(self):
-        """Return True if the distributed session has finished
+    def session_finished(self) -> bool:
+        """Return True if the distributed session has finished.
 
         This means all nodes have executed all test items.  This is
         used by pytest_runtestloop to break out of its loop.
         """
         return bool(self.shuttingdown and not self._active_nodes)
 
-    def report_line(self, line):
+    def report_line(self, line: str) -> None:
         if self.terminal and self.config.option.verbose >= 0:
             self.terminal.write_line(line)
 
     @pytest.hookimpl(trylast=True)
-    def pytest_sessionstart(self, session):
+    def pytest_sessionstart(self, session: pytest.Session) -> None:
         """Creates and starts the nodes.
 
         The nodes are setup to put their events onto self.queue.  As
@@ -81,7 +92,7 @@ class DSession:
         self._session = session
 
     @pytest.hookimpl
-    def pytest_sessionfinish(self, session):
+    def pytest_sessionfinish(self) -> None:
         """Shutdown all nodes."""
         nm = getattr(self, "nodemanager", None)  # if not fully initialized
         if nm is not None:
@@ -89,38 +100,50 @@ class DSession:
         self._session = None
 
     @pytest.hookimpl
-    def pytest_collection(self):
+    def pytest_collection(self) -> bool:
         # prohibit collection of test items in controller process
         return True
 
     @pytest.hookimpl(trylast=True)
-    def pytest_xdist_make_scheduler(self, config, log):
+    def pytest_xdist_make_scheduler(
+        self,
+        config: pytest.Config,
+        log: Producer,
+    ) -> Scheduling | None:
         dist = config.getvalue("dist")
-        schedulers = {
-            "each": EachScheduling,
-            "load": LoadScheduling,
-            "loadscope": LoadScopeScheduling,
-            "loadfile": LoadFileScheduling,
-            "loadgroup": LoadGroupScheduling,
-        }
-        return schedulers[dist](config, log)
+        if dist == "each":
+            return EachScheduling(config, log)
+        if dist == "load":
+            return LoadScheduling(config, log)
+        if dist == "loadscope":
+            return LoadScopeScheduling(config, log)
+        if dist == "loadfile":
+            return LoadFileScheduling(config, log)
+        if dist == "loadgroup":
+            return LoadGroupScheduling(config, log)
+        if dist == "worksteal":
+            return WorkStealingScheduling(config, log)
+        return None
 
     @pytest.hookimpl
-    def pytest_runtestloop(self):
+    def pytest_runtestloop(self) -> bool:
         self.sched = self.config.hook.pytest_xdist_make_scheduler(
             config=self.config, log=self.log
         )
         assert self.sched is not None
 
         self.shouldstop = False
+        pending_exception = None
         while not self.session_finished:
             self.loop_once()
             if self.shouldstop:
                 self.triggershutdown()
-                raise Interrupted(str(self.shouldstop))
+                pending_exception = Interrupted(str(self.shouldstop))
+        if pending_exception:
+            raise pending_exception
         return True
 
-    def loop_once(self):
+    def loop_once(self) -> None:
         """Process one callback from one of the workers."""
         while 1:
             if not self._active_nodes:
@@ -138,6 +161,7 @@ class DSession:
         call = getattr(self, method)
         self.log("calling method", method, kwargs)
         call(**kwargs)
+        assert self.sched is not None
         if self.sched.tests_finished:
             self.triggershutdown()
 
@@ -145,7 +169,11 @@ class DSession:
     # callbacks for processing events from workers
     #
 
-    def worker_workerready(self, node, workerinfo):
+    def worker_workerready(
+        self,
+        node: WorkerController,
+        workerinfo: WorkerInfo,
+    ) -> None:
         """Emitted when a node first starts up.
 
         This adds the node to the scheduler, nodes continue with
@@ -159,9 +187,10 @@ class DSession:
         if self.shuttingdown:
             node.shutdown()
         else:
+            assert self.sched is not None
             self.sched.add_node(node)
 
-    def worker_workerfinished(self, node):
+    def worker_workerfinished(self, node: WorkerController) -> None:
         """Emitted when node executes its pytest_sessionfinish hook.
 
         Removes the node from the scheduler.
@@ -171,15 +200,26 @@ class DSession:
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
         if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
-            self.shouldstop = "{} received keyboard-interrupt".format(node)
+            self.shouldstop = f"{node} received keyboard-interrupt"
             self.worker_errordown(node, "keyboard-interrupt")
             return
+        shouldfail = node.workeroutput["shouldfail"]
+        shouldstop = node.workeroutput["shouldstop"]
+        for shouldx in [shouldfail, shouldstop]:
+            if shouldx:
+                if not self.shouldstop:
+                    self.shouldstop = shouldx
+                break
+        else:
+            assert self.sched is not None
             if node in self.sched.nodes:
                 crashitem = self.sched.remove_node(node)
                 assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
-    def worker_internal_error(self, node, formatted_error):
+    def worker_internal_error(
+        self, node: WorkerController, formatted_error: str
+    ) -> None:
         """
         pytest_internalerror() was called on the worker.
 
@@ -191,15 +231,14 @@ class DSession:
         try:
             assert False, formatted_error
         except AssertionError:
-            from _pytest._code import ExceptionInfo
-
-            excinfo = ExceptionInfo.from_current()
+            excinfo = pytest.ExceptionInfo.from_current()
             excrepr = excinfo.getrepr()
             self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
 
-    def worker_errordown(self, node, error):
+    def worker_errordown(self, node: WorkerController, error: object | None) -> None:
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
+        assert self.sched is not None
         try:
             crashitem = self.sched.remove_node(node)
         except KeyError:
@@ -215,26 +254,27 @@ class DSession:
         )
         if maximum_reached:
             if self._max_worker_restart == 0:
-                msg = "worker {} crashed and worker restarting disabled".format(
-                    node.gateway.id
-                )
+                msg = f"worker {node.gateway.id} crashed and worker restarting disabled"
             else:
-                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
+                msg = f"maximum crashed workers reached: {self._max_worker_restart}"
             self._summary_report = msg
             self.report_line("\n" + msg)
             self.triggershutdown()
         else:
             self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
+            self.shuttingdown = False
             self._clone_node(node)
         self._active_nodes.remove(node)
 
     @pytest.hookimpl
-    def pytest_terminal_summary(self, terminalreporter):
+    def pytest_terminal_summary(self, terminalreporter: Any) -> None:
         if self.config.option.verbose >= 0 and self._summary_report:
-            terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
+            terminalreporter.write_sep("=", f"xdist: {self._summary_report}")
 
-    def worker_collectionfinish(self, node, ids):
-        """worker has finished test collection.
+    def worker_collectionfinish(
+        self, node: WorkerController, ids: Sequence[str]
+    ) -> None:
+        """Worker has finished test collection.
 
         This adds the collection for this node to the scheduler.  If
         the scheduler indicates collection is finished (i.e. all
@@ -247,43 +287,78 @@ class DSession:
         self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
         # tell session which items were effectively collected otherwise
         # the controller node will finish the session with EXIT_NOTESTSCOLLECTED
+        assert self._session is not None
         self._session.testscollected = len(ids)
+        assert self.sched is not None
         self.sched.add_node_collection(node, ids)
         if self.terminal:
-            self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids)))
+            self.trdist.setstatus(
+                node.gateway.spec, WorkerStatus.CollectionDone, tests_collected=len(ids)
+            )
         if self.sched.collection_is_completed:
             if self.terminal and not self.sched.has_pending:
                 self.trdist.ensure_show_status()
                 self.terminal.write_line("")
                 if self.config.option.verbose > 0:
                     self.terminal.write_line(
-                        "scheduling tests via %s" % (self.sched.__class__.__name__)
+                        f"scheduling tests via {self.sched.__class__.__name__}"
                     )
             self.sched.schedule()
 
-    def worker_logstart(self, node, nodeid, location):
+    def worker_logstart(
+        self,
+        node: WorkerController,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
         """Emitted when a node calls the pytest_runtest_logstart hook."""
         self.config.hook.pytest_runtest_logstart(nodeid=nodeid, location=location)
 
-    def worker_logfinish(self, node, nodeid, location):
+    def worker_logfinish(
+        self,
+        node: WorkerController,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
         """Emitted when a node calls the pytest_runtest_logfinish hook."""
         self.config.hook.pytest_runtest_logfinish(nodeid=nodeid, location=location)
 
-    def worker_testreport(self, node, rep):
+    def worker_testreport(self, node: WorkerController, rep: pytest.TestReport) -> None:
         """Emitted when a node calls the pytest_runtest_logreport hook."""
-        rep.node = node
+        rep.node = node  # type: ignore[attr-defined]
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
-    def worker_runtest_protocol_complete(self, node, item_index, duration):
+    def worker_runtest_protocol_complete(
+        self, node: WorkerController, item_index: int, duration: float
+    ) -> None:
         """
         Emitted when a node fires the 'runtest_protocol_complete' event,
         signalling that a test has completed the runtestprotocol and should be
         removed from the pending list in the scheduler.
         """
+        assert self.sched is not None
         self.sched.mark_test_complete(node, item_index, duration)
 
-    def worker_collectreport(self, node, rep):
+    def worker_unscheduled(
+        self, node: WorkerController, indices: Sequence[int]
+    ) -> None:
+        """
+        Emitted when a node fires the 'unscheduled' event, signalling that
+        some tests have been removed from the worker's queue and should be
+        sent to some worker again.
+
+        This should happen only in response to 'steal' command, so schedulers
+        not using 'steal' command don't have to implement it.
+        """
+        assert self.sched is not None
+        self.sched.remove_pending_tests_from_node(node, indices)
+
+    def worker_collectreport(
+        self,
+        node: WorkerController,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
         """Emitted when a node calls the pytest_collectreport hook.
 
         Because we only need the report when there's a failure/skip, as optimization
@@ -292,21 +367,20 @@ class DSession:
         assert not rep.passed
         self._failed_worker_collectreport(node, rep)
 
-    def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
-        # This hook as been removed in pytest 7.1, and we can remove support once we only
-        # support pytest >=7.1.
-        kwargs = dict(warning_message=warning_message, when=when, item=item)
-        self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
-
-    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+    def worker_warning_recorded(
+        self,
+        warning_message: warnings.WarningMessage,
+        when: str,
+        nodeid: str,
+        location: tuple[str, int, str] | None,
+    ) -> None:
         """Emitted when a node calls the pytest_warning_recorded hook."""
         kwargs = dict(
             warning_message=warning_message, when=when, nodeid=nodeid, location=location
         )
         self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
 
-    def _clone_node(self, node):
+    def _clone_node(self, node: WorkerController) -> WorkerController:
         """Return new node based on an existing one.
 
         This is normally for when a node dies, this will copy the spec
@@ -316,12 +390,17 @@ class DSession:
         """
         spec = node.gateway.spec
         spec.id = None
+        assert self.nodemanager is not None
         self.nodemanager.group.allocate_id(spec)
-        node = self.nodemanager.setup_node(spec, self.queue.put)
-        self._active_nodes.add(node)
-        return node
+        clone = self.nodemanager.setup_node(spec, self.queue.put)
+        self._active_nodes.add(clone)
+        return clone
 
-    def _failed_worker_collectreport(self, node, rep):
+    def _failed_worker_collectreport(
+        self,
+        node: WorkerController,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
         # Check we haven't already seen this report (from
         # another worker).
         if rep.longrepr not in self._failed_collection_errors:
@@ -329,28 +408,41 @@ class DSession:
             self.config.hook.pytest_collectreport(report=rep)
             self._handlefailures(rep)
 
-    def _handlefailures(self, rep):
+    def _handlefailures(
+        self,
+        rep: pytest.CollectReport | pytest.TestReport,
+    ) -> None:
         if rep.failed:
             self.countfailures += 1
-            if self.maxfail and self.countfailures >= self.maxfail:
-                self.shouldstop = "stopping after %d failures" % (self.countfailures)
+            if (
+                self.maxfail
+                and self.countfailures >= self.maxfail
+                and not self.shouldstop
+            ):
+                self.shouldstop = f"stopping after {self.countfailures} failures"
 
-    def triggershutdown(self):
+    def triggershutdown(self) -> None:
+        if not self.shuttingdown:
             self.log("triggering shutdown")
             self.shuttingdown = True
+            assert self.sched is not None
             for node in self.sched.nodes:
                 node.shutdown()
 
-    def handle_crashitem(self, nodeid, worker):
+    def handle_crashitem(self, nodeid: str, worker: WorkerController) -> None:
         # XXX get more reporting info by recording pytest_runtest_logstart?
         # XXX count no of failures and retry N times
-        runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "worker {!r} crashed while running {!r}".format(worker.gateway.id, nodeid)
-        rep = runner.TestReport(
-            nodeid, (fspath, None, fspath), (), "failed", msg, "???"
+        msg = f"worker {worker.gateway.id!r} crashed while running {nodeid!r}"
+        rep = pytest.TestReport(
+            nodeid=nodeid,
+            location=(fspath, None, fspath),
+            keywords={},
+            outcome="failed",
+            longrepr=msg,
+            when="???",  # type: ignore[arg-type]
         )
-        rep.node = worker
+        rep.node = worker  # type: ignore[attr-defined]
 
         self.config.hook.pytest_handlecrashitem(
             crashitem=nodeid,
@@ -360,36 +452,58 @@ class DSession:
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
+class WorkerStatus(Enum):
+    """Status of each worker during creation/collection."""
+
+    # Worker spec has just been created.
+    Created = auto()
+
+    # Worker has been initialized.
+    Initialized = auto()
+
+    # Worker is now ready for collection.
+    ReadyForCollection = auto()
+
+    # Worker has finished collection.
+    CollectionDone = auto()
+
+
 class TerminalDistReporter:
-    def __init__(self, config):
+    def __init__(self, config: pytest.Config) -> None:
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")
-        self._status = {}
+        self._status: dict[object, tuple[WorkerStatus, int]] = {}
         self._lastlen = 0
         self._isatty = getattr(self.tr, "isatty", self.tr.hasmarkup)
 
-    def write_line(self, msg):
+    def write_line(self, msg: str) -> None:
         self.tr.write_line(msg)
 
-    def ensure_show_status(self):
+    def ensure_show_status(self) -> None:
         if not self._isatty:
             self.write_line(self.getstatus())
 
-    def setstatus(self, spec, status, show=True):
-        self._status[spec.id] = status
+    def setstatus(
+        self,
+        spec: execnet.XSpec,
+        status: WorkerStatus,
+        *,
+        tests_collected: int,
+        show: bool = True,
+    ) -> None:
+        self._status[spec.id] = (status, tests_collected)
         if show and self._isatty:
             self.rewrite(self.getstatus())
 
-    def getstatus(self):
+    def getstatus(self) -> str:
         if self.config.option.verbose >= 0:
-            parts = [
-                "{} {}".format(spec.id, self._status[spec.id]) for spec in self._specs
-            ]
-            return " / ".join(parts)
-        else:
+            line = get_workers_status_line(list(self._status.values()))
+            if line:
+                return line
+
         return "bringing up nodes..."
 
-    def rewrite(self, line, newline=False):
+    def rewrite(self, line: str, newline: bool = False) -> None:
         pline = line + " " * max(self._lastlen - len(line), 0)
         if newline:
             self._lastlen = 0
@@ -399,51 +513,89 @@ class TerminalDistReporter:
         self.tr.rewrite(pline, bold=True)
 
     @pytest.hookimpl
-    def pytest_xdist_setupnodes(self, specs):
+    def pytest_xdist_setupnodes(self, specs: Sequence[execnet.XSpec]) -> None:
         self._specs = specs
         for spec in specs:
-            self.setstatus(spec, "I", show=False)
-        self.setstatus(spec, "I", show=True)
+            self.setstatus(spec, WorkerStatus.Created, tests_collected=0, show=False)
+        self.setstatus(spec, WorkerStatus.Created, tests_collected=0, show=True)
         self.ensure_show_status()
 
     @pytest.hookimpl
-    def pytest_xdist_newgateway(self, gateway):
+    def pytest_xdist_newgateway(self, gateway: execnet.Gateway) -> None:
         if self.config.option.verbose > 0:
             rinfo = gateway._rinfo()
-            version = "%s.%s.%s" % rinfo.version_info[:3]
+            different_interpreter = rinfo.executable != sys.executable
+            if different_interpreter:
+                version = "{}.{}.{}".format(*rinfo.version_info[:3])
                 self.rewrite(
-                "[%s] %s Python %s cwd: %s"
-                % (gateway.id, rinfo.platform, version, rinfo.cwd),
+                    f"[{gateway.id}] {rinfo.platform} Python {version} cwd: {rinfo.cwd}",
                     newline=True,
                 )
-        self.setstatus(gateway.spec, "C")
+        self.setstatus(gateway.spec, WorkerStatus.Initialized, tests_collected=0)
 
     @pytest.hookimpl
-    def pytest_testnodeready(self, node):
+    def pytest_testnodeready(self, node: WorkerController) -> None:
         if self.config.option.verbose > 0:
             d = node.workerinfo
-            infoline = "[{}] Python {}".format(
-                d["id"], d["version"].replace("\n", " -- ")
+            different_interpreter = d.get("executable") != sys.executable
+            if different_interpreter:
+                version = d["version"].replace("\n", " -- ")
+                self.rewrite(f"[{d['id']}] Python {version}", newline=True)
+        self.setstatus(
+            node.gateway.spec, WorkerStatus.ReadyForCollection, tests_collected=0
         )
-            self.rewrite(infoline, newline=True)
-        self.setstatus(node.gateway.spec, "ok")
 
     @pytest.hookimpl
-    def pytest_testnodedown(self, node, error):
+    def pytest_testnodedown(self, node: WorkerController, error: object) -> None:
         if not error:
             return
-        self.write_line("[{}] node down: {}".format(node.gateway.id, error))
+        self.write_line(f"[{node.gateway.id}] node down: {error}")
 
 
-def get_default_max_worker_restart(config):
-    """gets the default value of --max-worker-restart option if it is not provided.
+def get_default_max_worker_restart(config: pytest.Config) -> int | None:
+    """Gets the default value of --max-worker-restart option if it is not provided.
 
     Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
     """
-    result = config.option.maxworkerrestart
-    if result is not None:
-        result = int(result)
+    result_str: str | None = config.option.maxworkerrestart
+    if result_str is not None:
+        result = int(result_str)
     elif config.option.numprocesses:
         # if --max-worker-restart was not provided, use a reasonable default (#226)
         result = config.option.numprocesses * 4
+    else:
+        result = None
     return result
+
+
+def get_workers_status_line(
+    status_and_items: Sequence[tuple[WorkerStatus, int]],
+) -> str:
+    """
+    Return the line to display during worker setup/collection based on the
+    status of the workers and number of tests collected for each.
+    """
+    statuses = [s for s, c in status_and_items]
+    total_workers = len(statuses)
+    workers_noun = "worker" if total_workers == 1 else "workers"
+    if status_and_items and all(s == WorkerStatus.CollectionDone for s in statuses):
+        # All workers collect the same number of items, so we grab
+        # the total number of items from the first worker.
+        first = status_and_items[0]
+        status, tests_collected = first
+        tests_noun = "item" if tests_collected == 1 else "items"
+        return f"{total_workers} {workers_noun} [{tests_collected} {tests_noun}]"
+    if WorkerStatus.CollectionDone in statuses:
+        done = sum(1 for s, c in status_and_items if c > 0)
+        return f"collecting: {done}/{total_workers} {workers_noun}"
+    if WorkerStatus.ReadyForCollection in statuses:
+        ready = statuses.count(WorkerStatus.ReadyForCollection)
+        return f"ready: {ready}/{total_workers} {workers_noun}"
+    if WorkerStatus.Initialized in statuses:
+        initialized = statuses.count(WorkerStatus.Initialized)
+        return f"initialized: {initialized}/{total_workers} {workers_noun}"
+    if WorkerStatus.Created in statuses:
+        created = statuses.count(WorkerStatus.Created)
+        return f"created: {created}/{total_workers} {workers_noun}"
+
+    return ""

src/xdist/looponfail.py

@@ -6,15 +6,25 @@
 processes) otherwise changes to source code can crash
 the controlling process which should best never happen.
 """
-import py
-import pytest
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import os
+from pathlib import Path
 import sys
 import time
+from typing import Any
+
+from _pytest._io import TerminalWriter
 import execnet
+import pytest
+
+from xdist._path import visit_path
 
 
 @pytest.hookimpl
-def pytest_addoption(parser):
+def pytest_addoption(parser: pytest.Parser) -> None:
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
         "-f",
@@ -22,25 +32,28 @@ def pytest_addoption(parser):
         action="store_true",
         dest="looponfail",
         default=False,
-        help="run tests in subprocess, wait for modified files "
-        "and re-run failing test set until all pass.",
+        help="Run tests in subprocess: wait for files to be modified, then "
+        "re-run failing test set until all pass.",
     )
 
 
 @pytest.hookimpl
-def pytest_cmdline_main(config):
-
+def pytest_cmdline_main(config: pytest.Config) -> int | None:
     if config.getoption("looponfail"):
         usepdb = config.getoption("usepdb", False)  # a core option
         if usepdb:
             raise pytest.UsageError("--pdb is incompatible with --looponfail.")
         looponfail_main(config)
         return 2  # looponfail only can get stop with ctrl-C anyway
+    return None
 
 
-def looponfail_main(config):
+def looponfail_main(config: pytest.Config) -> None:
     remotecontrol = RemoteControl(config)
-    rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")]
+    config_roots = config.getini("looponfailroots")
+    if not config_roots:
+        config_roots = [Path.cwd()]
+    rootdirs = [Path(root) for root in config_roots]
     statrecorder = StatRecorder(rootdirs)
     try:
         while 1:
@@ -57,21 +70,21 @@ def looponfail_main(config):
 
 
 class RemoteControl:
-    def __init__(self, config):
-        self.config = config
-        self.failures = []
+    gateway: execnet.Gateway
 
-    def trace(self, *args):
+    def __init__(self, config: pytest.Config) -> None:
+        self.config = config
+        self.failures: list[str] = []
+
+    def trace(self, *args: object) -> None:
         if self.config.option.debug:
             msg = " ".join(str(x) for x in args)
             print("RemoteControl:", msg)
 
-    def initgateway(self):
-        return execnet.makegateway("popen")
+    def initgateway(self) -> execnet.Gateway:
+        return execnet.makegateway("execmodel=main_thread_only//popen")
 
-    def setup(self, out=None):
-        if out is None:
-            out = py.io.TerminalWriter()
+    def setup(self) -> None:
         if hasattr(self, "gateway"):
             raise ValueError("already have gateway %r" % self.gateway)
         self.trace("setting up worker session")
@@ -81,15 +94,17 @@ class RemoteControl:
             args=self.config.args,
             option_dict=vars(self.config.option),
         )
-        remote_outchannel = channel.receive()
+        remote_outchannel: execnet.Channel = channel.receive()
 
-        def write(s):
+        out = TerminalWriter()
+
+        def write(s: str) -> None:
             out._file.write(s)
             out._file.flush()
 
         remote_outchannel.setcallback(write)
 
-    def ensure_teardown(self):
+    def ensure_teardown(self) -> None:
         if hasattr(self, "channel"):
             if not self.channel.isclosed():
                 self.trace("closing", self.channel)
@@ -100,12 +115,12 @@ class RemoteControl:
             self.gateway.exit()
             del self.gateway
 
-    def runsession(self):
+    def runsession(self) -> tuple[list[str], list[str], bool]:
         try:
             self.trace("sending", self.failures)
             self.channel.send(self.failures)
             try:
-                return self.channel.receive()
+                return self.channel.receive()  # type: ignore[no-any-return]
             except self.channel.RemoteError:
                 e = sys.exc_info()[1]
                 self.trace("ERROR", e)
@@ -113,7 +128,7 @@ class RemoteControl:
         finally:
             self.ensure_teardown()
 
-    def loop_once(self):
+    def loop_once(self) -> None:
         self.setup()
         self.wasfailing = self.failures and len(self.failures)
         result = self.runsession()
@@ -128,8 +143,10 @@ class RemoteControl:
             self.failures = uniq_failures
 
 
-def repr_pytest_looponfailinfo(failreports, rootdirs):
-    tr = py.io.TerminalWriter()
+def repr_pytest_looponfailinfo(
+    failreports: Sequence[str], rootdirs: Sequence[Path]
+) -> None:
+    tr = TerminalWriter()
     if failreports:
         tr.sep("#", "LOOPONFAILING", bold=True)
         for report in failreports:
@@ -137,10 +154,14 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
                 tr.line(report, red=True)
     tr.sep("#", "waiting for changes", bold=True)
     for rootdir in rootdirs:
-        tr.line("### Watching:   {}".format(rootdir), bold=True)
+        tr.line(f"### Watching:   {rootdir}", bold=True)
 
 
-def init_worker_session(channel, args, option_dict):
+def init_worker_session(
+    channel: "execnet.Channel",  # noqa: UP037
+    args: list[str],
+    option_dict: dict[str, "Any"],  # noqa: UP037
+) -> None:
     import os
     import sys
 
@@ -151,13 +172,14 @@ def init_worker_session(channel, args, option_dict):
     newpaths = []
     for p in sys.path:
         if p:
-            if not os.path.isabs(p):
+            # Ignore path placeholders created for editable installs
+            if not os.path.isabs(p) and not p.endswith(".__path_hook__"):
                 p = os.path.abspath(p)
             newpaths.append(p)
     sys.path[:] = newpaths
 
     # fullwidth, hasmarkup = channel.receive()
-    from _pytest.config import Config
+    from pytest import Config
 
     config = Config.fromdictargs(option_dict, list(args))
     config.args = args
@@ -167,21 +189,21 @@ def init_worker_session(channel, args, option_dict):
 
 
 class WorkerFailSession:
-    def __init__(self, config, channel):
+    def __init__(self, config: pytest.Config, channel: execnet.Channel) -> None:
         self.config = config
         self.channel = channel
-        self.recorded_failures = []
+        self.recorded_failures: list[pytest.CollectReport | pytest.TestReport] = []
         self.collection_failed = False
         config.pluginmanager.register(self)
         config.option.looponfail = False
         config.option.usepdb = False
 
-    def DEBUG(self, *args):
+    def DEBUG(self, *args: object) -> None:
         if self.config.option.debug:
             print(" ".join(map(str, args)))
 
     @pytest.hookimpl
-    def pytest_collection(self, session):
+    def pytest_collection(self, session: pytest.Session) -> bool:
         self.session = session
         self.trails = self.current_command
         hook = self.session.ihook
@@ -196,17 +218,17 @@ class WorkerFailSession:
         return True
 
     @pytest.hookimpl
-    def pytest_runtest_logreport(self, report):
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
         if report.failed:
             self.recorded_failures.append(report)
 
     @pytest.hookimpl
-    def pytest_collectreport(self, report):
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
         if report.failed:
             self.recorded_failures.append(report)
             self.collection_failed = True
 
-    def main(self):
+    def main(self) -> None:
         self.DEBUG("WORKER: received configuration, waiting for command trails")
         try:
             command = self.channel.receive()
@@ -221,56 +243,57 @@ class WorkerFailSession:
             loc = rep.longrepr
             loc = str(getattr(loc, "reprcrash", loc))
             failreports.append(loc)
-        self.channel.send((trails, failreports, self.collection_failed))
+        result = (trails, failreports, self.collection_failed)
+        self.channel.send(result)
 
 
 class StatRecorder:
-    def __init__(self, rootdirlist):
+    def __init__(self, rootdirlist: Sequence[Path]) -> None:
         self.rootdirlist = rootdirlist
-        self.statcache = {}
+        self.statcache: dict[Path, os.stat_result] = {}
         self.check()  # snapshot state
 
-    def fil(self, p):
-        return p.check(file=1, dotfile=0) and p.ext != ".pyc"
+    def fil(self, p: Path) -> bool:
+        return p.is_file() and not p.name.startswith(".") and p.suffix != ".pyc"
 
-    def rec(self, p):
-        return p.check(dotfile=0)
+    def rec(self, p: Path) -> bool:
+        return not p.name.startswith(".") and p.exists()
 
-    def waitonchange(self, checkinterval=1.0):
+    def waitonchange(self, checkinterval: float = 1.0) -> None:
         while 1:
             changed = self.check()
             if changed:
                 return
             time.sleep(checkinterval)
 
-    def check(self, removepycfiles=True):  # noqa, too complex
+    def check(self, removepycfiles: bool = True) -> bool:
         changed = False
-        statcache = self.statcache
-        newstat = {}
+        newstat: dict[Path, os.stat_result] = {}
         for rootdir in self.rootdirlist:
-            for path in rootdir.visit(self.fil, self.rec):
-                oldstat = statcache.pop(path, None)
+            for path in visit_path(rootdir, filter=self.fil, recurse=self.rec):
+                oldstat = self.statcache.pop(path, None)
                 try:
-                    newstat[path] = curstat = path.stat()
-                except py.error.ENOENT:
+                    curstat = path.stat()
+                except OSError:
                     if oldstat:
                         changed = True
                 else:
-                    if oldstat:
+                    newstat[path] = curstat
+                    if oldstat is not None:
                         if (
-                            oldstat.mtime != curstat.mtime
-                            or oldstat.size != curstat.size
+                            oldstat.st_mtime != curstat.st_mtime
+                            or oldstat.st_size != curstat.st_size
                         ):
                             changed = True
                             print("# MODIFIED", path)
-                            if removepycfiles and path.ext == ".py":
-                                pycfile = path + "c"
-                                if pycfile.check():
-                                    pycfile.remove()
+                            if removepycfiles and path.suffix == ".py":
+                                pycfile = path.with_suffix(".pyc")
+                                if pycfile.is_file():
+                                    os.unlink(pycfile)
 
                     else:
                         changed = True
-        if statcache:
+        if self.statcache:
             changed = True
         self.statcache = newstat
         return changed

src/xdist/newhooks.py

@@ -11,71 +11,109 @@ must be taken in plugins in case ``xdist`` is not installed. Please see:
 
     http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
 """
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+import os
+from typing import Any
+from typing import TYPE_CHECKING
+
+import execnet
 import pytest
 
 
-@pytest.hookspec()
-def pytest_xdist_setupnodes(config, specs):
-    """called before any remote node is set up."""
+if TYPE_CHECKING:
+    from xdist.remote import Producer
+    from xdist.scheduler.protocol import Scheduling
+    from xdist.workermanage import WorkerController
 
 
 @pytest.hookspec()
-def pytest_xdist_newgateway(gateway):
-    """called on new raw gateway creation."""
+def pytest_xdist_setupnodes(
+    config: pytest.Config, specs: Sequence[execnet.XSpec]
+) -> None:
+    """Called before any remote node is set up."""
 
 
 @pytest.hookspec()
-def pytest_xdist_rsyncstart(source, gateways):
-    """called before rsyncing a directory to remote gateways takes place."""
+def pytest_xdist_newgateway(gateway: execnet.Gateway) -> None:
+    """Called on new raw gateway creation."""
 
 
-@pytest.hookspec()
-def pytest_xdist_rsyncfinish(source, gateways):
-    """called after rsyncing a directory to remote gateways takes place."""
+@pytest.hookspec(
+    warn_on_impl=DeprecationWarning(
+        "rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+    )
+)
+def pytest_xdist_rsyncstart(
+    source: str | os.PathLike[str],
+    gateways: Sequence[execnet.Gateway],
+) -> None:
+    """Called before rsyncing a directory to remote gateways takes place."""
+
+
+@pytest.hookspec(
+    warn_on_impl=DeprecationWarning(
+        "rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+    )
+)
+def pytest_xdist_rsyncfinish(
+    source: str | os.PathLike[str],
+    gateways: Sequence[execnet.Gateway],
+) -> None:
+    """Called after rsyncing a directory to remote gateways takes place."""
 
 
 @pytest.hookspec(firstresult=True)
-def pytest_xdist_getremotemodule():
-    """called when creating remote node"""
+def pytest_xdist_getremotemodule() -> Any:
+    """Called when creating remote node."""
 
 
 @pytest.hookspec()
-def pytest_configure_node(node):
-    """configure node information before it gets instantiated."""
+def pytest_configure_node(node: WorkerController) -> None:
+    """Configure node information before it gets instantiated."""
 
 
 @pytest.hookspec()
-def pytest_testnodeready(node):
+def pytest_testnodeready(node: WorkerController) -> None:
     """Test Node is ready to operate."""
 
 
 @pytest.hookspec()
-def pytest_testnodedown(node, error):
+def pytest_testnodedown(node: WorkerController, error: object | None) -> None:
     """Test Node is down."""
 
 
 @pytest.hookspec()
-def pytest_xdist_node_collection_finished(node, ids):
-    """called by the controller node when a worker node finishes collecting."""
+def pytest_xdist_node_collection_finished(
+    node: WorkerController, ids: Sequence[str]
+) -> None:
+    """Called by the controller node when a worker node finishes collecting."""
 
 
 @pytest.hookspec(firstresult=True)
-def pytest_xdist_make_scheduler(config, log):
-    """return a node scheduler implementation"""
+def pytest_xdist_make_scheduler(
+    config: pytest.Config, log: Producer
+) -> Scheduling | None:
+    """Return a node scheduler implementation."""
 
 
 @pytest.hookspec(firstresult=True)
-def pytest_xdist_auto_num_workers(config):
+def pytest_xdist_auto_num_workers(config: pytest.Config) -> int:
     """
     Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
     command-line.
 
     .. versionadded:: 2.1
     """
+    raise NotImplementedError()
 
 
 @pytest.hookspec(firstresult=True)
-def pytest_handlecrashitem(crashitem, report, sched):
+def pytest_handlecrashitem(
+    crashitem: str, report: pytest.TestReport, sched: Scheduling
+) -> None:
     """
     Handle a crashitem, modifying the report if necessary.
 

src/xdist/plugin.py

@@ -1,32 +1,41 @@
-import os
-import uuid
-import sys
-from pathlib import Path
+from __future__ import annotations
+
+import os
+import sys
+from typing import Literal
+import uuid
+import warnings
 
-import py
 import pytest
 
 
-PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
-
 _sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
 
 
 @pytest.hookimpl
-def pytest_xdist_auto_num_workers(config):
+def pytest_xdist_auto_num_workers(config: pytest.Config) -> int:
+    env_var = os.environ.get("PYTEST_XDIST_AUTO_NUM_WORKERS")
+    if env_var:
+        try:
+            return int(env_var)
+        except ValueError:
+            warnings.warn(
+                "PYTEST_XDIST_AUTO_NUM_WORKERS is not a number: {env_var!r}. Ignoring it."
+            )
+
     try:
         import psutil
     except ImportError:
         pass
     else:
-        use_logical = config.option.numprocesses == "logical"
+        use_logical: bool = config.option.numprocesses == "logical"
         count = psutil.cpu_count(logical=use_logical) or psutil.cpu_count()
         if count:
             return count
     try:
         from os import sched_getaffinity
 
-        def cpu_count():
+        def cpu_count() -> int:
             return len(sched_getaffinity(0))
 
     except ImportError:
@@ -34,7 +43,7 @@ def pytest_xdist_auto_num_workers(config):
             # workaround https://bitbucket.org/pypy/pypy/issues/2375
             return 2
         try:
-            from os import cpu_count
+            from os import cpu_count  # type: ignore[assignment]
         except ImportError:
             from multiprocessing import cpu_count
     try:
@@ -44,15 +53,19 @@ def pytest_xdist_auto_num_workers(config):
     return n if n else 1
 
 
-def parse_numprocesses(s):
+def parse_numprocesses(s: str) -> int | Literal["auto", "logical"]:
     if s in ("auto", "logical"):
-        return s
+        return s  # type: ignore[return-value]
     elif s is not None:
         return int(s)
 
 
 @pytest.hookimpl
-def pytest_addoption(parser):
+def pytest_addoption(parser: pytest.Parser) -> None:
+    # 'Help' formatting (same rules as pytest's):
+    # Start with capitalized letters.
+    # If a single phrase, do not end with period. If more than one phrase, all phrases end with periods.
+    # Use \n to separate logical lines.
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
         "-n",
@@ -61,10 +74,11 @@ def pytest_addoption(parser):
         metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
-        help="Shortcut for '--dist=load --tx=NUM*popen'. With 'auto', attempt "
-        "to detect physical CPU count. With 'logical', detect logical CPU "
-        "count. If physical CPU count cannot be found, falls back to logical "
-        "count. This will be 0 when used with --pdb.",
+        help="Shortcut for '--dist=load --tx=NUM*popen'.\n"
+        "With 'logical', attempt to detect logical CPU count (requires psutil, falls back to 'auto').\n"
+        "With 'auto', attempt to detect physical CPU count. If physical CPU count cannot be determined, "
+        "falls back to 1.\n"
+        "Forced to 0 (disabled) when used with --pdb.",
     )
     group.addoption(
         "--maxprocesses",
@@ -72,34 +86,71 @@ def pytest_addoption(parser):
         metavar="maxprocesses",
         action="store",
         type=int,
-        help="limit the maximum number of workers to process the tests when using --numprocesses=auto",
+        help="Limit the maximum number of workers to process the tests when using --numprocesses "
+        "with 'auto' or 'logical'",
     )
     group.addoption(
         "--max-worker-restart",
         action="store",
         default=None,
         dest="maxworkerrestart",
-        help="maximum number of workers that can be restarted "
+        help="Maximum number of workers that can be restarted "
         "when crashed (set to zero to disable this feature)",
     )
     group.addoption(
         "--dist",
         metavar="distmode",
         action="store",
-        choices=["each", "load", "loadscope", "loadfile", "loadgroup", "no"],
+        choices=[
+            "each",
+            "load",
+            "loadscope",
+            "loadfile",
+            "loadgroup",
+            "worksteal",
+            "no",
+        ],
         dest="dist",
         default="no",
         help=(
-            "set mode for distributing tests to exec environments.\n\n"
-            "each: send each test to all available environments.\n\n"
-            "load: load balance by sending any pending test to any"
+            "Set mode for distributing tests to exec environments.\n\n"
+            "each: Send each test to all available environments.\n\n"
+            "load: Load balance by sending any pending test to any"
             " available environment.\n\n"
-            "loadscope: load balance by sending pending groups of tests in"
+            "loadscope: Load balance by sending pending groups of tests in"
             " the same scope to any available environment.\n\n"
-            "loadfile: load balance by sending test grouped by file"
+            "loadfile: Load balance by sending test grouped by file"
             " to any available environment.\n\n"
-            "loadgroup: like load, but sends tests marked with 'xdist_group' to the same worker.\n\n"
-            "(default) no: run tests inprocess, don't distribute."
+            "loadgroup: Like 'load', but sends tests marked with 'xdist_group' to the same worker.\n\n"
+            "worksteal: Split the test suite between available environments,"
+            " then re-balance when any worker runs out of tests.\n\n"
+            "(default) no: Run tests inprocess, don't distribute."
+        ),
+    )
+    group.addoption(
+        "--loadscope-reorder",
+        dest="loadscopereorder",
+        action="store_true",
+        default=True,
+        help=(
+            "Pytest-xdist will default reorder tests by number of tests per scope "
+            "when used in conjunction with loadscope.\n"
+            "This option will enable loadscope reorder which will improve the "
+            "parallelism of the test suite.\n"
+            "However, the partial order of tests might not be retained.\n"
+        ),
+    )
+    group.addoption(
+        "--no-loadscope-reorder",
+        dest="loadscopereorder",
+        action="store_false",
+        help=(
+            "Pytest-xdist will default reorder tests by number of tests per scope "
+            "when used in conjunction with loadscope.\n"
+            "This option will disable loadscope reorder, "
+            "and the partial order of tests can be retained.\n"
+            "This is useful when pytest-xdist is used together with "
+            "other plugins that specify tests in a specific order."
         ),
     )
     group.addoption(
@@ -109,63 +160,83 @@ def pytest_addoption(parser):
         default=[],
         metavar="xspec",
         help=(
-            "add a test execution environment. some examples: "
-            "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
+            "Add a test execution environment. Some examples:\n"
+            "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888\n"
             "--tx ssh=user@codespeak.net//chdir=testcache"
         ),
     )
+    group.addoption(
+        "--px",
+        dest="px",
+        action="append",
+        default=[],
+        metavar="xspec",
+        help=(
+            "Add a proxy gateway to pass to test execution environments using `via`. Example:\n"
+            "--px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy"
+        ),
+    )
     group._addoption(
         "-d",
         action="store_true",
         dest="distload",
         default=False,
-        help="load-balance tests.  shortcut for '--dist=load'",
+        help="Load-balance tests. Shortcut for '--dist=load'.",
     )
     group.addoption(
         "--rsyncdir",
         action="append",
         default=[],
         metavar="DIR",
-        help="add directory for rsyncing to remote tx nodes.",
+        help="Add directory for rsyncing to remote tx nodes",
     )
     group.addoption(
         "--rsyncignore",
         action="append",
         default=[],
         metavar="GLOB",
-        help="add expression for ignores when rsyncing to remote tx nodes.",
-    )
-    group.addoption(
-        "--boxed",
-        action="store_true",
-        help="backward compatibility alias for pytest-forked --forked",
+        help="Add expression for ignores when rsyncing to remote tx nodes",
     )
     group.addoption(
         "--testrunuid",
         action="store",
         help=(
-            "provide an identifier shared amongst all workers as the value of "
-            "the 'testrun_uid' fixture,\n\n,"
-            "if not provided, 'testrun_uid' is filled with a new unique string "
+            "Provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture.\n"
+            "If not provided, 'testrun_uid' is filled with a new unique string "
             "on every test run."
         ),
     )
+    group.addoption(
+        "--maxschedchunk",
+        action="store",
+        type=int,
+        help=(
+            "Maximum number of tests scheduled in one step for --dist=load.\n"
+            "Setting it to 1 will force pytest to send tests to workers one by "
+            "one - might be useful for a small number of slow tests.\n"
+            "Larger numbers will allow the scheduler to submit consecutive "
+            "chunks of tests to workers - allows reusing fixtures.\n"
+            "Due to implementation reasons, at least 2 tests are scheduled per "
+            "worker at the start. Only later tests can be scheduled one by one.\n"
+            "Unlimited if not set."
+        ),
+    )
 
     parser.addini(
         "rsyncdirs",
         "list of (relative) paths to be rsynced for remote distributed testing.",
-        type="paths" if PYTEST_GTE_7 else "pathlist",
+        type="paths",
     )
     parser.addini(
         "rsyncignore",
         "list of (relative) glob-style paths to be ignored for rsyncing.",
-        type="paths" if PYTEST_GTE_7 else "pathlist",
+        type="paths",
     )
     parser.addini(
         "looponfailroots",
-        type="paths" if PYTEST_GTE_7 else "pathlist",
-        help="directories to check for changes",
-        default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()],
+        type="paths",
+        help="directories to check for changes. Default: current directory.",
     )
 
 
@@ -175,7 +246,7 @@ def pytest_addoption(parser):
 
 
 @pytest.hookimpl
-def pytest_addhooks(pluginmanager):
+def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None:
     from xdist import newhooks
 
     pluginmanager.add_hookspecs(newhooks)
@@ -187,8 +258,20 @@ def pytest_addhooks(pluginmanager):
 
 
 @pytest.hookimpl(trylast=True)
-def pytest_configure(config):
-    if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
+def pytest_configure(config: pytest.Config) -> None:
+    config_line = (
+        "xdist_group: specify group for tests should run in same session."
+        "in relation to one another. Provided by pytest-xdist."
+    )
+    config.addinivalue_line("markers", config_line)
+
+    # Skip this plugin entirely when only doing collection.
+    if config.getvalue("collectonly"):
+        return
+
+    # Create the distributed session in case we have a valid distribution
+    # mode and test environments.
+    if _is_distribution_mode(config):
         from xdist.dsession import DSession
 
         session = DSession(config)
@@ -196,24 +279,33 @@ def pytest_configure(config):
         tr = config.pluginmanager.getplugin("terminalreporter")
         if tr:
             tr.showfspath = False
-    if config.getoption("boxed"):
+
+    # Deprecation warnings for deprecated command-line/configuration options.
+    if config.getoption("looponfail", None) or config.getini("looponfailroots"):
         warning = DeprecationWarning(
-            "The --boxed commmand line argument is deprecated. "
-            "Install pytest-forked and use --forked instead. "
-            "pytest-xdist 3.0.0 will remove the --boxed argument and pytest-forked dependency."
+            "The --looponfail command line argument and looponfailroots config variable are deprecated.\n"
+            "The loop-on-fail feature will be removed in pytest-xdist 4.0."
         )
         config.issue_config_time_warning(warning, 2)
-        config.option.forked = True
 
-    config_line = (
-        "xdist_group: specify group for tests should run in same session."
-        "in relation to one another. " + "Provided by pytest-xdist."
+    if config.getoption("rsyncdir", None) or config.getini("rsyncdirs"):
+        warning = DeprecationWarning(
+            "The --rsyncdir command line argument and rsyncdirs config variable are deprecated.\n"
+            "The rsync feature will be removed in pytest-xdist 4.0."
         )
-    config.addinivalue_line("markers", config_line)
+        config.issue_config_time_warning(warning, 2)
+
+
+def _is_distribution_mode(config: pytest.Config) -> bool:
+    """Whether distribution mode is on."""
+    return config.getoption("dist") != "no" and bool(config.getoption("tx"))
 
 
 @pytest.hookimpl(tryfirst=True)
-def pytest_cmdline_main(config):
+def pytest_cmdline_main(config: pytest.Config) -> None:
+    if config.option.distload:
+        config.option.dist = "load"
+
     usepdb = config.getoption("usepdb", False)  # a core option
     if config.option.numprocesses in ("auto", "logical"):
         if usepdb:
@@ -230,13 +322,16 @@ def pytest_cmdline_main(config):
         if config.option.maxprocesses:
             numprocesses = min(numprocesses, config.option.maxprocesses)
         config.option.tx = ["popen"] * numprocesses
-    if config.option.distload:
-        config.option.dist = "load"
+
+    if config.option.numprocesses == 0:
+        config.option.dist = "no"
+        config.option.tx = []
+
     val = config.getvalue
-    if not val("collectonly") and val("dist") != "no" and usepdb:
+    if not val("collectonly") and _is_distribution_mode(config) and usepdb:
         raise pytest.UsageError(
             "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
-        )  # noqa: E501
+        )
 
 
 # -------------------------------------------------------------------------
@@ -244,16 +339,20 @@ def pytest_cmdline_main(config):
 # -------------------------------------------------------------------------
 
 
-def is_xdist_worker(request_or_session) -> bool:
-    """Return `True` if this is an xdist worker, `False` otherwise
+def is_xdist_worker(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise.
 
     :param request_or_session: the `pytest` `request` or `session` object
     """
     return hasattr(request_or_session.config, "workerinput")
 
 
-def is_xdist_controller(request_or_session) -> bool:
-    """Return `True` if this is the xdist controller, `False` otherwise
+def is_xdist_controller(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> bool:
+    """Return `True` if this is the xdist controller, `False` otherwise.
 
     Note: this method also returns `False` when distribution has not been
     activated at all.
@@ -270,7 +369,9 @@ def is_xdist_controller(request_or_session) -> bool:
 is_xdist_master = is_xdist_controller
 
 
-def get_xdist_worker_id(request_or_session):
+def get_xdist_worker_id(
+    request_or_session: pytest.FixtureRequest | pytest.Session,
+) -> str:
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the controller node.
 
@@ -280,14 +381,15 @@ def get_xdist_worker_id(request_or_session):
     :param request_or_session: the `pytest` `request` or `session` object
     """
     if hasattr(request_or_session.config, "workerinput"):
-        return request_or_session.config.workerinput["workerid"]
+        workerid: str = request_or_session.config.workerinput["workerid"]
+        return workerid
     else:
         # TODO: remove "master", ideally for a None
         return "master"
 
 
 @pytest.fixture(scope="session")
-def worker_id(request):
+def worker_id(request: pytest.FixtureRequest) -> str:
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
@@ -296,9 +398,10 @@ def worker_id(request):
 
 
 @pytest.fixture(scope="session")
-def testrun_uid(request):
+def testrun_uid(request: pytest.FixtureRequest) -> str:
     """Return the unique id of the current test."""
     if hasattr(request.config, "workerinput"):
-        return request.config.workerinput["testrunuid"]
+        testrunid: str = request.config.workerinput["testrunuid"]
+        return testrunid
     else:
         return uuid.uuid4().hex

src/xdist/remote.py

@@ -6,25 +6,60 @@
 needs not to be installed in remote environments.
 """
 
-import sys
+from __future__ import annotations
+
+import collections
+from collections.abc import Generator
+from collections.abc import Iterable
+from collections.abc import Sequence
+import contextlib
+import enum
 import os
+import sys
 import time
+from typing import Any
+from typing import Literal
+from typing import TypedDict
+from typing import Union
+import warnings
 
-import py
+from _pytest.config import _prepareconfig
+import execnet
 import pytest
-from execnet.gateway_base import dumps, DumpError
 
-from _pytest.config import _prepareconfig, Config
 
 try:
     from setproctitle import setproctitle
 except ImportError:
 
-    def setproctitle(title):
+    def setproctitle(title: str) -> None:
         pass
 
 
-def worker_title(title):
+class Producer:
+    """
+    Simplified implementation of the same interface as py.log, for backward compatibility
+    since we dropped the dependency on pylib.
+    Note: this is defined here because this module can't depend on xdist, so we need
+    to have the other way around.
+    """
+
+    def __init__(self, name: str, *, enabled: bool = True) -> None:
+        self.name = name
+        self.enabled = enabled
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r}, enabled={self.enabled})"
+
+    def __call__(self, *a: Any, **k: Any) -> None:
+        if self.enabled:
+            print(f"[{self.name}]", *a, **k, file=sys.stderr)
+
+    def __getattr__(self, name: str) -> Producer:
+        return type(self)(name, enabled=self.enabled)
+
+
+def worker_title(title: str) -> None:
     try:
         setproctitle(title)
     except Exception:
@@ -32,83 +67,165 @@ def worker_title(title):
         pass
 
 
+class Marker(enum.Enum):
+    SHUTDOWN = 0
+
+
+class TestQueue:
+    """A simple queue that can be inspected and modified while the lock is held via the ``lock()`` method."""
+
+    Item = Union[int, Literal[Marker.SHUTDOWN]]
+
+    def __init__(self, execmodel: execnet.gateway_base.ExecModel):
+        self._items: collections.deque[TestQueue.Item] = collections.deque()
+        self._lock = execmodel.RLock()  # type: ignore[no-untyped-call]
+        self._has_items_event = execmodel.Event()
+
+    def get(self) -> Item:
+        while True:
+            with self.lock() as locked_items:
+                if locked_items:
+                    return locked_items.popleft()
+
+            self._has_items_event.wait()
+
+    def put(self, item: Item) -> None:
+        with self.lock() as locked_items:
+            locked_items.append(item)
+
+    def replace(self, iterable: Iterable[Item]) -> None:
+        with self.lock():
+            self._items = collections.deque(iterable)
+
+    @contextlib.contextmanager
+    def lock(self) -> Generator[collections.deque[Item]]:
+        with self._lock:
+            try:
+                yield self._items
+            finally:
+                if self._items:
+                    self._has_items_event.set()
+                else:
+                    self._has_items_event.clear()
+
+
 class WorkerInteractor:
-    def __init__(self, config, channel):
+    def __init__(self, config: pytest.Config, channel: execnet.Channel) -> None:
         self.config = config
-        self.workerid = config.workerinput.get("workerid", "?")
-        self.testrunuid = config.workerinput["testrunuid"]
-        self.log = py.log.Producer("worker-%s" % self.workerid)
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
+        workerinput: dict[str, Any] = config.workerinput  # type: ignore[attr-defined]
+        self.workerid = workerinput.get("workerid", "?")
+        self.testrunuid = workerinput["testrunuid"]
+        self.log = Producer(f"worker-{self.workerid}", enabled=config.option.debug)
         self.channel = channel
+        self.torun = TestQueue(self.channel.gateway.execmodel)
+        self.nextitem_index: int | None | Literal[Marker.SHUTDOWN] = None
         config.pluginmanager.register(self)
 
-    def sendevent(self, name, **kwargs):
+    def sendevent(self, name: str, **kwargs: object) -> None:
         self.log("sending", name, kwargs)
         self.channel.send((name, kwargs))
 
     @pytest.hookimpl
-    def pytest_internalerror(self, excrepr):
+    def pytest_internalerror(self, excrepr: object) -> None:
         formatted_error = str(excrepr)
         for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
         interactor.sendevent("internal_error", formatted_error=formatted_error)
 
     @pytest.hookimpl
-    def pytest_sessionstart(self, session):
+    def pytest_sessionstart(self, session: pytest.Session) -> None:
         self.session = session
         workerinfo = getinfodict()
         self.sendevent("workerready", workerinfo=workerinfo)
 
     @pytest.hookimpl(hookwrapper=True)
-    def pytest_sessionfinish(self, exitstatus):
+    def pytest_sessionfinish(self, exitstatus: int) -> Generator[None, object, None]:
+        workeroutput: dict[str, Any] = self.config.workeroutput  # type: ignore[attr-defined]
         # in pytest 5.0+, exitstatus is an IntEnum object
-        self.config.workeroutput["exitstatus"] = int(exitstatus)
+        workeroutput["exitstatus"] = int(exitstatus)
+        workeroutput["shouldfail"] = self.session.shouldfail
+        workeroutput["shouldstop"] = self.session.shouldstop
         yield
-        self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
+        self.sendevent("workerfinished", workeroutput=workeroutput)
 
     @pytest.hookimpl
-    def pytest_collection(self, session):
+    def pytest_collection(self) -> None:
         self.sendevent("collectionstart")
 
-    @pytest.hookimpl
-    def pytest_runtestloop(self, session):
-        self.log("entering main loop")
-        torun = []
-        while 1:
-            try:
-                name, kwargs = self.channel.receive()
-            except EOFError:
-                return True
+    def handle_command(
+        self, command: tuple[str, dict[str, Any]] | Literal[Marker.SHUTDOWN]
+    ) -> None:
+        if command is Marker.SHUTDOWN:
+            self.torun.put(Marker.SHUTDOWN)
+            return
+
+        name, kwargs = command
+
         self.log("received command", name, kwargs)
         if name == "runtests":
-                torun.extend(kwargs["indices"])
+            for i in kwargs["indices"]:
+                self.torun.put(i)
         elif name == "runtests_all":
-                torun.extend(range(len(session.items)))
-            self.log("items to run:", torun)
-            # only run if we have an item and a next item
-            while len(torun) >= 2:
-                self.run_one_test(torun)
-            if name == "shutdown":
-                if torun:
-                    self.run_one_test(torun)
+            for i in range(len(self.session.items)):
+                self.torun.put(i)
+        elif name == "shutdown":
+            self.torun.put(Marker.SHUTDOWN)
+        elif name == "steal":
+            self.steal(kwargs["indices"])
+
+    def steal(self, indices: Sequence[int]) -> None:
+        """
+        Remove tests from the queue.
+
+        Removes either all requested tests, or none, if some of these tests
+        are not in the queue (for example, if they were processed already).
+
+        :param indices: indices of the tests to remove.
+        """
+        requested_set = set(indices)
+
+        with self.torun.lock() as locked_queue:
+            stolen = list(item for item in locked_queue if item in requested_set)
+
+            # Stealing only if all requested tests are still pending
+            if len(stolen) == len(requested_set):
+                self.torun.replace(
+                    item for item in locked_queue if item not in requested_set
+                )
+            else:
+                stolen = []
+
+        self.sendevent("unscheduled", indices=stolen)
+
+    @pytest.hookimpl
+    def pytest_runtestloop(self, session: pytest.Session) -> bool:
+        self.log("entering main loop")
+        self.channel.setcallback(self.handle_command, endmarker=Marker.SHUTDOWN)
+        self.nextitem_index = self.torun.get()
+        while self.nextitem_index is not Marker.SHUTDOWN:
+            self.run_one_test()
+            if session.shouldfail or session.shouldstop:
                 break
         return True
 
-    def run_one_test(self, torun):
+    def run_one_test(self) -> None:
+        assert isinstance(self.nextitem_index, int)
+        self.item_index = self.nextitem_index
+        self.nextitem_index = self.torun.get()
+
         items = self.session.items
-        self.item_index = torun.pop(0)
         item = items[self.item_index]
-        if torun:
-            nextitem = items[torun[0]]
-        else:
+        if self.nextitem_index is Marker.SHUTDOWN:
             nextitem = None
+        else:
+            assert self.nextitem_index is not None
+            nextitem = items[self.nextitem_index]
 
         worker_title("[pytest-xdist running] %s" % item.nodeid)
 
-        start = time.time()
+        start = time.perf_counter()
         self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
-        duration = time.time() - start
+        duration = time.perf_counter() - start
 
         worker_title("[pytest-xdist idle]")
 
@@ -116,43 +233,52 @@ class WorkerInteractor:
             "runtest_protocol_complete", item_index=self.item_index, duration=duration
         )
 
-    def pytest_collection_modifyitems(self, session, config, items):
+    def pytest_collection_modifyitems(
+        self,
+        config: pytest.Config,
+        items: list[pytest.Item],
+    ) -> None:
         # add the group name to nodeid as suffix if --dist=loadgroup
         if config.getvalue("loadgroup"):
             for item in items:
-                mark = item.get_closest_marker("xdist_group")
-                if not mark:
-                    continue
-                gname = (
+                gnames: set[str] = set()
+                for mark in item.iter_markers("xdist_group"):
+                    name = (
                         mark.args[0]
                         if len(mark.args) > 0
                         else mark.kwargs.get("name", "default")
                     )
-                item._nodeid = "{}@{}".format(item.nodeid, gname)
+                    gnames.add(str(name))
+                if not gnames:
+                    continue
+                item._nodeid = f"{item.nodeid}@{'_'.join(sorted(gnames))}"
 
     @pytest.hookimpl
-    def pytest_collection_finish(self, session):
-        try:
-            topdir = str(self.config.rootpath)
-        except AttributeError:  # pytest <= 6.1.0
-            topdir = str(self.config.rootdir)
-
+    def pytest_collection_finish(self, session: pytest.Session) -> None:
         self.sendevent(
             "collectionfinish",
-            topdir=topdir,
+            topdir=str(self.config.rootpath),
             ids=[item.nodeid for item in session.items],
         )
 
     @pytest.hookimpl
-    def pytest_runtest_logstart(self, nodeid, location):
+    def pytest_runtest_logstart(
+        self,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
     @pytest.hookimpl
-    def pytest_runtest_logfinish(self, nodeid, location):
+    def pytest_runtest_logfinish(
+        self,
+        nodeid: str,
+        location: tuple[str, int | None, str],
+    ) -> None:
         self.sendevent("logfinish", nodeid=nodeid, location=location)
 
     @pytest.hookimpl
-    def pytest_runtest_logreport(self, report):
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
         data = self.config.hook.pytest_report_to_serializable(
             config=self.config, report=report
         )
@@ -163,7 +289,7 @@ class WorkerInteractor:
         self.sendevent("testreport", data=data)
 
     @pytest.hookimpl
-    def pytest_collectreport(self, report):
+    def pytest_collectreport(self, report: pytest.CollectReport) -> None:
         # send only reports that have not passed to controller as optimization (#330)
         if not report.passed:
             data = self.config.hook.pytest_report_to_serializable(
@@ -172,7 +298,13 @@ class WorkerInteractor:
             self.sendevent("collectreport", data=data)
 
     @pytest.hookimpl
-    def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+    def pytest_warning_recorded(
+        self,
+        warning_message: warnings.WarningMessage,
+        when: str,
+        nodeid: str,
+        location: tuple[str, int, str] | None,
+    ) -> None:
         self.sendevent(
             "warning_recorded",
             warning_message_data=serialize_warning_message(warning_message),
@@ -182,7 +314,9 @@ class WorkerInteractor:
         )
 
 
-def serialize_warning_message(warning_message):
+def serialize_warning_message(
+    warning_message: warnings.WarningMessage,
+) -> dict[str, Any]:
     if isinstance(warning_message.message, Warning):
         message_module = type(warning_message.message).__module__
         message_class_name = type(warning_message.message).__name__
@@ -190,8 +324,8 @@ def serialize_warning_message(warning_message):
         # check now if we can serialize the warning arguments (#349)
         # if not, we will just use the exception message on the controller node
         try:
-            dumps(warning_message.message.args)
-        except DumpError:
+            execnet.dumps(warning_message.message.args)
+        except execnet.DumpError:
             message_args = None
         else:
             message_args = warning_message.message.args
@@ -216,27 +350,38 @@ def serialize_warning_message(warning_message):
         "category_class_name": category_class_name,
     }
     # access private _WARNING_DETAILS because the attributes vary between Python versions
-    for attr_name in warning_message._WARNING_DETAILS:
+    for attr_name in warning_message._WARNING_DETAILS:  # type: ignore[attr-defined]
         if attr_name in ("message", "category"):
             continue
         attr = getattr(warning_message, attr_name)
         # Check if we can serialize the warning detail, marking `None` otherwise
         # Note that we need to define the attr (even as `None`) to allow deserializing
         try:
-            dumps(attr)
-        except DumpError:
+            execnet.dumps(attr)
+        except execnet.DumpError:
             result[attr_name] = repr(attr)
         else:
             result[attr_name] = attr
     return result
 
 
-def getinfodict():
+class WorkerInfo(TypedDict):
+    version: str
+    version_info: tuple[int, int, int, str, int]
+    sysplatform: str
+    platform: str
+    executable: str
+    cwd: str
+    id: str
+    spec: execnet.XSpec
+
+
+def getinfodict() -> WorkerInfo:
     import platform
 
     return dict(
         version=sys.version,
-        version_info=tuple(sys.version_info),
+        version_info=tuple(sys.version_info),  # type: ignore[typeddict-item]
         sysplatform=sys.platform,
         platform=platform.platform(),
         executable=sys.executable,
@@ -244,12 +389,7 @@ def getinfodict():
     )
 
 
-def remote_initconfig(option_dict, args):
-    option_dict["plugins"].append("no:terminal")
-    return Config.fromdictargs(option_dict, args)
-
-
-def setup_config(config, basetemp):
+def setup_config(config: pytest.Config, basetemp: str | None) -> None:
     config.option.loadgroup = config.getvalue("dist") == "loadgroup"
     config.option.looponfail = False
     config.option.usepdb = False
@@ -261,7 +401,7 @@ def setup_config(config, basetemp):
 
 
 if __name__ == "__channelexec__":
-    channel = channel  # type: ignore[name-defined] # noqa: F821
+    channel: execnet.Channel = channel  # type: ignore[name-defined] # noqa: F821, PLW0127
     workerinput, args, option_dict, change_sys_path = channel.receive()  # type: ignore[name-defined]
 
     if change_sys_path is None:
@@ -277,11 +417,7 @@ if __name__ == "__channelexec__":
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
     os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
 
-    if hasattr(Config, "InvocationParams"):
     config = _prepareconfig(args, None)
-    else:
-        config = remote_initconfig(option_dict, args)
-        config.args = args
 
     setup_config(config, option_dict.get("basetemp"))
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])

src/xdist/report.py

@@ -1,7 +1,15 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
 from difflib import unified_diff
 
 
-def report_collection_diff(from_collection, to_collection, from_id, to_id):
+def report_collection_diff(
+    from_collection: Sequence[str],
+    to_collection: Sequence[str],
+    from_id: str,
+    to_id: str,
+) -> str | None:
     """Report the collected test difference between two nodes.
 
     :returns: detailed message describing the difference between the given
@@ -14,7 +22,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
     error_message = (
         "Different tests were collected between {from_id} and {to_id}. "
         "The difference is:\n"
-        "{diff}"
+        "{diff}\n"
+        "To see why this happens see 'Known limitations' in documentation "
+        "for pytest-xdist"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
     msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/__init__.py

@@ -1,5 +1,7 @@
-from xdist.scheduler.each import EachScheduling  # noqa
-from xdist.scheduler.load import LoadScheduling  # noqa
-from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
-from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
-from xdist.scheduler.loadgroup import LoadGroupScheduling  # noqa
+from xdist.scheduler.each import EachScheduling as EachScheduling
+from xdist.scheduler.load import LoadScheduling as LoadScheduling
+from xdist.scheduler.loadfile import LoadFileScheduling as LoadFileScheduling
+from xdist.scheduler.loadgroup import LoadGroupScheduling as LoadGroupScheduling
+from xdist.scheduler.loadscope import LoadScopeScheduling as LoadScopeScheduling
+from xdist.scheduler.protocol import Scheduling as Scheduling
+from xdist.scheduler.worksteal import WorkStealingScheduling as WorkStealingScheduling

src/xdist/scheduler/each.py

@@ -1,11 +1,17 @@
-from py.log import Producer
+from __future__ import annotations
 
-from xdist.workermanage import parse_spec_config
+from collections.abc import Sequence
+
+import pytest
+
+from xdist.remote import Producer
 from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
 
 
 class EachScheduling:
-    """Implement scheduling of test items on all nodes
+    """Implement scheduling of test items on all nodes.
 
     If a node gets added after the test run is started then it is
     assumed to replace a node which got removed before it finished
@@ -18,13 +24,13 @@ class EachScheduling:
     assigned the remaining items from the removed node.
     """
 
-    def __init__(self, config, log=None):
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
         self.config = config
-        self.numnodes = len(parse_spec_config(config))
-        self.node2collection = {}
-        self.node2pending = {}
-        self._started = []
-        self._removed2pending = {}
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self._started: list[WorkerController] = []
+        self._removed2pending: dict[WorkerController, list[int]] = {}
         if log is None:
             self.log = Producer("eachsched")
         else:
@@ -32,12 +38,12 @@ class EachScheduling:
         self.collection_is_completed = False
 
     @property
-    def nodes(self):
+    def nodes(self) -> list[WorkerController]:
         """A list of all nodes in the scheduler."""
         return list(self.node2pending.keys())
 
     @property
-    def tests_finished(self):
+    def tests_finished(self) -> bool:
         if not self.collection_is_completed:
             return False
         if self._removed2pending:
@@ -48,8 +54,8 @@ class EachScheduling:
         return True
 
     @property
-    def has_pending(self):
-        """Return True if there are pending test items
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
 
         This indicates that collection has finished and nodes are
         still processing test items, so this can be thought of as
@@ -60,12 +66,14 @@ class EachScheduling:
                 return True
         return False
 
-    def add_node(self, node):
+    def add_node(self, node: WorkerController) -> None:
         assert node not in self.node2pending
         self.node2pending[node] = []
 
-    def add_node_collection(self, node, collection):
-        """Add the collected test items from a node
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
 
         Collection is complete once all nodes have submitted their
         collection.  In this case its pending list is set to an empty
@@ -98,29 +106,33 @@ class EachScheduling:
                     self.node2pending[node] = pending
                     break
 
-    def mark_test_complete(self, node, item_index, duration=0):
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
         self.node2pending[node].remove(item_index)
 
-    def mark_test_pending(self, item):
-        self.pending.insert(
-            0,
-            self.collection.index(item),
-        )
-        for node in self.node2pending:
-            self.check_schedule(node)
+    def mark_test_pending(self, item: str) -> None:
+        raise NotImplementedError()
 
-    def remove_node(self, node):
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def remove_node(self, node: WorkerController) -> str | None:
         # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)
         if not pending:
-            return
+            return None
         crashitem = self.node2collection[node][pending.pop(0)]
         if pending:
             self._removed2pending[node] = pending
         return crashitem
 
-    def schedule(self):
-        """Schedule the test items on the nodes
+    def schedule(self) -> None:
+        """Schedule the test items on the nodes.
 
         If the node's pending list is empty it is a new node which
         needs to run all the tests.  If the pending list is already

src/xdist/scheduler/load.py

@@ -1,10 +1,14 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
 from itertools import cycle
 
-from py.log import Producer
-from _pytest.runner import CollectReport
+import pytest
 
-from xdist.workermanage import parse_spec_config
+from xdist.remote import Producer
 from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
 
 
 class LoadScheduling:
@@ -23,7 +27,7 @@ class LoadScheduling:
     submit a collection. This is used to know when all nodes have
     finished collection or how large the chunks need to be created.
 
-    Attributes:
+    Attributes::
 
     :numnodes: The expected number of nodes taking part.  The actual
        number of nodes will vary during the scheduler's lifetime as
@@ -53,25 +57,26 @@ class LoadScheduling:
     :config: Config object, used for handling hooks.
     """
 
-    def __init__(self, config, log=None):
-        self.numnodes = len(parse_spec_config(config))
-        self.node2collection = {}
-        self.node2pending = {}
-        self.pending = []
-        self.collection = None
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self.pending: list[int] = []
+        self.collection: list[str] | None = None
         if log is None:
             self.log = Producer("loadsched")
         else:
             self.log = log.loadsched
         self.config = config
+        self.maxschedchunk = self.config.getoption("maxschedchunk")
 
     @property
-    def nodes(self):
+    def nodes(self) -> list[WorkerController]:
         """A list of all nodes in the scheduler."""
         return list(self.node2pending.keys())
 
     @property
-    def collection_is_completed(self):
+    def collection_is_completed(self) -> bool:
         """Boolean indication initial test collection is complete.
 
         This is a boolean indicating all initial participating nodes
@@ -81,7 +86,7 @@ class LoadScheduling:
         return len(self.node2collection) >= self.numnodes
 
     @property
-    def tests_finished(self):
+    def tests_finished(self) -> bool:
         """Return True if all tests have been executed by the nodes."""
         if not self.collection_is_completed:
             return False
@@ -93,8 +98,8 @@ class LoadScheduling:
         return True
 
     @property
-    def has_pending(self):
-        """Return True if there are pending test items
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
 
         This indicates that collection has finished and nodes are
         still processing test items, so this can be thought of as
@@ -107,7 +112,7 @@ class LoadScheduling:
                 return True
         return False
 
-    def add_node(self, node):
+    def add_node(self, node: WorkerController) -> None:
         """Add a new node to the scheduler.
 
         From now on the node will be allocated chunks of tests to
@@ -119,8 +124,10 @@ class LoadScheduling:
         assert node not in self.node2pending
         self.node2pending[node] = []
 
-    def add_node_collection(self, node, collection):
-        """Add the collected test items from a node
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
 
         The collection is stored in the ``.node2collection`` map.
         Called by the ``DSession.worker_collectionfinish`` hook.
@@ -140,8 +147,10 @@ class LoadScheduling:
                 return
         self.node2collection[node] = list(collection)
 
-    def mark_test_complete(self, node, item_index, duration=0):
-        """Mark test item as completed by node
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
+        """Mark test item as completed by node.
 
         The duration it took to execute the item is used as a hint to
         the scheduler.
@@ -151,7 +160,8 @@ class LoadScheduling:
         self.node2pending[node].remove(item_index)
         self.check_schedule(node, duration=duration)
 
-    def mark_test_pending(self, item):
+    def mark_test_pending(self, item: str) -> None:
+        assert self.collection is not None
         self.pending.insert(
             0,
             self.collection.index(item),
@@ -159,8 +169,15 @@ class LoadScheduling:
         for node in self.node2pending:
             self.check_schedule(node)
 
-    def check_schedule(self, node, duration=0):
-        """Maybe schedule new items on the node
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def check_schedule(self, node: WorkerController, duration: float = 0) -> None:
+        """Maybe schedule new items on the node.
 
         If there are any globally pending nodes left then this will
         check if the given node should be given any more tests.  The
@@ -185,14 +202,16 @@ class LoadScheduling:
                     # so let's rather wait with sending new items
                     return
                 num_send = items_per_node_max - len(node_pending)
-                self._send_tests(node, num_send)
+                # keep at least 2 tests pending even if --maxschedchunk=1
+                maxschedchunk = max(2 - len(node_pending), self.maxschedchunk)
+                self._send_tests(node, min(num_send, maxschedchunk))
         else:
             node.shutdown()
 
         self.log("num items waiting for node:", len(self.pending))
 
-    def remove_node(self, node):
-        """Remove a node from the scheduler
+    def remove_node(self, node: WorkerController) -> str | None:
+        """Remove a node from the scheduler.
 
         This should be called either when the node crashed or at
         shutdown time.  In the former case any pending items assigned
@@ -206,17 +225,18 @@ class LoadScheduling:
         """
         pending = self.node2pending.pop(node)
         if not pending:
-            return
+            return None
 
         # The node crashed, reassing pending items
+        assert self.collection is not None
         crashitem = self.collection[pending.pop(0)]
         self.pending.extend(pending)
         for node in self.node2pending:
             self.check_schedule(node)
         return crashitem
 
-    def schedule(self):
-        """Initiate distribution of the test collection
+    def schedule(self) -> None:
+        """Initiate distribution of the test collection.
 
         Initiate scheduling of the items across the nodes.  If this
         gets called again later it behaves the same as calling
@@ -240,35 +260,53 @@ class LoadScheduling:
             return
 
         # Collections are identical, create the index of pending items.
-        self.collection = list(self.node2collection.values())[0]
+        self.collection = next(iter(self.node2collection.values()))
         self.pending[:] = range(len(self.collection))
         if not self.collection:
             return
 
+        if self.maxschedchunk is None:
+            self.maxschedchunk = len(self.collection)
+
         # Send a batch of tests to run. If we don't have at least two
         # tests per node, we have to send them all so that we can send
         # shutdown signals and get all nodes working.
-        initial_batch = max(len(self.pending) // 4, 2 * len(self.nodes))
-
-        # distribute tests round-robin up to the batch size
-        # (or until we run out)
+        if len(self.pending) < 2 * len(self.nodes):
+            # Distribute tests round-robin. Try to load all nodes if there are
+            # enough tests. The other branch tries sends at least 2 tests
+            # to each node - which is suboptimal when you have less than
+            # 2 * len(nodes) tests.
             nodes = cycle(self.nodes)
-        for i in range(initial_batch):
+            for _ in range(len(self.pending)):
                 self._send_tests(next(nodes), 1)
+        else:
+            # Send batches of consecutive tests. By default, pytest sorts tests
+            # in order for optimal single-threaded execution, minimizing the
+            # number of necessary fixture setup/teardown. Try to keep that
+            # optimal order for every worker.
+
+            # how many items per node do we have about?
+            items_per_node = len(self.collection) // len(self.node2pending)
+            # take a fraction of tests for initial distribution
+            node_chunksize = min(items_per_node // 4, self.maxschedchunk)
+            node_chunksize = max(node_chunksize, 2)
+            # and initialize each node with a chunk of tests
+            for node in self.nodes:
+                self._send_tests(node, node_chunksize)
 
         if not self.pending:
             # initial distribution sent all tests, start node shutdown
             for node in self.nodes:
                 node.shutdown()
 
-    def _send_tests(self, node, num):
+    def _send_tests(self, node: WorkerController, num: int) -> None:
         tests_per_node = self.pending[:num]
         if tests_per_node:
             del self.pending[:num]
             self.node2pending[node].extend(tests_per_node)
             node.send_runtest_some(tests_per_node)
 
-    def _check_nodes_have_same_collection(self):
+    def _check_nodes_have_same_collection(self) -> bool:
         """Return True if all nodes have collected the same items.
 
         If collections differ, this method returns False while logging
@@ -286,8 +324,11 @@ class LoadScheduling:
                 same_collection = False
                 self.log(msg)
                 if self.config is not None:
-                    rep = CollectReport(
-                        node.gateway.id, "failed", longrepr=msg, result=[]
+                    rep = pytest.CollectReport(
+                        nodeid=node.gateway.id,
+                        outcome="failed",
+                        longrepr=msg,
+                        result=[],
                     )
                     self.config.hook.pytest_collectreport(report=rep)
 

src/xdist/scheduler/loadfile.py

@@ -1,5 +1,10 @@
+from __future__ import annotations
+
+import pytest
+
+from xdist.remote import Producer
+
 from .loadscope import LoadScopeScheduling
-from py.log import Producer
 
 
 class LoadFileScheduling(LoadScopeScheduling):
@@ -20,14 +25,14 @@ class LoadFileScheduling(LoadScopeScheduling):
     This class behaves very much like LoadScopeScheduling, but with a file-level scope.
     """
 
-    def __init__(self, config, log=None):
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
         super().__init__(config, log)
         if log is None:
             self.log = Producer("loadfilesched")
         else:
             self.log = log.loadfilesched
 
-    def _split_scope(self, nodeid):
+    def _split_scope(self, nodeid: str) -> str:
         """Determine the scope (grouping) of a nodeid.
 
         There are usually 3 cases for a nodeid::
@@ -43,8 +48,11 @@ class LoadFileScheduling(LoadScopeScheduling):
         This function will group tests with the scope determined by splitting
         the first ``::`` from the left. That is, test will be grouped in a
         single work unit when they reside in the same file.
+
         In the above example, scopes will be::
 
+        .. code-block:: text
+
             example/loadsuite/test/test_beta.py
             example/loadsuite/test/test_delta.py
             example/loadsuite/epsilon/__init__.py

src/xdist/scheduler/loadgroup.py

@@ -1,5 +1,10 @@
+from __future__ import annotations
+
+import pytest
+
+from xdist.remote import Producer
+
 from .loadscope import LoadScopeScheduling
-from py.log import Producer
 
 
 class LoadGroupScheduling(LoadScopeScheduling):
@@ -9,14 +14,14 @@ class LoadGroupScheduling(LoadScopeScheduling):
     instead of the module or class to which they belong to.
     """
 
-    def __init__(self, config, log=None):
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
         super().__init__(config, log)
         if log is None:
             self.log = Producer("loadgroupsched")
         else:
             self.log = log.loadgroupsched
 
-    def _split_scope(self, nodeid):
+    def _split_scope(self, nodeid: str) -> str:
         """Determine the scope (grouping) of a nodeid.
 
         There are usually 3 cases for a nodeid::

src/xdist/scheduler/loadscope.py

@@ -1,9 +1,15 @@
-from collections import OrderedDict
+from __future__ import annotations
 
-from _pytest.runner import CollectReport
-from py.log import Producer
+from collections import OrderedDict
+from collections.abc import Sequence
+from typing import NoReturn
+
+import pytest
+
+from xdist.remote import Producer
 from xdist.report import report_collection_diff
-from xdist.workermanage import parse_spec_config
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
 
 
 class LoadScopeScheduling:
@@ -21,7 +27,7 @@ class LoadScopeScheduling:
     When created, ``numnodes`` defines how many nodes are expected to submit a
     collection. This is used to know when all nodes have finished collection.
 
-    Attributes:
+    Attributes::
 
     :numnodes: The expected number of nodes taking part.  The actual number of
        nodes will vary during the scheduler's lifetime as nodes are added by
@@ -84,13 +90,13 @@ class LoadScopeScheduling:
     :config: Config object, used for handling hooks.
     """
 
-    def __init__(self, config, log=None):
-        self.numnodes = len(parse_spec_config(config))
-        self.collection = None
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.collection: list[str] | None = None
 
-        self.workqueue = OrderedDict()
-        self.assigned_work = OrderedDict()
-        self.registered_collections = OrderedDict()
+        self.workqueue: OrderedDict[str, dict[str, bool]] = OrderedDict()
+        self.assigned_work: dict[WorkerController, dict[str, dict[str, bool]]] = {}
+        self.registered_collections: dict[WorkerController, list[str]] = {}
 
         if log is None:
             self.log = Producer("loadscopesched")
@@ -100,12 +106,12 @@ class LoadScopeScheduling:
         self.config = config
 
     @property
-    def nodes(self):
+    def nodes(self) -> list[WorkerController]:
         """A list of all active nodes in the scheduler."""
         return list(self.assigned_work.keys())
 
     @property
-    def collection_is_completed(self):
+    def collection_is_completed(self) -> bool:
         """Boolean indication initial test collection is complete.
 
         This is a boolean indicating all initial participating nodes have
@@ -115,7 +121,7 @@ class LoadScopeScheduling:
         return len(self.registered_collections) >= self.numnodes
 
     @property
-    def tests_finished(self):
+    def tests_finished(self) -> bool:
         """Return True if all tests have been executed by the nodes."""
         if not self.collection_is_completed:
             return False
@@ -130,7 +136,7 @@ class LoadScopeScheduling:
         return True
 
     @property
-    def has_pending(self):
+    def has_pending(self) -> bool:
         """Return True if there are pending test items.
 
         This indicates that collection has finished and nodes are still
@@ -146,7 +152,7 @@ class LoadScopeScheduling:
 
         return False
 
-    def add_node(self, node):
+    def add_node(self, node: WorkerController) -> None:
         """Add a new node to the scheduler.
 
         From now on the node will be assigned work units to be executed.
@@ -155,9 +161,9 @@ class LoadScopeScheduling:
         bootstraps a new node.
         """
         assert node not in self.assigned_work
-        self.assigned_work[node] = OrderedDict()
+        self.assigned_work[node] = {}
 
-    def remove_node(self, node):
+    def remove_node(self, node: WorkerController) -> str | None:
         """Remove a node from the scheduler.
 
         This should be called either when the node crashed or at shutdown time.
@@ -198,7 +204,9 @@ class LoadScopeScheduling:
 
         return crashitem
 
-    def add_node_collection(self, node, collection):
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
         """Add the collected test items from a node.
 
         The collection is stored in the ``.registered_collections`` dictionary.
@@ -207,19 +215,16 @@ class LoadScopeScheduling:
 
         - ``DSession.worker_collectionfinish``.
         """
-
         # Check that add_node() was called on the node before
         assert node in self.assigned_work
 
         # A new node has been added later, perhaps an original one died.
         if self.collection_is_completed:
-
             # Assert that .schedule() should have been called by now
             assert self.collection
 
             # Check that the new collection matches the official collection
             if collection != self.collection:
-
                 other_node = next(iter(self.registered_collections.keys()))
 
                 msg = report_collection_diff(
@@ -230,7 +235,9 @@ class LoadScopeScheduling:
 
         self.registered_collections[node] = list(collection)
 
-    def mark_test_complete(self, node, item_index, duration=0):
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float = 0
+    ) -> None:
         """Mark test item as completed by node.
 
         Called by the hook:
@@ -243,10 +250,17 @@ class LoadScopeScheduling:
         self.assigned_work[node][scope][nodeid] = True
         self._reschedule(node)
 
-    def mark_test_pending(self, item):
+    def mark_test_pending(self, item: str) -> NoReturn:
         raise NotImplementedError()
 
-    def _assign_work_unit(self, node):
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        raise NotImplementedError()
+
+    def _assign_work_unit(self, node: WorkerController) -> None:
         """Assign a work unit to a node."""
         assert self.workqueue
 
@@ -254,7 +268,7 @@ class LoadScopeScheduling:
         scope, work_unit = self.workqueue.popitem(last=False)
 
         # Keep track of the assigned work
-        assigned_to_node = self.assigned_work.setdefault(node, default=OrderedDict())
+        assigned_to_node = self.assigned_work.setdefault(node, {})
         assigned_to_node[scope] = work_unit
 
         # Ask the node to execute the workload
@@ -267,7 +281,7 @@ class LoadScopeScheduling:
 
         node.send_runtest_some(nodeids_indexes)
 
-    def _split_scope(self, nodeid):
+    def _split_scope(self, nodeid: str) -> str:
         """Determine the scope (grouping) of a nodeid.
 
         There are usually 3 cases for a nodeid::
@@ -291,18 +305,17 @@ class LoadScopeScheduling:
         """
         return nodeid.rsplit("::", 1)[0]
 
-    def _pending_of(self, workload):
+    def _pending_of(self, workload: dict[str, dict[str, bool]]) -> int:
         """Return the number of pending tests in a workload."""
         pending = sum(list(scope.values()).count(False) for scope in workload.values())
         return pending
 
-    def _reschedule(self, node):
+    def _reschedule(self, node: WorkerController) -> None:
         """Maybe schedule new items on the node.
 
         If there are any globally pending work units left then this will check
         if the given node should be given any more tests.
         """
-
         # Do not add more work to a node shutting down
         if node.shutting_down:
             return
@@ -322,7 +335,7 @@ class LoadScopeScheduling:
         # Pop one unit of work and assign it
         self._assign_work_unit(node)
 
-    def schedule(self):
+    def schedule(self) -> None:
         """Initiate distribution of the test collection.
 
         Initiate scheduling of the items across the nodes.  If this gets called
@@ -352,21 +365,32 @@ class LoadScopeScheduling:
             return
 
         # Determine chunks of work (scopes)
+        unsorted_workqueue: dict[str, dict[str, bool]] = {}
         for nodeid in self.collection:
             scope = self._split_scope(nodeid)
-            work_unit = self.workqueue.setdefault(scope, default=OrderedDict())
+            work_unit = unsorted_workqueue.setdefault(scope, {})
             work_unit[nodeid] = False
 
+        if self.config.option.loadscopereorder:
+            # Insert tests scopes into work queue ordered by number of tests.
+            for scope, nodeids in sorted(
+                unsorted_workqueue.items(), key=lambda item: -len(item[1])
+            ):
+                self.workqueue[scope] = nodeids
+        else:
+            for scope, nodeids in unsorted_workqueue.items():
+                self.workqueue[scope] = nodeids
+
         # Avoid having more workers than work
         extra_nodes = len(self.nodes) - len(self.workqueue)
 
         if extra_nodes > 0:
-            self.log("Shuting down {} nodes".format(extra_nodes))
+            self.log(f"Shutting down {extra_nodes} nodes")
 
             for _ in range(extra_nodes):
-                unused_node, assigned = self.assigned_work.popitem(last=True)
+                unused_node, assigned = self.assigned_work.popitem()
 
-                self.log("Shuting down unused node {}".format(unused_node))
+                self.log(f"Shutting down unused node {unused_node}")
                 unused_node.shutdown()
 
         # Assign initial workload
@@ -382,7 +406,7 @@ class LoadScopeScheduling:
             for node in self.nodes:
                 node.shutdown()
 
-    def _check_nodes_have_same_collection(self):
+    def _check_nodes_have_same_collection(self) -> bool:
         """Return True if all nodes have collected the same items.
 
         If collections differ, this method returns False while logging
@@ -403,10 +427,12 @@ class LoadScopeScheduling:
             same_collection = False
             self.log(msg)
 
-            if self.config is None:
-                continue
-
-            rep = CollectReport(node.gateway.id, "failed", longrepr=msg, result=[])
+            rep = pytest.CollectReport(
+                nodeid=node.gateway.id,
+                outcome="failed",
+                longrepr=msg,
+                result=[],
+            )
             self.config.hook.pytest_collectreport(report=rep)
 
         return same_collection

src/xdist/scheduler/protocol.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Protocol
+
+from xdist.workermanage import WorkerController
+
+
+class Scheduling(Protocol):
+    @property
+    def nodes(self) -> list[WorkerController]: ...
+
+    @property
+    def collection_is_completed(self) -> bool: ...
+
+    @property
+    def tests_finished(self) -> bool: ...
+
+    @property
+    def has_pending(self) -> bool: ...
+
+    def add_node(self, node: WorkerController) -> None: ...
+
+    def add_node_collection(
+        self,
+        node: WorkerController,
+        collection: Sequence[str],
+    ) -> None: ...
+
+    def mark_test_complete(
+        self,
+        node: WorkerController,
+        item_index: int,
+        duration: float = 0,
+    ) -> None: ...
+
+    def mark_test_pending(self, item: str) -> None: ...
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None: ...
+
+    def remove_node(self, node: WorkerController) -> str | None: ...
+
+    def schedule(self) -> None: ...

src/xdist/scheduler/worksteal.py

@@ -0,0 +1,345 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import NamedTuple
+
+import pytest
+
+from xdist.remote import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_tx_spec_config
+from xdist.workermanage import WorkerController
+
+
+class NodePending(NamedTuple):
+    node: WorkerController
+    pending: list[int]
+
+
+# Every worker needs at least 2 tests in queue - the current and the next one.
+MIN_PENDING = 2
+
+
+class WorkStealingScheduling:
+    """Implement work-stealing scheduling.
+
+    Initially, tests are distributed evenly among all nodes.
+
+    When some node completes most of its assigned tests (when only one pending
+    test remains), an attempt is made to reassign ("steal") some tests from
+    other nodes to this node.
+
+    Attributes::
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+
+    :steal_requested_from_node: The node to which the current "steal" request
+       was sent. ``None`` if there is no request in progress. Only one request
+       can be in progress at any time, the scheduler doesn't send multiple
+       simultaneous requests.
+    """
+
+    def __init__(self, config: pytest.Config, log: Producer | None = None) -> None:
+        self.numnodes = len(parse_tx_spec_config(config))
+        self.node2collection: dict[WorkerController, list[str]] = {}
+        self.node2pending: dict[WorkerController, list[int]] = {}
+        self.pending: list[int] = []
+        self.collection: list[str] | None = None
+        if log is None:
+            self.log = Producer("workstealsched")
+        else:
+            self.log = log.workstealsched
+        self.config = config
+        self.steal_requested_from_node: WorkerController | None = None
+
+    @property
+    def nodes(self) -> list[WorkerController]:
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self) -> bool:
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self) -> bool:
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        if self.steal_requested_from_node is not None:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= MIN_PENDING:
+                return False
+        return True
+
+    @property
+    def has_pending(self) -> bool:
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node: WorkerController) -> None:
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(
+        self, node: WorkerController, collection: Sequence[str]
+    ) -> None:
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(
+                    self.collection, collection, other_node.gateway.id, node.gateway.id
+                )
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(
+        self, node: WorkerController, item_index: int, duration: float | None = None
+    ) -> None:
+        """Mark test item as completed by node.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule()
+
+    def mark_test_pending(self, item: str) -> None:
+        assert self.collection is not None
+        self.pending.insert(
+            0,
+            self.collection.index(item),
+        )
+        self.check_schedule()
+
+    def remove_pending_tests_from_node(
+        self,
+        node: WorkerController,
+        indices: Sequence[int],
+    ) -> None:
+        """Node returned some test indices back in response to 'steal' command.
+
+        This is called by ``DSession.worker_unscheduled``.
+        """
+        assert node is self.steal_requested_from_node
+        self.steal_requested_from_node = None
+
+        indices_set = set(indices)
+        self.node2pending[node] = [
+            i for i in self.node2pending[node] if i not in indices_set
+        ]
+        self.pending.extend(indices)
+        self.check_schedule()
+
+    def check_schedule(self) -> None:
+        """Reschedule tests/perform load balancing."""
+        nodes_up = [
+            NodePending(node, pending)
+            for node, pending in self.node2pending.items()
+            if not node.shutting_down
+        ]
+
+        def get_idle_nodes() -> list[WorkerController]:
+            return [node for node, pending in nodes_up if len(pending) < MIN_PENDING]
+
+        idle_nodes = get_idle_nodes()
+        if not idle_nodes:
+            return
+
+        if self.pending:
+            # Distribute pending tests evenly among idle nodes
+            for i, node in enumerate(idle_nodes):
+                nodes_remaining = len(idle_nodes) - i
+                num_send = len(self.pending) // nodes_remaining
+                self._send_tests(node, num_send)
+
+            idle_nodes = get_idle_nodes()
+            # No need to steal anything if all nodes have enough work to continue
+            if not idle_nodes:
+                return
+
+        # Only one active stealing request is allowed
+        if self.steal_requested_from_node is not None:
+            return
+
+        # Find the node that has the longest test queue
+        steal_from = max(
+            nodes_up, key=lambda node_pending: len(node_pending.pending), default=None
+        )
+
+        if steal_from is None:
+            num_steal = 0
+        else:
+            # Steal half of the test queue - but keep that node running too.
+            # If the node has 2 or less tests queued, stealing will fail
+            # anyway.
+            max_steal = max(0, len(steal_from.pending) - MIN_PENDING)
+            num_steal = min(len(steal_from.pending) // 2, max_steal)
+
+        if num_steal == 0:
+            # Can't get more work - shutdown idle nodes. This will force them
+            # to run the last test now instead of waiting for more tests.
+            for node in idle_nodes:
+                node.shutdown()
+            return
+
+        assert steal_from is not None
+        steal_from.node.send_steal(steal_from.pending[-num_steal:])
+        self.steal_requested_from_node = steal_from.node
+
+    def remove_node(self, node: WorkerController) -> str | None:
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+        """
+        pending = self.node2pending.pop(node)
+
+        # If node was removed without completing its assigned tests - it crashed
+        if pending:
+            assert self.collection is not None
+            crashitem = self.collection[pending.pop(0)]
+        else:
+            crashitem = None
+
+        self.pending.extend(pending)
+
+        # Dead node won't respond to "steal" request
+        if self.steal_requested_from_node is node:
+            self.steal_requested_from_node = None
+
+        self.check_schedule()
+        return crashitem
+
+    def schedule(self) -> None:
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            self.check_schedule()
+            return
+
+        if not self._check_nodes_have_same_collection():
+            self.log("**Different tests collected, aborting run**")
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = next(iter(self.node2collection.values()))
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        self.check_schedule()
+
+    def _send_tests(self, node: WorkerController, num: int) -> None:
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self) -> bool:
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col, collection, first_node.gateway.id, node.gateway.id
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = pytest.CollectReport(
+                        nodeid=node.gateway.id,
+                        outcome="failed",
+                        longrepr=msg,
+                        result=[],
+                    )
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

src/xdist/workermanage.py

@@ -1,20 +1,33 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+import enum
 import fnmatch
 import os
+from pathlib import Path
 import re
 import sys
+from typing import Any
+from typing import Callable
+from typing import Literal
+from typing import Union
 import uuid
+import warnings
 
-import py
-import pytest
 import execnet
+import pytest
 
-import xdist.remote
 from xdist.plugin import _sys_path
+import xdist.remote
+from xdist.remote import Producer
+from xdist.remote import WorkerInfo
 
 
-def parse_spec_config(config):
+def parse_tx_spec_config(config: pytest.Config) -> list[str]:
     xspeclist = []
-    for xspec in config.getvalue("tx"):
+    tx: list[str] = config.getvalue("tx")
+    for xspec in tx:
         i = xspec.find("*")
         try:
             num = int(xspec[:i])
@@ -33,64 +46,101 @@ class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
-    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
+    def __init__(
+        self,
+        config: pytest.Config,
+        specs: Sequence[execnet.XSpec | str] | None = None,
+        defaultchdir: str = "pyexecnetcache",
+    ) -> None:
         self.config = config
         self.trace = self.config.trace.get("nodemanager")
         self.testrunuid = self.config.getoption("testrunuid")
         if self.testrunuid is None:
             self.testrunuid = uuid.uuid4().hex
-        self.group = execnet.Group()
+        self.group = execnet.Group(execmodel="main_thread_only")
+        for proxy_spec in self._getpxspecs():
+            # Proxy gateways do not run workers, and are meant to be passed with the `via` attribute
+            # to additional gateways.
+            # They are useful for running multiple workers on remote machines.
+            if getattr(proxy_spec, "id", None) is None:
+                raise pytest.UsageError(
+                    f"Proxy gateway {proxy_spec} must include an id"
+                )
+            self.group.makegateway(proxy_spec)
         if specs is None:
-            specs = self._getxspecs()
-        self.specs = []
+            specs = self._gettxspecs()
+        self.specs: list[execnet.XSpec] = []
         for spec in specs:
             if not isinstance(spec, execnet.XSpec):
                 spec = execnet.XSpec(spec)
+            if getattr(spec, "execmodel", None) != "main_thread_only":
+                spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
             if not spec.chdir and not spec.popen:
                 spec.chdir = defaultchdir
             self.group.allocate_id(spec)
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
         self.rsyncoptions = self._getrsyncoptions()
-        self._rsynced_specs = set()
+        self._rsynced_specs: set[tuple[Any, Any]] = set()
 
-    def rsync_roots(self, gateway):
+    def rsync_roots(self, gateway: execnet.Gateway) -> None:
         """Rsync the set of roots to the node's gateway cwd."""
         if self.roots:
             for root in self.roots:
                 self.rsync(gateway, root, **self.rsyncoptions)
 
-    def setup_nodes(self, putevent):
+    def setup_nodes(
+        self,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+    ) -> list[WorkerController]:
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        return [self.setup_node(spec, putevent) for spec in self.specs]
+        with ThreadPoolExecutor(max_workers=len(self.specs)) as executor:
+            futs = [
+                executor.submit(self.setup_node, spec, putevent, idx)
+                for idx, spec in enumerate(self.specs)
+            ]
+            return [f.result() for f in futs]
 
-    def setup_node(self, spec, putevent):
+    def setup_node(
+        self,
+        spec: execnet.XSpec,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+        idx: int | None = None,
+    ) -> WorkerController:
+        if getattr(spec, "execmodel", None) != "main_thread_only":
+            spec = execnet.XSpec(f"execmodel=main_thread_only//{spec}")
+        if idx is not None:
+            spec = execnet.XSpec(f"{spec}//id=gw{idx}")
         gw = self.group.makegateway(spec)
         self.config.hook.pytest_xdist_newgateway(gateway=gw)
         self.rsync_roots(gw)
         node = WorkerController(self, gw, self.config, putevent)
-        gw.node = node  # keep the node alive
+        # Keep the node alive.
+        gw.node = node  # type: ignore[attr-defined]
         node.setup()
         self.trace("started node %r" % node)
         return node
 
-    def teardown_nodes(self):
+    def teardown_nodes(self) -> None:
         self.group.terminate(self.EXIT_TIMEOUT)
 
-    def _getxspecs(self):
-        return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
+    def _gettxspecs(self) -> list[execnet.XSpec]:
+        return [execnet.XSpec(x) for x in parse_tx_spec_config(self.config)]
 
-    def _getrsyncdirs(self):
+    def _getpxspecs(self) -> list[execnet.XSpec]:
+        return [execnet.XSpec(x) for x in self.config.getoption("px")]
+
+    def _getrsyncdirs(self) -> list[Path]:
         for spec in self.specs:
             if not spec.popen or spec.chdir:
                 break
         else:
             return []
-        import pytest
         import _pytest
+        import pytest
 
-        def get_dir(p):
+        def get_dir(p: str) -> str:
             """Return the directory path if p is a package or the path to the .py file otherwise."""
             stripped = p.rstrip("co")
             if os.path.basename(stripped) == "__init__.py":
@@ -101,21 +151,21 @@ class NodeManager:
         pytestpath = get_dir(pytest.__file__)
         pytestdir = get_dir(_pytest.__file__)
         config = self.config
-        candidates = [py._pydir, pytestpath, pytestdir]
+        candidates = [pytestpath, pytestdir]
         candidates += config.option.rsyncdir
         rsyncroots = config.getini("rsyncdirs")
         if rsyncroots:
             candidates.extend(rsyncroots)
         roots = []
         for root in candidates:
-            root = py.path.local(root).realpath()
-            if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root))
-            if root not in roots:
-                roots.append(root)
+            root_path = Path(root).resolve()
+            if not root_path.exists():
+                raise pytest.UsageError(f"rsyncdir doesn't exist: {root!r}")
+            if root_path not in roots:
+                roots.append(root_path)
         return roots
 
-    def _getrsyncoptions(self):
+    def _getrsyncoptions(self) -> dict[str, Any]:
         """Get options to be passed for rsync."""
         ignores = list(self.DEFAULT_IGNORES)
         ignores += [str(path) for path in self.config.option.rsyncignore]
@@ -126,12 +176,21 @@ class NodeManager:
             "verbose": getattr(self.config.option, "verbose", 0),
         }
 
-    def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
+    def rsync(
+        self,
+        gateway: execnet.Gateway,
+        source: str | os.PathLike[str],
+        notify: (
+            Callable[[str, execnet.XSpec, str | os.PathLike[str]], Any] | None
+        ) = None,
+        verbose: int = False,
+        ignores: Sequence[str] | None = None,
+    ) -> None:
         """Perform rsync to remote hosts for node."""
         # XXX This changes the calling behaviour of
         #     pytest_xdist_rsyncstart and pytest_xdist_rsyncfinish to
         #     be called once per rsync target.
-        rsync = HostRSync(source, verbose=verbose, ignores=ignores)
+        rsync = HostRSync(source, verbose=verbose > 0, ignores=ignores)
         spec = gateway.spec
         if spec.popen and not spec.chdir:
             # XXX This assumes that sources are python-packages
@@ -146,7 +205,7 @@ class NodeManager:
         if (spec, source) in self._rsynced_specs:
             return
 
-        def finished():
+        def finished() -> None:
             if notify:
                 notify("rsyncrootready", spec, source)
 
@@ -158,65 +217,98 @@ class NodeManager:
 
 
 class HostRSync(execnet.RSync):
-    """RSyncer that filters out common files"""
+    """RSyncer that filters out common files."""
 
-    def __init__(self, sourcedir, *args, **kwargs):
-        self._synced = {}
-        ignores = kwargs.pop("ignores", None) or []
-        self._ignores = [
-            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
-        ]
-        super().__init__(sourcedir=sourcedir, **kwargs)
+    PathLike = Union[str, "os.PathLike[str]"]
 
-    def filter(self, path):
-        path = py.path.local(path)
+    def __init__(
+        self,
+        sourcedir: PathLike,
+        *,
+        ignores: Sequence[PathLike] | None = None,
+        verbose: bool = True,
+    ) -> None:
+        if ignores is None:
+            ignores = []
+        self._ignores = [re.compile(fnmatch.translate(os.fspath(x))) for x in ignores]
+        super().__init__(sourcedir=Path(sourcedir), verbose=verbose)
+
+    def filter(self, path: PathLike) -> bool:
+        path = Path(path)
         for cre in self._ignores:
-            if cre.match(path.basename) or cre.match(path.strpath):
+            if cre.match(path.name) or cre.match(str(path)):
                 return False
         else:
             return True
 
-    def add_target_host(self, gateway, finished=None):
+    def add_target_host(
+        self,
+        gateway: execnet.Gateway,
+        finished: Callable[[], None] | None = None,
+    ) -> None:
         remotepath = os.path.basename(self._sourcedir)
         super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
 
-    def _report_send_file(self, gateway, modified_rel_path):
+    def _report_send_file(
+        self,
+        gateway: execnet.Gateway,  # type: ignore[override]
+        modified_rel_path: str,
+    ) -> None:
         if self._verbose > 0:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
-            print("{}:{} <= {}".format(gateway.spec, remotepath, path))
+            print(f"{gateway.spec}:{remotepath} <= {path}")
 
 
-def make_reltoroot(roots, args):
+def make_reltoroot(roots: Sequence[Path], args: list[str]) -> list[str]:
     # XXX introduce/use public API for splitting pytest args
     splitcode = "::"
     result = []
     for arg in args:
         parts = arg.split(splitcode)
-        fspath = py.path.local(parts[0])
-        if not fspath.exists():
+        fspath = Path(parts[0])
+        try:
+            exists = fspath.exists()
+        except OSError:
+            exists = False
+        if not exists:
             result.append(arg)
             continue
         for root in roots:
-            x = fspath.relto(root)
+            x: Path | None
+            try:
+                x = fspath.relative_to(root)
+            except ValueError:
+                x = None
             if x or fspath == root:
-                parts[0] = root.basename + "/" + x
+                parts[0] = root.name + "/" + str(x)
                 break
         else:
-            raise ValueError("arg {} not relative to an rsync root".format(arg))
+            raise ValueError(f"arg {arg} not relative to an rsync root")
         result.append(splitcode.join(parts))
     return result
 
 
+class Marker(enum.Enum):
+    END = -1
+
+
 class WorkerController:
-    ENDMARK = -1
+    # Set when the worker is ready.
+    workerinfo: WorkerInfo
 
     class RemoteHook:
         @pytest.hookimpl(trylast=True)
-        def pytest_xdist_getremotemodule(self):
+        def pytest_xdist_getremotemodule(self) -> Any:
             return xdist.remote
 
-    def __init__(self, nodemanager, gateway, config, putevent):
+    def __init__(
+        self,
+        nodemanager: NodeManager,
+        gateway: execnet.Gateway,
+        config: pytest.Config,
+        putevent: Callable[[tuple[str, dict[str, Any]]], None],
+    ) -> None:
         config.pluginmanager.register(self.RemoteHook())
         self.nodemanager = nodemanager
         self.putevent = putevent
@@ -230,26 +322,25 @@ class WorkerController:
         }
         self._down = False
         self._shutdown_sent = False
-        self.log = py.log.Producer("workerctl-%s" % gateway.id)
-        if not self.config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
+        self.log = Producer(f"workerctl-{gateway.id}", enabled=config.option.debug)
 
-    def __repr__(self):
-        return "<{} {}>".format(self.__class__.__name__, self.gateway.id)
+    def __repr__(self) -> str:
+        return f"<{self.__class__.__name__} {self.gateway.id}>"
 
     @property
-    def shutting_down(self):
+    def shutting_down(self) -> bool:
         return self._down or self._shutdown_sent
 
-    def setup(self):
+    def setup(self) -> None:
         self.log("setting up worker session")
+        # Cache rinfo for backward compatibility, since pytest-cov
+        # accesses rinfo while the main thread is busy executing our
+        # remote_exec call, which triggers a deadlock error for the
+        # main_thread_only execmodel if the rinfo has not been cached.
+        self.gateway._rinfo()
         spec = self.gateway.spec
-        if hasattr(self.config, "invocation_params"):
         args = [str(x) for x in self.config.invocation_params.args or ()]
         option_dict = {}
-        else:
-            args = self.config.args
-            option_dict = vars(self.config.option)
         if not spec.popen or spec.chdir:
             args = make_reltoroot(self.nodemanager.roots, args)
         if spec.popen:
@@ -266,10 +357,11 @@ class WorkerController:
         change_sys_path = _sys_path if self.gateway.spec.popen else None
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
 
-        if self.putevent:
-            self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
+        # putevent is only None in a test.
+        if self.putevent:  # type: ignore[truthy-function]
+            self.channel.setcallback(self.process_from_remote, endmarker=Marker.END)
 
-    def ensure_teardown(self):
+    def ensure_teardown(self) -> None:
         if hasattr(self, "channel"):
             if not self.channel.isclosed():
                 self.log("closing", self.channel)
@@ -280,13 +372,16 @@ class WorkerController:
             self.gateway.exit()
             # del self.gateway
 
-    def send_runtest_some(self, indices):
+    def send_runtest_some(self, indices: Sequence[int]) -> None:
         self.sendcommand("runtests", indices=indices)
 
-    def send_runtest_all(self):
+    def send_runtest_all(self) -> None:
         self.sendcommand("runtests_all")
 
-    def shutdown(self):
+    def send_steal(self, indices: Sequence[int]) -> None:
+        self.sendcommand("steal", indices=indices)
+
+    def shutdown(self) -> None:
         if not self._down:
             try:
                 self.sendcommand("shutdown")
@@ -294,17 +389,19 @@ class WorkerController:
                 pass
             self._shutdown_sent = True
 
-    def sendcommand(self, name, **kwargs):
-        """send a named parametrized command to the other side."""
-        self.log("sending command {}(**{})".format(name, kwargs))
+    def sendcommand(self, name: str, **kwargs: object) -> None:
+        """Send a named parametrized command to the other side."""
+        self.log(f"sending command {name}(**{kwargs})")
         self.channel.send((name, kwargs))
 
-    def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing {}(**{})".format(eventname, kwargs))
+    def notify_inproc(self, eventname: str, **kwargs: object) -> None:
+        self.log(f"queuing {eventname}(**{kwargs})")
         self.putevent((eventname, kwargs))
 
-    def process_from_remote(self, eventcall):  # noqa too complex
-        """this gets called for each object we receive from
+    def process_from_remote(
+        self, eventcall: tuple[str, dict[str, Any]] | Literal[Marker.END]
+    ) -> None:
+        """This gets called for each object we receive from
         the other side and if the channel closes.
 
         Note that channel callbacks run in the receiver
@@ -312,8 +409,8 @@ class WorkerController:
         avoid raising exceptions or doing heavy work.
         """
         try:
-            if eventcall == self.ENDMARK:
-                err = self.channel._getremoteerror()
+            if eventcall is Marker.END:
+                err: object | None = self.channel._getremoteerror()  # type: ignore[no-untyped-call]
                 if not self._down:
                     if not err or isinstance(err, EOFError):
                         err = "Not properly terminated"  # lost connection?
@@ -322,7 +419,7 @@ class WorkerController:
                 return
             eventname, kwargs = eventcall
             if eventname in ("collectionstart",):
-                self.log("ignoring {}({})".format(eventname, kwargs))
+                self.log(f"ignoring {eventname}({kwargs})")
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "internal_error":
@@ -345,6 +442,8 @@ class WorkerController:
                 self.notify_inproc(eventname, node=self, ids=kwargs["ids"])
             elif eventname == "runtest_protocol_complete":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "unscheduled":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "logwarning":
                 self.notify_inproc(
                     eventname,
@@ -353,16 +452,6 @@ class WorkerController:
                     nodeid=kwargs["nodeid"],
                     fslocation=kwargs["nodeid"],
                 )
-            elif eventname == "warning_captured":
-                warning_message = unserialize_warning_message(
-                    kwargs["warning_message_data"]
-                )
-                self.notify_inproc(
-                    eventname,
-                    warning_message=warning_message,
-                    when=kwargs["when"],
-                    item=kwargs["item"],
-                )
             elif eventname == "warning_recorded":
                 warning_message = unserialize_warning_message(
                     kwargs["warning_message_data"]
@@ -375,22 +464,19 @@ class WorkerController:
                     location=kwargs["location"],
                 )
             else:
-                raise ValueError("unknown event: {}".format(eventname))
+                raise ValueError(f"unknown event: {eventname}")
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise
-        except:  # noqa
-            from _pytest._code import ExceptionInfo
-
-            excinfo = ExceptionInfo.from_current()
+        except BaseException:
+            excinfo = pytest.ExceptionInfo.from_current()
             print("!" * 20, excinfo)
             self.config.notify_exception(excinfo)
             self.shutdown()
             self.notify_inproc("errordown", node=self, error=excinfo)
 
 
-def unserialize_warning_message(data):
-    import warnings
+def unserialize_warning_message(data: dict[str, Any]) -> warnings.WarningMessage:
     import importlib
 
     if data["message_module"]:
@@ -428,4 +514,4 @@ def unserialize_warning_message(data):
             continue
         kwargs[attr_name] = data[attr_name]
 
-    return warnings.WarningMessage(**kwargs)  # type: ignore[arg-type]
+    return warnings.WarningMessage(**kwargs)

testing/acceptance_test.py

@@ -1,11 +1,12 @@
+from __future__ import annotations
+
 import os
 import re
 import shutil
-from typing import Dict
-from typing import List
-from typing import Tuple
+from typing import cast
 
 import pytest
+
 import xdist
 
 
@@ -101,21 +102,51 @@ class TestDistribution:
             """
         )
         result = pytester.runpytest(p1, "-v", "-d", "--tx=popen", "--tx=popen")
-        result.stdout.fnmatch_lines(["*1*Python*", "*2 failed, 1 passed, 1 skipped*"])
+        result.stdout.fnmatch_lines(
+            [
+                "created: 2/2 workers",
+                "*2 failed, 1 passed, 1 skipped*",
+            ]
+        )
         assert result.ret == 1
 
-    def test_n1_fail_minus_x(self, pytester: pytest.Pytester) -> None:
+    def test_exitfirst_waits_for_workers_to_finish(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        """The DSession waits for workers before exiting early on failure.
+
+        When -x/--exitfirst is set, the DSession wait for all workers to finish
+        before raising an Interrupt exception. This prevents reports from the
+        faiing test and other tests from being discarded.
+        """
         p1 = pytester.makepyfile(
             """
+            import time
+
             def test_fail1():
+                time.sleep(0.1)
                 assert 0
             def test_fail2():
+                time.sleep(0.2)
+            def test_fail3():
+                time.sleep(0.3)
                 assert 0
+            def test_fail4():
+                time.sleep(0.3)
+            def test_fail5():
+                time.sleep(0.3)
+            def test_fail6():
+                time.sleep(0.3)
         """
         )
-        result = pytester.runpytest(p1, "-x", "-v", "-n1")
+        # Two workers are used
+        result = pytester.runpytest(p1, "-x", "-rA", "-v", "-n2")
         assert result.ret == 2
-        result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"])
+        # DSession should stop when the first failure is reached. Two failures
+        # may actually occur, due to timing.
+        outcomes = result.parseoutcomes()
+        assert "failed" in outcomes, "Expected at least one failure"
+        assert 1 <= outcomes["failed"] <= 2, "Expected no more than 2 failures"
 
     def test_basetemp_in_subprocesses(self, pytester: pytest.Pytester) -> None:
         p1 = pytester.makepyfile(
@@ -151,10 +182,14 @@ class TestDistribution:
         """
         )
         result = pytester.runpytest(p1, "-d", "-v")
-        result.stdout.fnmatch_lines(["*2*Python*", "*2 failed, 1 passed, 1 skipped*"])
+        result.stdout.fnmatch_lines(
+            [
+                "created: 3/3 workers",
+                "*2 failed, 1 passed, 1 skipped*",
+            ]
+        )
         assert result.ret == 1
 
-    @pytest.mark.xfail("sys.platform.startswith('java')", run=False)
     def test_dist_tests_with_crash(self, pytester: pytest.Pytester) -> None:
         if not hasattr(os, "kill"):
             pytest.skip("no os.kill")
@@ -189,7 +224,7 @@ class TestDistribution:
         assert result.ret == 1
 
     def test_distribution_rsyncdirs_example(
-        self, pytester: pytest.Pytester, monkeypatch
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
     ) -> None:
         # use a custom plugin that has a custom command-line option to ensure
         # this is propagated to workers (see #491)
@@ -231,16 +266,13 @@ class TestDistribution:
             "-pfoobarplugin",
             "--foobar=123",
             "--dist=load",
-            "--rsyncdir=%(subdir)s" % locals(),
-            "--tx=popen//chdir=%(dest)s" % locals(),
+            f"--rsyncdir={subdir}",
+            f"--tx=popen//chdir={dest}",
             p,
         )
         assert result.ret == 0
         result.stdout.fnmatch_lines(
             [
-                "*0* *cwd*",
-                # "RSyncStart: [G1]",
-                # "RSyncFinished: [G1]",
                 "*1 passed*",
             ]
         )
@@ -277,7 +309,11 @@ class TestDistribution:
         p1 = pytester.makepyfile("def test_func(): pass")
         result = pytester.runpytest("-v", p1, "-d", "--tx=popen")
         result.stdout.fnmatch_lines(
-            ["*0*Python*", "*calculated result is 49*", "*1 passed*"]
+            [
+                "created: 1/1 worker",
+                "*calculated result is 49*",
+                "*1 passed*",
+            ]
         )
         assert result.ret == 0
 
@@ -380,7 +416,7 @@ class TestDistEach:
 
 class TestTerminalReporting:
     @pytest.mark.parametrize("verbosity", ["", "-q", "-v"])
-    def test_output_verbosity(self, pytester, verbosity: str) -> None:
+    def test_output_verbosity(self, pytester: pytest.Pytester, verbosity: str) -> None:
         pytester.makepyfile(
             """
             def test_ok():
@@ -394,14 +430,14 @@ class TestTerminalReporting:
         out = result.stdout.str()
         if verbosity == "-v":
             assert "scheduling tests" in out
-            assert "gw" in out
+            assert "1 worker [1 item]" in out
         elif verbosity == "-q":
             assert "scheduling tests" not in out
             assert "gw" not in out
             assert "bringing up nodes..." in out
         else:
             assert "scheduling tests" not in out
-            assert "gw" in out
+            assert "1 worker [1 item]" in out
 
     def test_pass_skip_fail(self, pytester: pytest.Pytester) -> None:
         pytester.makepyfile(
@@ -446,7 +482,7 @@ class TestTerminalReporting:
         )
 
     def test_logfinish_hook(self, pytester: pytest.Pytester) -> None:
-        """Ensure the pytest_runtest_logfinish hook is being properly handled"""
+        """Ensure the pytest_runtest_logfinish hook is being properly handled."""
         pytester.makeconftest(
             """
             def pytest_runtest_logfinish():
@@ -575,9 +611,9 @@ def test_fixture_teardown_failure(pytester: pytest.Pytester) -> None:
 
 
 def test_config_initialization(
-    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, pytestconfig
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    """Ensure workers and controller are initialized consistently. Integration test for #445"""
+    """Ensure workers and controller are initialized consistently. Integration test for #445."""
     pytester.makepyfile(
         **{
             "dir_a/test_foo.py": """
@@ -600,14 +636,13 @@ def test_config_initialization(
 
 
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
-def test_crashing_item(pytester, when) -> None:
-    """Ensure crashing item is correctly reported during all testing stages"""
+def test_crashing_item(pytester: pytest.Pytester, when: str) -> None:
+    """Ensure crashing item is correctly reported during all testing stages."""
     code = dict(setup="", call="", teardown="")
-    code[when] = "py.process.kill(os.getpid())"
+    code[when] = "os._exit(1)"
     p = pytester.makepyfile(
         """
         import os
-        import py
         import pytest
 
         @pytest.fixture
@@ -622,9 +657,7 @@ def test_crashing_item(pytester, when) -> None:
 
         def test_ok():
             pass
-    """.format(
-            **code
-        )
+    """.format(**code)
     )
     passes = 2 if when == "teardown" else 1
     result = pytester.runpytest("-n2", p)
@@ -734,22 +767,22 @@ def test_tmpdir_disabled(pytester: pytest.Pytester) -> None:
 
 
 @pytest.mark.parametrize("plugin", ["xdist.looponfail"])
-def test_sub_plugins_disabled(pytester, plugin) -> None:
-    """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)"""
+def test_sub_plugins_disabled(pytester: pytest.Pytester, plugin: str) -> None:
+    """Test that xdist doesn't break if we disable any of its sub-plugins (#32)."""
     p1 = pytester.makepyfile(
         """
         def test_ok():
             pass
     """
     )
-    result = pytester.runpytest(p1, "-n1", "-p", "no:%s" % plugin)
+    result = pytester.runpytest(p1, "-n1", "-p", f"no:{plugin}")
     assert result.ret == 0
     result.stdout.fnmatch_lines("*1 passed*")
 
 
 class TestWarnings:
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    def test_warnings(self, pytester, n) -> None:
+    def test_warnings(self, pytester: pytest.Pytester, n: str) -> None:
         pytester.makepyfile(
             """
             import warnings, py, pytest
@@ -765,9 +798,7 @@ class TestWarnings:
     def test_warning_captured_deprecated_in_pytest_6(
         self, pytester: pytest.Pytester
     ) -> None:
-        """
-        Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
-        """
+        """Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)."""
         from _pytest import hookspec
 
         if not hasattr(hookspec, "pytest_warning_captured"):
@@ -797,9 +828,9 @@ class TestWarnings:
         result.stdout.no_fnmatch_line("*this hook should not be called in this version")
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    def test_custom_subclass(self, pytester, n) -> None:
+    def test_custom_subclass(self, pytester: pytest.Pytester, n: str) -> None:
         """Check that warning subclasses that don't honor the args attribute don't break
-        pytest-xdist (#344)
+        pytest-xdist (#344).
         """
         pytester.makepyfile(
             """
@@ -821,7 +852,7 @@ class TestWarnings:
         result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warning*"])
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    def test_unserializable_arguments(self, pytester, n) -> None:
+    def test_unserializable_arguments(self, pytester: pytest.Pytester, n: str) -> None:
         """Check that warnings with unserializable arguments are handled correctly (#349)."""
         pytester.makepyfile(
             """
@@ -839,7 +870,9 @@ class TestWarnings:
         result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warning*"])
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    def test_unserializable_warning_details(self, pytester, n) -> None:
+    def test_unserializable_warning_details(
+        self, pytester: pytest.Pytester, n: str
+    ) -> None:
         """Check that warnings with unserializable _WARNING_DETAILS are
         handled correctly (#379).
         """
@@ -1019,7 +1052,7 @@ class TestNodeFailure:
 
 
 @pytest.mark.parametrize("n", [0, 2])
-def test_worker_id_fixture(pytester, n) -> None:
+def test_worker_id_fixture(pytester: pytest.Pytester, n: int) -> None:
     import glob
 
     f = pytester.makepyfile(
@@ -1035,8 +1068,8 @@ def test_worker_id_fixture(pytester, n) -> None:
     result.stdout.fnmatch_lines("* 2 passed in *")
     worker_ids = set()
     for fname in glob.glob(str(pytester.path / "*.txt")):
-        with open(fname) as f:
-            worker_ids.add(f.read().strip())
+        with open(fname) as fp:
+            worker_ids.add(fp.read().strip())
     if n == 0:
         assert worker_ids == {"master"}
     else:
@@ -1044,7 +1077,7 @@ def test_worker_id_fixture(pytester, n) -> None:
 
 
 @pytest.mark.parametrize("n", [0, 2])
-def test_testrun_uid_fixture(pytester, n) -> None:
+def test_testrun_uid_fixture(pytester: pytest.Pytester, n: int) -> None:
     import glob
 
     f = pytester.makepyfile(
@@ -1060,14 +1093,14 @@ def test_testrun_uid_fixture(pytester, n) -> None:
     result.stdout.fnmatch_lines("* 2 passed in *")
     testrun_uids = set()
     for fname in glob.glob(str(pytester.path / "*.txt")):
-        with open(fname) as f:
-            testrun_uids.add(f.read().strip())
+        with open(fname) as fp:
+            testrun_uids.add(fp.read().strip())
     assert len(testrun_uids) == 1
     assert len(testrun_uids.pop()) == 32
 
 
 @pytest.mark.parametrize("tb", ["auto", "long", "short", "no", "line", "native"])
-def test_error_report_styles(pytester, tb) -> None:
+def test_error_report_styles(pytester: pytest.Pytester, tb: str) -> None:
     pytester.makepyfile(
         """
         import pytest
@@ -1081,14 +1114,11 @@ def test_error_report_styles(pytester, tb) -> None:
     result.assert_outcomes(failed=1)
 
 
-def test_color_yes_collection_on_non_atty(pytester, request) -> None:
-    """skip collect progress report when working on non-terminals.
+def test_color_yes_collection_on_non_atty(pytester: pytest.Pytester) -> None:
+    """Skip collect progress report when working on non-terminals.
 
     Similar to pytest-dev/pytest#1397
     """
-    tr = request.config.pluginmanager.getplugin("terminalreporter")
-    if not hasattr(tr, "isatty"):
-        pytest.skip("only valid for newer pytest versions")
     pytester.makepyfile(
         """
         import pytest
@@ -1101,14 +1131,13 @@ def test_color_yes_collection_on_non_atty(pytester, request) -> None:
     result = pytester.runpytest(*args)
     assert "test session starts" in result.stdout.str()
     assert "\x1b[1m" in result.stdout.str()
-    assert "gw0 [10] / gw1 [10]" in result.stdout.str()
-    assert "gw0 C / gw1 C" not in result.stdout.str()
+    assert "created: 2/2 workers" in result.stdout.str()
+    assert "2 workers [10 items]" in result.stdout.str()
+    assert "collecting:" not in result.stdout.str()
 
 
-def test_without_terminal_plugin(pytester, request) -> None:
-    """
-    No output when terminal plugin is disabled
-    """
+def test_without_terminal_plugin(pytester: pytest.Pytester) -> None:
+    """No output when terminal plugin is disabled."""
     pytester.makepyfile(
         """
         def test_1():
@@ -1122,9 +1151,7 @@ def test_without_terminal_plugin(pytester, request) -> None:
 
 
 def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None:
-    """
-    Internal error when using --maxfail option (#62, #65).
-    """
+    """Internal error when using --maxfail option (#62, #65)."""
     pytester.makepyfile(
         """
         import pytest
@@ -1140,10 +1167,25 @@ def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None:
     """
     )
     result = pytester.runpytest_subprocess("--maxfail=1", "-n1")
-    result.stdout.fnmatch_lines(["* 1 error in *"])
+    result.stdout.re_match_lines([".* [12] errors? in .*"])
     assert "INTERNALERROR" not in result.stderr.str()
 
 
+def test_maxfail_causes_early_termination(pytester: pytest.Pytester) -> None:
+    """Ensure subsequent tests on a worker aren't run when using --maxfail (#1024)."""
+    pytester.makepyfile(
+        """
+        def test1():
+            assert False
+
+        def test2():
+            pass
+    """
+    )
+    result = pytester.runpytest_subprocess("--maxfail=1", "-n 1")
+    result.assert_outcomes(failed=1)
+
+
 def test_internal_errors_propagate_to_controller(pytester: pytest.Pytester) -> None:
     pytester.makeconftest(
         """
@@ -1196,6 +1238,40 @@ class TestLoadScope:
             "test_a.py::TestB", result.outlines
         ) in ({"gw0": 10}, {"gw1": 10})
 
+    def test_workqueue_ordered_by_size(self, pytester: pytest.Pytester) -> None:
+        test_file = """
+            import pytest
+            @pytest.mark.parametrize('i', range({}))
+            def test(i):
+                pass
+        """
+        pytester.makepyfile(test_a=test_file.format(10), test_b=test_file.format(20))
+        result = pytester.runpytest("-n2", "--dist=loadscope", "-v")
+        assert get_workers_and_test_count_by_prefix(
+            "test_a.py::test", result.outlines
+        ) == {"gw1": 10}
+        assert get_workers_and_test_count_by_prefix(
+            "test_b.py::test", result.outlines
+        ) == {"gw0": 20}
+
+    def test_workqueue_ordered_by_input(self, pytester: pytest.Pytester) -> None:
+        test_file = """
+            import pytest
+            @pytest.mark.parametrize('i', range({}))
+            def test(i):
+                pass
+        """
+        pytester.makepyfile(test_a=test_file.format(10), test_b=test_file.format(20))
+        result = pytester.runpytest(
+            "-n2", "--dist=loadscope", "--no-loadscope-reorder", "-v"
+        )
+        assert get_workers_and_test_count_by_prefix(
+            "test_a.py::test", result.outlines
+        ) == {"gw0": 10}
+        assert get_workers_and_test_count_by_prefix(
+            "test_b.py::test", result.outlines
+        ) == {"gw1": 20}
+
     def test_module_single_start(self, pytester: pytest.Pytester) -> None:
         """Fix test suite never finishing in case all workers start with a single test (#277)."""
         test_file1 = """
@@ -1245,22 +1321,14 @@ class TestFileScope:
             "test_b.py::TestB", result.outlines
         )
 
-        assert (
-            test_a_workers_and_test_count
-            in (
+        assert test_a_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
 
     def test_by_class(self, pytester: pytest.Pytester) -> None:
         pytester.makepyfile(
@@ -1285,22 +1353,14 @@ class TestFileScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert (
-            test_a_workers_and_test_count
-            in (
+        assert test_a_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
 
     def test_module_single_start(self, pytester: pytest.Pytester) -> None:
         """Fix test suite never finishing in case all workers start with a single test (#277)."""
@@ -1329,7 +1389,7 @@ class TestFileScope:
 
 
 class TestGroupScope:
-    def test_by_module(self, testdir):
+    def test_by_module(self, pytester: pytest.Pytester) -> None:
         test_file = """
             import pytest
             class TestA:
@@ -1338,8 +1398,8 @@ class TestGroupScope:
                 def test(self, i):
                     pass
         """
-        testdir.makepyfile(test_a=test_file, test_b=test_file)
-        result = testdir.runpytest("-n2", "--dist=loadgroup", "-v")
+        pytester.makepyfile(test_a=test_file, test_b=test_file)
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
         test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
             "test_a.py::TestA", result.outlines
         )
@@ -1347,29 +1407,21 @@ class TestGroupScope:
             "test_b.py::TestA", result.outlines
         )
 
-        assert (
-            test_a_workers_and_test_count
-            in (
+        assert test_a_workers_and_test_count in (
             {"gw0": 5},
             {"gw1": 0},
-            )
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
+        assert test_b_workers_and_test_count in (
             {"gw0": 5},
             {"gw1": 0},
-            )
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
-        )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
         assert (
             test_a_workers_and_test_count.items()
             == test_b_workers_and_test_count.items()
         )
 
-    def test_by_class(self, testdir):
-        testdir.makepyfile(
+    def test_by_class(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
             test_a="""
             import pytest
             class TestA:
@@ -1384,7 +1436,7 @@ class TestGroupScope:
                     pass
         """
         )
-        result = testdir.runpytest("-n2", "--dist=loadgroup", "-v")
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
         test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
             "test_a.py::TestA", result.outlines
         )
@@ -1392,28 +1444,20 @@ class TestGroupScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert (
-            test_a_workers_and_test_count
-            in (
+        assert test_a_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
             {"gw0": 10},
             {"gw1": 0},
-            )
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
         assert (
             test_a_workers_and_test_count.items()
             == test_b_workers_and_test_count.items()
         )
 
-    def test_module_single_start(self, testdir):
+    def test_module_single_start(self, pytester: pytest.Pytester) -> None:
         test_file1 = """
             import pytest
             @pytest.mark.xdist_group(name="xdist_group")
@@ -1428,15 +1472,15 @@ class TestGroupScope:
             def test_2():
                 pass
         """
-        testdir.makepyfile(test_a=test_file1, test_b=test_file1, test_c=test_file2)
-        result = testdir.runpytest("-n2", "--dist=loadgroup", "-v")
+        pytester.makepyfile(test_a=test_file1, test_b=test_file1, test_c=test_file2)
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
         a = get_workers_and_test_count_by_prefix("test_a.py::test", result.outlines)
         b = get_workers_and_test_count_by_prefix("test_b.py::test", result.outlines)
         c = get_workers_and_test_count_by_prefix("test_c.py::test_2", result.outlines)
 
         assert a.keys() == b.keys() and b.keys() == c.keys()
 
-    def test_with_two_group_names(self, testdir):
+    def test_with_two_group_names(self, pytester: pytest.Pytester) -> None:
         test_file = """
             import pytest
             @pytest.mark.xdist_group(name="group1")
@@ -1446,8 +1490,8 @@ class TestGroupScope:
             def test_2():
                 pass
         """
-        testdir.makepyfile(test_a=test_file, test_b=test_file)
-        result = testdir.runpytest("-n2", "--dist=loadgroup", "-v")
+        pytester.makepyfile(test_a=test_file, test_b=test_file)
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
         a_1 = get_workers_and_test_count_by_prefix("test_a.py::test_1", result.outlines)
         a_2 = get_workers_and_test_count_by_prefix("test_a.py::test_2", result.outlines)
         b_1 = get_workers_and_test_count_by_prefix("test_b.py::test_1", result.outlines)
@@ -1455,6 +1499,46 @@ class TestGroupScope:
 
         assert a_1.keys() == b_1.keys() and a_2.keys() == b_2.keys()
 
+    def test_multiple_group_marks(self, pytester: pytest.Pytester) -> None:
+        test_file = """
+            import pytest
+            @pytest.mark.xdist_group(name="group1")
+            @pytest.mark.xdist_group(name="group2")
+            def test_1():
+                pass
+        """
+        pytester.makepyfile(test_a=test_file, test_b=test_file)
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
+        res = parse_tests_and_workers_from_output(result.outlines)
+        assert len(res) == 2
+        # get test names
+        a_1 = next(t[2] for t in res if "test_a.py::test_1" in t[2])
+        b_1 = next(t[2] for t in res if "test_b.py::test_1" in t[2])
+        # check groups
+        assert a_1.split("@")[1] == b_1.split("@")[1] == "group1_group2"
+
+    def test_multiple_group_order(self, pytester: pytest.Pytester) -> None:
+        test_file = """
+            import pytest
+            @pytest.mark.xdist_group(name="b")
+            @pytest.mark.xdist_group(name="d")
+            @pytest.mark.xdist_group(name="c")
+            @pytest.mark.xdist_group(name="c2")
+            @pytest.mark.xdist_group(name="a")
+            @pytest.mark.xdist_group(name="aa")
+            def test_1():
+                pass
+        """
+        pytester.makepyfile(test_a=test_file, test_b=test_file)
+        result = pytester.runpytest("-n2", "--dist=loadgroup", "-v")
+        res = parse_tests_and_workers_from_output(result.outlines)
+        assert len(res) == 2
+        # get test names
+        a_1 = next(t[2] for t in res if "test_a.py::test_1" in t[2])
+        b_1 = next(t[2] for t in res if "test_b.py::test_1" in t[2])
+        # check groups, order should be sorted
+        assert a_1.split("@")[1] == b_1.split("@")[1] == "a_aa_b_c_c2_d"
+
 
 class TestLocking:
     _test_content = """
@@ -1484,18 +1568,20 @@ class TestLocking:
 
     FILE_LOCK = filelock.FileLock("test.lock")
 
-    """ + (
-        (_test_content * 4) % ("A", "B", "C", "D")
-    )
+    """ + ((_test_content * 4) % ("A", "B", "C", "D"))
 
-    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
-    def test_single_file(self, pytester, scope) -> None:
+    @pytest.mark.parametrize(
+        "scope", ["each", "load", "loadscope", "loadfile", "worksteal", "no"]
+    )
+    def test_single_file(self, pytester: pytest.Pytester, scope: str) -> None:
         pytester.makepyfile(test_a=self.test_file1)
         result = pytester.runpytest("-n2", "--dist=%s" % scope, "-v")
         result.assert_outcomes(passed=(12 if scope != "each" else 12 * 2))
 
-    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
-    def test_multi_file(self, pytester, scope) -> None:
+    @pytest.mark.parametrize(
+        "scope", ["each", "load", "loadscope", "loadfile", "worksteal", "no"]
+    )
+    def test_multi_file(self, pytester: pytest.Pytester, scope: str) -> None:
         pytester.makepyfile(
             test_a=self.test_file1,
             test_b=self.test_file1,
@@ -1506,7 +1592,7 @@ class TestLocking:
         result.assert_outcomes(passed=(48 if scope != "each" else 48 * 2))
 
 
-def parse_tests_and_workers_from_output(lines: List[str]) -> List[Tuple[str, str, str]]:
+def parse_tests_and_workers_from_output(lines: list[str]) -> list[tuple[str, str, str]]:
     result = []
     for line in lines:
         # example match: "[gw0] PASSED test_a.py::test[7]"
@@ -1528,9 +1614,9 @@ def parse_tests_and_workers_from_output(lines: List[str]) -> List[Tuple[str, str
 
 
 def get_workers_and_test_count_by_prefix(
-    prefix: str, lines: List[str], expected_status: str = "PASSED"
-) -> Dict[str, int]:
-    result: Dict[str, int] = {}
+    prefix: str, lines: list[str], expected_status: str = "PASSED"
+) -> dict[str, int]:
+    result: dict[str, int] = {}
     for worker, status, nodeid in parse_tests_and_workers_from_output(lines):
         if expected_status == status and nodeid.startswith(prefix):
             result[worker] = result.get(worker, 0) + 1
@@ -1539,32 +1625,32 @@ def get_workers_and_test_count_by_prefix(
 
 class TestAPI:
     @pytest.fixture
-    def fake_request(self):
+    def fake_request(self) -> pytest.FixtureRequest:
         class FakeOption:
-            def __init__(self):
+            def __init__(self) -> None:
                 self.dist = "load"
 
         class FakeConfig:
-            def __init__(self):
+            def __init__(self) -> None:
                 self.workerinput = {"workerid": "gw5"}
                 self.option = FakeOption()
 
         class FakeRequest:
-            def __init__(self):
+            def __init__(self) -> None:
                 self.config = FakeConfig()
 
-        return FakeRequest()
+        return cast(pytest.FixtureRequest, FakeRequest())
 
-    def test_is_xdist_worker(self, fake_request) -> None:
+    def test_is_xdist_worker(self, fake_request: pytest.FixtureRequest) -> None:
         assert xdist.is_xdist_worker(fake_request)
-        del fake_request.config.workerinput
+        del fake_request.config.workerinput  # type: ignore[attr-defined]
         assert not xdist.is_xdist_worker(fake_request)
 
-    def test_is_xdist_controller(self, fake_request) -> None:
+    def test_is_xdist_controller(self, fake_request: pytest.FixtureRequest) -> None:
         assert not xdist.is_xdist_master(fake_request)
         assert not xdist.is_xdist_controller(fake_request)
 
-        del fake_request.config.workerinput
+        del fake_request.config.workerinput  # type: ignore[attr-defined]
         assert xdist.is_xdist_master(fake_request)
         assert xdist.is_xdist_controller(fake_request)
 
@@ -1572,7 +1658,44 @@ class TestAPI:
         assert not xdist.is_xdist_master(fake_request)
         assert not xdist.is_xdist_controller(fake_request)
 
-    def test_get_xdist_worker_id(self, fake_request) -> None:
+    def test_get_xdist_worker_id(self, fake_request: pytest.FixtureRequest) -> None:
         assert xdist.get_xdist_worker_id(fake_request) == "gw5"
-        del fake_request.config.workerinput
+        del fake_request.config.workerinput  # type: ignore[attr-defined]
         assert xdist.get_xdist_worker_id(fake_request) == "master"
+
+
+def test_collection_crash(pytester: pytest.Pytester) -> None:
+    p1 = pytester.makepyfile(
+        """
+        assert 0
+    """
+    )
+    result = pytester.runpytest(p1, "-n1")
+    assert result.ret == 1
+    result.stdout.fnmatch_lines(
+        [
+            "created: 1/1 worker",
+            "1 worker [[]0 items[]]",
+            "*_ ERROR collecting test_collection_crash.py _*",
+            "E   assert 0",
+            "*= 1 error in *",
+        ]
+    )
+
+
+def test_dist_in_addopts(pytester: pytest.Pytester) -> None:
+    """Users can set a default distribution in the configuration file (#789)."""
+    pytester.makepyfile(
+        """
+        def test():
+            pass
+        """
+    )
+    pytester.makeini(
+        """
+        [pytest]
+        addopts = --dist loadscope
+        """
+    )
+    result = pytester.runpytest()
+    assert result.ret == 0

testing/conftest.py

@@ -1,18 +1,25 @@
+from __future__ import annotations
+
+from collections.abc import Generator
+import shutil
+from typing import Callable
+
 import execnet
 import pytest
-import shutil
-from typing import List
+
 
 pytest_plugins = "pytester"
 
 
 @pytest.fixture(autouse=True)
-def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
+def _divert_atexit(monkeypatch: pytest.MonkeyPatch) -> Generator[None]:
     import atexit
 
     finalizers = []
 
-    def fake_register(func, *args, **kwargs):
+    def fake_register(
+        func: Callable[..., object], *args: object, **kwargs: object
+    ) -> None:
         finalizers.append((func, args, kwargs))
 
     monkeypatch.setattr(atexit, "register", fake_register)
@@ -24,7 +31,7 @@ def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
         func(*args, **kwargs)
 
 
-def pytest_addoption(parser) -> None:
+def pytest_addoption(parser: pytest.Parser) -> None:
     parser.addoption(
         "--gx",
         action="append",
@@ -34,16 +41,16 @@ def pytest_addoption(parser) -> None:
 
 
 @pytest.fixture
-def specssh(request) -> str:
+def specssh(request: pytest.FixtureRequest) -> str:
     return getspecssh(request.config)
 
 
 # configuration information for tests
-def getgspecs(config) -> List[execnet.XSpec]:
+def getgspecs(config: pytest.Config) -> list[execnet.XSpec]:
     return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
 
 
-def getspecssh(config) -> str:  # type: ignore[return]
+def getspecssh(config: pytest.Config) -> str:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.ssh:
@@ -53,7 +60,7 @@ def getspecssh(config) -> str:  # type: ignore[return]
     pytest.skip("need '--gx ssh=...'")
 
 
-def getsocketspec(config) -> execnet.XSpec:
+def getsocketspec(config: pytest.Config) -> execnet.XSpec:
     xspecs = getgspecs(config)
     for spec in xspecs:
         if spec.socket:

testing/test_dsession.py

@@ -1,31 +1,55 @@
-from xdist.dsession import DSession, get_default_max_worker_restart
-from xdist.report import report_collection_diff
-from xdist.scheduler import EachScheduling, LoadScheduling
-from typing import Optional
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+from typing import cast
+from typing import TYPE_CHECKING
 
-import pytest
 import execnet
+import pytest
+
+from xdist.dsession import DSession
+from xdist.dsession import get_default_max_worker_restart
+from xdist.dsession import get_workers_status_line
+from xdist.dsession import WorkerStatus
+from xdist.report import report_collection_diff
+from xdist.scheduler import EachScheduling
+from xdist.scheduler import LoadScheduling
+from xdist.scheduler import WorkStealingScheduling
+from xdist.workermanage import WorkerController
 
 
-class MockGateway:
+if TYPE_CHECKING:
+    BaseOfMockGateway = execnet.Gateway
+    BaseOfMockNode = WorkerController
+else:
+    BaseOfMockGateway = object
+    BaseOfMockNode = object
+
+
+class MockGateway(BaseOfMockGateway):
     def __init__(self) -> None:
         self._count = 0
         self.id = str(self._count)
         self._count += 1
 
 
-class MockNode:
+class MockNode(BaseOfMockNode):
     def __init__(self) -> None:
-        self.sent = []  # type: ignore[var-annotated]
+        self.sent: list[int | str] = []
+        self.stolen: list[int] = []
         self.gateway = MockGateway()
         self._shutdown = False
 
-    def send_runtest_some(self, indices) -> None:
+    def send_runtest_some(self, indices: Sequence[int]) -> None:
         self.sent.extend(indices)
 
     def send_runtest_all(self) -> None:
         self.sent.append("ALL")
 
+    def send_steal(self, indices: Sequence[int]) -> None:
+        self.stolen.extend(indices)
+
     def shutdown(self) -> None:
         self._shutdown = True
 
@@ -36,10 +60,9 @@ class MockNode:
 
 class TestEachScheduling:
     def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
-        node1 = MockNode()
-        node2 = MockNode()
         config = pytester.parseconfig("--tx=2*popen")
         sched = EachScheduling(config)
+        node1, node2 = MockNode(), MockNode()
         sched.add_node(node1)
         sched.add_node(node2)
         collection = ["a.py::test_1"]
@@ -47,7 +70,7 @@ class TestEachScheduling:
         sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
         sched.add_node_collection(node2, collection)
-        assert sched.collection_is_completed
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.schedule()
@@ -60,14 +83,14 @@ class TestEachScheduling:
         assert sched.tests_finished
 
     def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
-        node1 = MockNode()
         config = pytester.parseconfig("--tx=popen")
         sched = EachScheduling(config)
+        node1 = MockNode()
         sched.add_node(node1)
         collection = ["a.py::test_1"]
         assert not sched.collection_is_completed
         sched.add_node_collection(node1, collection)
-        assert sched.collection_is_completed
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
         sched.schedule()
         assert sched.tests_finished
@@ -81,15 +104,15 @@ class TestLoadScheduling:
     def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
         config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        node1, node2 = sched.nodes
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
         collection = ["a.py::test_1", "a.py::test_2"]
         assert not sched.collection_is_completed
         sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
         sched.add_node_collection(node2, collection)
-        assert sched.collection_is_completed
+        assert bool(sched.collection_is_completed)
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.schedule()
@@ -99,15 +122,17 @@ class TestLoadScheduling:
         assert len(node2.sent) == 1
         assert node1.sent == [0]
         assert node2.sent == [1]
-        sched.mark_test_complete(node1, node1.sent[0])
+        sent10 = node1.sent[0]
+        assert isinstance(sent10, int)
+        sched.mark_test_complete(node1, sent10)
         assert sched.tests_finished
 
     def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
         config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        node1, node2 = sched.nodes
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
         col = ["xyz"] * 6
         sched.add_node_collection(node1, col)
         sched.add_node_collection(node2, col)
@@ -115,66 +140,116 @@ class TestLoadScheduling:
         # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
-        assert sent1 == [0, 2]
-        assert sent2 == [1, 3]
+        assert sent1 == [0, 1]
+        assert sent2 == [2, 3]
         assert sched.pending == [4, 5]
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
         assert len(sched.pending) == 2
         sched.mark_test_complete(node1, 0)
-        assert node1.sent == [0, 2, 4]
+        assert node1.sent == [0, 1, 4]
         assert sched.pending == [5]
-        assert node2.sent == [1, 3]
-        sched.mark_test_complete(node1, 2)
-        assert node1.sent == [0, 2, 4, 5]
+        assert node2.sent == [2, 3]
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
         assert not sched.pending
 
-    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+    def test_schedule_maxchunk_none(self, pytester: pytest.Pytester) -> None:
         config = pytester.parseconfig("--tx=2*popen")
         sched = LoadScheduling(config)
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        node1, node2, node3 = sched.nodes
-        col = ["xyz"] * 2
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        col = [f"test{i}" for i in range(16)]
         sched.add_node_collection(node1, col)
         sched.add_node_collection(node2, col)
         sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+        sched.mark_test_complete(node1, 0)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+        sched.mark_test_complete(node1, 1)
+        assert node1.sent == [0, 1, 4, 5]
+        assert sched.pending == list(range(6, 16))
+
+        for i in range(7, 16):
+            sched.mark_test_complete(node1, i - 3)
+            assert node1.sent == [0, 1, *range(4, i)]
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(i, 16))
+
+    def test_schedule_maxchunk_1(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen", "--maxschedchunk=1")
+        sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        col = [f"test{i}" for i in range(16)]
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        assert node1.sent == [0, 1]
+        assert node2.sent == [2, 3]
+        assert sched.pending == list(range(4, 16))
+        assert sched.node2pending[node1] == node1.sent
+        assert sched.node2pending[node2] == node2.sent
+
+        for complete_index, first_pending in enumerate(range(5, 16)):
+            sent_index = node1.sent[complete_index]
+            assert isinstance(sent_index, int)
+            sched.mark_test_complete(node1, sent_index)
+            assert node1.sent == [0, 1, *range(4, first_pending)]
+            assert node2.sent == [2, 3]
+            assert sched.pending == list(range(first_pending, 16))
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = LoadScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        assert sched.collection_is_completed
+        sched.schedule()
         # assert not sched.tests_finished
-        sent1 = node1.sent
-        sent2 = node2.sent
-        sent3 = node3.sent
-        assert sent1 == [0]
-        assert sent2 == [1]
-        assert sent3 == []
+        assert node1.sent == [0]
+        assert node2.sent == [1]
+        assert node3.sent == []
         assert not sched.pending
 
     def test_schedule_fewer_than_two_tests_per_node(
         self, pytester: pytest.Pytester
     ) -> None:
-        config = pytester.parseconfig("--tx=2*popen")
+        config = pytester.parseconfig("--tx=3*popen")
         sched = LoadScheduling(config)
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        sched.add_node(MockNode())
-        node1, node2, node3 = sched.nodes
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
         col = ["xyz"] * 5
         sched.add_node_collection(node1, col)
         sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        assert sched.collection_is_completed
         sched.schedule()
         # assert not sched.tests_finished
-        sent1 = node1.sent
-        sent2 = node2.sent
-        sent3 = node3.sent
-        assert sent1 == [0, 3]
-        assert sent2 == [1, 4]
-        assert sent3 == [2]
+        assert node1.sent == [0, 3]
+        assert node2.sent == [1, 4]
+        assert node3.sent == [2]
         assert not sched.pending
 
     def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
-        node = MockNode()
         config = pytester.parseconfig("--tx=popen")
         sched = LoadScheduling(config)
+        node = MockNode()
         sched.add_node(node)
         collection = ["test_file.py::test_func"]
         sched.add_node_collection(node, collection)
@@ -191,22 +266,19 @@ class TestLoadScheduling:
         """
 
         class CollectHook:
-            """
-            Dummy hook that stores collection reports.
-            """
+            """Dummy hook that stores collection reports."""
 
-            def __init__(self):
-                self.reports = []
+            def __init__(self) -> None:
+                self.reports: list[pytest.CollectReport] = []
 
-            def pytest_collectreport(self, report):
+            def pytest_collectreport(self, report: pytest.CollectReport) -> None:
                 self.reports.append(report)
 
         collect_hook = CollectHook()
         config = pytester.parseconfig("--tx=2*popen")
         config.pluginmanager.register(collect_hook, "collect_hook")
-        node1 = MockNode()
-        node2 = MockNode()
         sched = LoadScheduling(config)
+        node1, node2 = MockNode(), MockNode()
         sched.add_node(node1)
         sched.add_node(node2)
         sched.add_node_collection(node1, ["a.py::test_1"])
@@ -214,14 +286,192 @@ class TestLoadScheduling:
         sched.schedule()
         assert len(collect_hook.reports) == 1
         rep = collect_hook.reports[0]
+        assert isinstance(rep.longrepr, str)
+        assert "Different tests were collected between" in rep.longrepr
+
+
+class TestWorkStealingScheduling:
+    def test_ideal_case(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node1, collection)
+        assert not sched.collection_is_completed
+        sched.add_node_collection(node2, collection)
+        assert bool(sched.collection_is_completed)
+        assert sched.node2collection[node1] == collection
+        assert sched.node2collection[node2] == collection
+        sched.schedule()
+        assert not sched.pending
+        assert not sched.tests_finished
+        assert node1.sent == list(range(8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sent1, sent2 = node1.sent[i], node2.sent[i]
+            assert isinstance(sent1, int) and isinstance(sent2, int)
+            sched.mark_test_complete(node1, sent1)
+            sched.mark_test_complete(node2, sent2)
+        assert bool(sched.tests_finished)
+        assert node1.stolen == []
+        assert node2.stolen == []
+
+    def test_stealing(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=2*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(16)]
+        sched.add_node_collection(node1, collection)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node1.sent == list(range(8))
+        assert node2.sent == list(range(8, 16))
+        for i in range(8):
+            sent = node1.sent[i]
+            assert isinstance(sent, int)
+            sched.mark_test_complete(node1, sent)
+        assert node2.stolen == list(range(12, 16))
+        sched.remove_pending_tests_from_node(node2, node2.stolen)
+        for i in range(4):
+            sent = node2.sent[i]
+            assert isinstance(sent, int)
+            sched.mark_test_complete(node2, sent)
+        assert node1.stolen == [14, 15]
+        sched.remove_pending_tests_from_node(node1, node1.stolen)
+        sched.mark_test_complete(node1, 12)
+        sched.mark_test_complete(node2, 14)
+        assert node2.stolen == list(range(12, 16))
+        assert node1.stolen == [14, 15]
+        assert sched.tests_finished
+
+    def test_steal_on_add_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = [f"test_workstealing.py::test_{i}" for i in range(5)]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        sched.mark_test_complete(node, 0)
+        node2 = MockNode()
+        sched.add_node(node2)
+        sched.add_node_collection(node2, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert node.stolen == [3, 4]
+        sched.remove_pending_tests_from_node(node, node.stolen)
+        sched.mark_test_complete(node, 1)
+        sched.mark_test_complete(node2, 3)
+        assert sched.tests_finished
+        assert node2.stolen == []
+
+    def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 2
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == []
+        assert node1.stolen == []
+        assert node2.sent == [0]
+        assert node2.stolen == []
+        assert node3.sent == [1]
+        assert node3.stolen == []
+        assert not sched.pending
+        assert sched.tests_finished
+
+    def test_schedule_fewer_than_two_tests_per_node(
+        self, pytester: pytest.Pytester
+    ) -> None:
+        config = pytester.parseconfig("--tx=3*popen")
+        sched = WorkStealingScheduling(config)
+        node1, node2, node3 = MockNode(), MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node(node3)
+        col = ["xyz"] * 5
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.add_node_collection(node3, col)
+        sched.schedule()
+        assert node1.sent == [0]
+        assert node2.sent == [1, 2]
+        assert node3.sent == [3, 4]
+        assert not sched.pending
+        assert not sched.tests_finished
+        sent10 = node1.sent[0]
+        assert isinstance(sent10, int)
+        sent20 = node2.sent[0]
+        assert isinstance(sent20, int)
+        sent30 = node3.sent[0]
+        assert isinstance(sent30, int)
+        sent31 = node3.sent[1]
+        assert isinstance(sent31, int)
+        sched.mark_test_complete(node1, sent10)
+        sched.mark_test_complete(node2, sent20)
+        sched.mark_test_complete(node3, sent30)
+        sched.mark_test_complete(node3, sent31)
+        assert bool(sched.tests_finished)
+        assert node1.stolen == []
+        assert node2.stolen == []
+        assert node3.stolen == []
+
+    def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
+        node = MockNode()
+        config = pytester.parseconfig("--tx=popen")
+        sched = WorkStealingScheduling(config)
+        sched.add_node(node)
+        collection = ["test_file.py::test_func"]
+        sched.add_node_collection(node, collection)
+        assert sched.collection_is_completed
+        sched.schedule()
+        assert not sched.pending
+        crashitem = sched.remove_node(node)
+        assert crashitem == collection[0]
+
+    def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
+        class CollectHook:
+            def __init__(self) -> None:
+                self.reports: list[pytest.CollectReport] = []
+
+            def pytest_collectreport(self, report: pytest.CollectReport) -> None:
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = pytester.parseconfig("--tx=2*popen")
+        config.pluginmanager.register(collect_hook, "collect_hook")
+        sched = WorkStealingScheduling(config)
+        node1, node2 = MockNode(), MockNode()
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert isinstance(rep.longrepr, str)
         assert "Different tests were collected between" in rep.longrepr
 
 
 class TestDistReporter:
     @pytest.mark.xfail
-    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
+    def test_rsync_printing(self, pytester: pytest.Pytester, linecomp: Any) -> None:
         config = pytester.parseconfig()
-        from _pytest.pytest_terminal import TerminalReporter
+        from _pytest.terminal import TerminalReporter
 
         rep = TerminalReporter(config, file=linecomp.stringio)
         config.pluginmanager.register(rep, "terminalreporter")
@@ -252,15 +502,17 @@ class TestDistReporter:
 def test_report_collection_diff_equal() -> None:
     """Test reporting of equal collections."""
     from_collection = to_collection = ["aaa", "bbb", "ccc"]
-    assert report_collection_diff(from_collection, to_collection, 1, 2) is None
+    assert report_collection_diff(from_collection, to_collection, "1", "2") is None
 
 
 def test_default_max_worker_restart() -> None:
-    class config:
+    class MockConfig:
         class option:
-            maxworkerrestart: Optional[str] = None
+            maxworkerrestart: str | None = None
             numprocesses: int = 0
 
+    config = cast(pytest.Config, MockConfig)
+
     assert get_default_max_worker_restart(config) is None
 
     config.option.numprocesses = 2
@@ -290,7 +542,9 @@ def test_report_collection_diff_different() -> None:
         " bbb\n"
         "+XXX\n"
         " ccc\n"
-        "-YYY"
+        "-YYY\n"
+        "To see why this happens see 'Known limitations' in documentation "
+        "for pytest-xdist"
     )
 
     msg = report_collection_diff(from_collection, to_collection, "1", "2")
@@ -311,3 +565,70 @@ def test_pytest_issue419(pytester: pytest.Pytester) -> None:
     reprec = pytester.inline_run("-n1")
     reprec.assertoutcome(passed=2)
     assert 0
+
+
+Created = WorkerStatus.Created
+Initialized = WorkerStatus.Initialized
+ReadyForCollection = WorkerStatus.ReadyForCollection
+CollectionDone = WorkerStatus.CollectionDone
+
+
+@pytest.mark.parametrize(
+    "status_and_items, expected",
+    [
+        (
+            [],
+            "",
+        ),
+        (
+            [(Created, 0)],
+            "created: 1/1 worker",
+        ),
+        (
+            [(Created, 0), (Created, 0)],
+            "created: 2/2 workers",
+        ),
+        (
+            [(Initialized, 0), (Created, 0)],
+            "initialized: 1/2 workers",
+        ),
+        (
+            [(Initialized, 0), (Initialized, 0)],
+            "initialized: 2/2 workers",
+        ),
+        (
+            [(ReadyForCollection, 0), (Created, 0)],
+            "ready: 1/2 workers",
+        ),
+        (
+            [(ReadyForCollection, 0), (ReadyForCollection, 0)],
+            "ready: 2/2 workers",
+        ),
+        (
+            [(CollectionDone, 12), (Created, 0)],
+            "collecting: 1/2 workers",
+        ),
+        (
+            [(CollectionDone, 12), (CollectionDone, 12)],
+            "2 workers [12 items]",
+        ),
+        (
+            [(CollectionDone, 1), (CollectionDone, 1)],
+            "2 workers [1 item]",
+        ),
+        (
+            [(CollectionDone, 1)],
+            "1 worker [1 item]",
+        ),
+        # Different number of tests collected will raise an error and should not happen in practice,
+        # but we test for it anyway.
+        (
+            [(CollectionDone, 1), (CollectionDone, 12)],
+            "2 workers [1 item]",
+        ),
+    ],
+)
+def test_get_workers_status_line(
+    status_and_items: Sequence[tuple[WorkerStatus, int]], expected: str
+) -> None:
+    assert get_workers_status_line(status_and_items) == expected

testing/test_looponfail.py

@@ -1,22 +1,24 @@
-import py
-import pytest
-import shutil
-import textwrap
+from __future__ import annotations
+
+import pathlib
 from pathlib import Path
+import shutil
+import tempfile
+import textwrap
+import unittest.mock
+
+import pytest
 
 from xdist.looponfail import RemoteControl
 from xdist.looponfail import StatRecorder
 
 
-PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0)  # type: ignore[attr-defined]
-
-
 class TestStatRecorder:
     def test_filechange(self, tmp_path: Path) -> None:
         tmp = tmp_path
         hello = tmp / "hello.py"
         hello.touch()
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
@@ -56,15 +58,12 @@ class TestStatRecorder:
         tmp = tmp_path
         tmp.joinpath("dir").mkdir()
         tmp.joinpath("dir", "hello.py").touch()
-        sd = StatRecorder([py.path.local(tmp)])
-        assert not sd.fil(py.path.local(tmp / "dir"))
+        sd = StatRecorder([tmp])
+        assert not sd.fil(tmp / "dir")
 
-    def test_filechange_deletion_race(
-        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-    ) -> None:
+    def test_filechange_deletion_race(self, tmp_path: Path) -> None:
         tmp = tmp_path
-        pytmp = py.path.local(tmp)
-        sd = StatRecorder([pytmp])
+        sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
@@ -76,8 +75,12 @@ class TestStatRecorder:
         p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
-
+        dirname = str(tmp)
+        dirnames: list[str] = []
+        filenames = [str(p)]
+        with unittest.mock.patch(
+            "os.walk", return_value=[(dirname, dirnames, filenames)], autospec=True
+        ):
             changed = sd.check()
         assert changed
 
@@ -85,7 +88,7 @@ class TestStatRecorder:
         tmp = tmp_path
         hello = tmp / "hello.py"
         hello.touch()
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
@@ -100,7 +103,7 @@ class TestStatRecorder:
         self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:
         tmp = tmp_path
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
 
         ret_values = [True, False]
         monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -120,12 +123,11 @@ class TestRemoteControl:
         item = pytester.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
-        failures = control.runsession()
+        failures = control.runsession()[0]
         assert failures
         control.setup()
-        item_path = item.path if PYTEST_GTE_7 else Path(item.fspath)  # type: ignore[attr-defined]
-        item_path.write_text("def test_func():\n assert 1\n")
-        removepyc(item_path)
+        item.path.write_text("def test_func():\n assert 1\n")
+        removepyc(item.path)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
@@ -141,7 +143,8 @@ class TestRemoteControl:
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)  # type: ignore[attr-defined]
+        modcol_path = modcol.path
+
         modcol_path.write_text(
             textwrap.dedent(
                 """
@@ -170,10 +173,7 @@ class TestRemoteControl:
                 """
             )
         )
-        if PYTEST_GTE_7:
-            parent = modcol.path.parent.parent  # type: ignore[attr-defined]
-        else:
-            parent = Path(modcol.fspath.dirpath().dirpath())
+        parent = modcol.path.parent.parent
         monkeypatch.chdir(parent)
         modcol.config.args = [
             str(Path(x).relative_to(parent)) for x in modcol.config.args
@@ -184,6 +184,43 @@ class TestRemoteControl:
         control.loop_once()
         assert control.failures
 
+    def test_ignore_sys_path_hook_entry(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        # Modifying sys.path as seen by the worker process is a bit tricky,
+        # because any changes made in the current process do not carry over.
+        # However, we can leverage the `sitecustomize` behavior to run arbitrary
+        # code when the subprocess interpreter is starting up. We just need to
+        # install our module in the search path, which we can accomplish by
+        # adding a temporary directory to PYTHONPATH.
+        tmpdir = tempfile.TemporaryDirectory()
+        with open(pathlib.Path(tmpdir.name) / "sitecustomize.py", "w") as custom:
+            print(
+                textwrap.dedent(
+                    """
+                    import sys
+                    sys.path.append('dummy.__path_hook__')
+                    """
+                ),
+                file=custom,
+            )
+
+        monkeypatch.setenv("PYTHONPATH", tmpdir.name, prepend=":")
+
+        item = pytester.getitem(
+            textwrap.dedent(
+                """
+                def test_func():
+                    import sys
+                    assert "dummy.__path_hook__" in sys.path
+                """
+            )
+        )
+        control = RemoteControl(item.config)
+        control.setup()
+        topdir, failures = control.runsession()[:2]
+        assert not failures
+
 
 class TestLooponFailing:
     def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
@@ -202,8 +239,7 @@ class TestLooponFailing:
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
-        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
-        modcol_path.write_text(
+        modcol.path.write_text(
             textwrap.dedent(
                 """
                 def test_one():
@@ -213,7 +249,7 @@ class TestLooponFailing:
                 """
             )
         )
-        removepyc(modcol_path)
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert not remotecontrol.failures
 
@@ -231,8 +267,7 @@ class TestLooponFailing:
         assert len(remotecontrol.failures) == 1
         assert "test_one" in remotecontrol.failures[0]
 
-        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
-        modcol_path.write_text(
+        modcol.path.write_text(
             textwrap.dedent(
                 """
                 def test_one():
@@ -242,7 +277,7 @@ class TestLooponFailing:
                 """
             )
         )
-        removepyc(modcol_path)
+        removepyc(modcol.path)
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 0
         remotecontrol.loop_once()
@@ -297,7 +332,7 @@ class TestLooponFailing:
         remotecontrol = RemoteControl(modcol.config)
         orig_runsession = remotecontrol.runsession
 
-        def runsession_dups():
+        def runsession_dups() -> tuple[list[str], list[str], bool]:
             # twisted.trial test cases may report multiple errors.
             failures, reports, collection_failed = orig_runsession()
             print(failures)

testing/test_newhooks.py

@@ -14,9 +14,9 @@ class TestHooks:
         )
 
     def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
-        """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
-        """
+        """Test that log reports from pytest_runtest_logreport when running with
+        xdist contain "node", "nodeid", "worker_id", and "testrun_uid"
+        attributes (#8)."""
         pytester.makeconftest(
             """
             def pytest_runtest_logreport(report):
@@ -94,3 +94,28 @@ class TestCrashItem:
         res = pytester.runpytest("-n2", "-s")
         res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+    def test_handlecrashitem_one(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook with just one test."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n1", "-s", "-k", "test_b")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(
+            [
+                "FAILED test_handlecrashitem_one.py::test_b",
+                "FAILED test_handlecrashitem_one.py::test_b",
+            ]
+        )

testing/test_plugin.py

@@ -1,10 +1,21 @@
 from contextlib import suppress
+import os
 from pathlib import Path
+import sys
 
 import execnet
+import pytest
+
 from xdist.workermanage import NodeManager
 
-import pytest
+
+@pytest.fixture
+def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Make pytest-xdist believe the system has 3 CPUs."""
+    # block import
+    monkeypatch.setitem(sys.modules, "psutil", None)
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setattr(os, "cpu_count", lambda: 3)
 
 
 def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
@@ -37,13 +48,31 @@ def test_dist_options(pytester: pytest.Pytester) -> None:
     check_options(config)
     assert config.option.dist == "load"
 
+    config = pytester.parseconfigure("--numprocesses", "0")
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
+    config = pytester.parseconfigure("--numprocesses", "0", "-d")
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
+    config = pytester.parseconfigure(
+        "--numprocesses", "0", "--dist", "each", "--tx", "2*popen"
+    )
+    check_options(config)
+    assert config.option.dist == "no"
+    assert config.option.tx == []
+
 
 def test_auto_detect_cpus(
     pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    import os
     from xdist.plugin import pytest_cmdline_main as check_options
 
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
     with suppress(ImportError):
         import psutil
 
@@ -65,13 +94,8 @@ def test_auto_detect_cpus(
     check_options(config)
     assert config.getoption("numprocesses") == 99
 
-    config = pytester.parseconfigure("-nauto", "--pdb")
-    check_options(config)
-    assert config.getoption("usepdb")
-    assert config.getoption("numprocesses") == 0
-    assert config.getoption("dist") == "no"
-
-    config = pytester.parseconfigure("-nlogical", "--pdb")
+    for numprocesses in (0, "auto", "logical"):
+        config = pytester.parseconfigure(f"-n{numprocesses}", "--pdb")
         check_options(config)
         assert config.getoption("usepdb")
         assert config.getoption("numprocesses") == 0
@@ -91,6 +115,7 @@ def test_auto_detect_cpus_psutil(
 
     psutil = pytest.importorskip("psutil")
 
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
     monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
 
     config = pytester.parseconfigure("-nauto")
@@ -102,6 +127,22 @@ def test_auto_detect_cpus_psutil(
     assert config.getoption("numprocesses") == 84
 
 
+def test_auto_detect_cpus_os(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
 def test_hook_auto_num_workers(
     pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -122,20 +163,105 @@ def test_hook_auto_num_workers(
     assert config.getoption("numprocesses") == 42
 
 
-def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
+def test_hook_auto_num_workers_arg(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # config.option.numprocesses is a pytest feature,
+    # but we document it so let's test it.
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = pytester.parseconfigure("-n1", "--boxed")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers(config):
+            if config.option.numprocesses == 'auto':
+                return 42
+            if config.option.numprocesses == 'logical':
+                return 8
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
     check_options(config)
-    assert config.option.forked
+    assert config.getoption("numprocesses") == 42
 
-    config = pytester.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-nlogical")
     check_options(config)
-    assert not config.option.forked
+    assert config.getoption("numprocesses") == 8
 
-    config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
+
+def test_hook_auto_num_workers_none(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    # Returning None from a hook to skip it is pytest behavior,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.delenv("PYTEST_XDIST_AUTO_NUM_WORKERS", raising=False)
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return None
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
     check_options(config)
-    assert config.option.forked
+    assert config.getoption("numprocesses") == 3
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "5")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 5
+
+
+def test_envvar_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "7")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+
+def test_envvar_auto_num_workers_warn(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "fourscore")
+
+    config = pytester.parseconfigure("-nauto")
+    with pytest.warns(UserWarning):
+        check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_auto_num_workers_hook_overrides_envvar(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus: None
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "987")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 2
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
 
 
 def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
@@ -169,7 +295,7 @@ class TestDistOptions:
     def test_getxspecs(self, pytester: pytest.Pytester) -> None:
         config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
         nodemanager = NodeManager(config)
-        xspecs = nodemanager._getxspecs()
+        xspecs = nodemanager._gettxspecs()
         assert len(xspecs) == 2
         print(xspecs)
         assert xspecs[0].popen
@@ -177,7 +303,7 @@ class TestDistOptions:
 
     def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
         config = pytester.parseconfigure("--tx=3*popen")
-        xspecs = NodeManager(config)._getxspecs()
+        xspecs = NodeManager(config)._gettxspecs()
         assert len(xspecs) == 3
         assert xspecs[1].popen
 

testing/test_remote.py

@@ -1,58 +1,73 @@
+from __future__ import annotations
+
+import marshal
 import pprint
-import py
-import pytest
+from queue import Queue
 import sys
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Union
 import uuid
 
-from xdist.workermanage import WorkerController
 import execnet
-import marshal
+import pytest
+
+from xdist.workermanage import NodeManager
+from xdist.workermanage import WorkerController
 
-from queue import Queue
 
 WAIT_TIMEOUT = 10.0
 
 
-def check_marshallable(d):
+def check_marshallable(d: object) -> None:
     try:
-        marshal.dumps(d)
-    except ValueError:
+        marshal.dumps(d)  # type: ignore[arg-type]
+    except ValueError as e:
         pprint.pprint(d)
-        raise ValueError("not marshallable")
+        raise ValueError("not marshallable") from e
 
 
 class EventCall:
-    def __init__(self, eventcall):
+    def __init__(self, eventcall: tuple[str, dict[str, Any]]) -> None:
         self.name, self.kwargs = eventcall
 
-    def __str__(self):
-        return "<EventCall {}(**{})>".format(self.name, self.kwargs)
+    def __str__(self) -> str:
+        return f"<EventCall {self.name}(**{self.kwargs})>"
 
 
 class WorkerSetup:
-    use_callback = False
-
-    def __init__(self, request, pytester: pytest.Pytester) -> None:
+    def __init__(
+        self, request: pytest.FixtureRequest, pytester: pytest.Pytester
+    ) -> None:
         self.request = request
         self.pytester = pytester
+        self.use_callback = False
         self.events = Queue()  # type: ignore[var-annotated]
 
     def setup(self) -> None:
         self.pytester.chdir()
         # import os ; os.environ['EXECNET_DEBUG'] = "2"
-        self.gateway = execnet.makegateway()
+        self.gateway = execnet.makegateway("execmodel=main_thread_only//popen")
         self.config = config = self.pytester.parseconfigure()
-        putevent = self.use_callback and self.events.put or None
+        putevent = self.events.put if self.use_callback else None
 
         class DummyMananger:
             testrunuid = uuid.uuid4().hex
             specs = [0, 1]
 
-        self.slp = WorkerController(DummyMananger, self.gateway, config, putevent)
+        nodemanager = cast(NodeManager, DummyMananger)
+
+        self.slp = WorkerController(
+            nodemanager=nodemanager,
+            gateway=self.gateway,
+            config=config,
+            putevent=putevent,  # type: ignore[arg-type]
+        )
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
-    def popevent(self, name=None):
+    def popevent(self, name: str | None = None) -> EventCall:
         while 1:
             if self.use_callback:
                 data = self.events.get(timeout=WAIT_TIMEOUT)
@@ -61,39 +76,35 @@ class WorkerSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping {}".format(ev))
+            print(f"skipping {ev}")
 
-    def sendcommand(self, name, **kwargs):
+    def sendcommand(self, name: str, **kwargs: Any) -> None:
         self.slp.sendcommand(name, **kwargs)
 
 
 @pytest.fixture
-def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
+def worker(request: pytest.FixtureRequest, pytester: pytest.Pytester) -> WorkerSetup:
     return WorkerSetup(request, pytester)
 
 
-@pytest.mark.xfail(reason="#59")
-def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
-    from xdist.remote import remote_initconfig
-
-    config1 = pytester.parseconfig()
-    config2 = remote_initconfig(config1.option.__dict__, config1.args)
-    assert config2.option.__dict__ == config1.option.__dict__
-    assert config2.pluginmanager.getplugin("terminal") in (-1, None)
-
-
 class TestWorkerInteractor:
+    UnserializerReport = Callable[
+        [dict[str, Any]], Union[pytest.CollectReport, pytest.TestReport]
+    ]
+
     @pytest.fixture
-    def unserialize_report(self, pytestconfig):
-        def unserialize(data):
-            return pytestconfig.hook.pytest_report_from_serializable(
+    def unserialize_report(self, pytestconfig: pytest.Config) -> UnserializerReport:
+        def unserialize(
+            data: dict[str, Any],
+        ) -> pytest.CollectReport | pytest.TestReport:
+            return pytestconfig.hook.pytest_report_from_serializable(  # type: ignore[no-any-return]
                 config=pytestconfig, data=data
             )
 
         return unserialize
 
     def test_basic_collect_and_runtests(
-        self, worker: WorkerSetup, unserialize_report
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
     ) -> None:
         worker.pytester.makepyfile(
             """
@@ -108,7 +119,7 @@ class TestWorkerInteractor:
         assert ev.name == "collectionstart"
         assert not ev.kwargs
         ev = worker.popevent("collectionfinish")
-        assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
+        assert ev.kwargs["topdir"] == str(worker.pytester.path)
         ids = ev.kwargs["ids"]
         assert len(ids) == 1
         worker.sendcommand("runtests", indices=list(range(len(ids))))
@@ -126,7 +137,9 @@ class TestWorkerInteractor:
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
+    def test_remote_collect_skip(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
         worker.pytester.makepyfile(
             """
             import pytest
@@ -140,11 +153,14 @@ class TestWorkerInteractor:
         assert ev.name == "collectreport"
         rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
+        assert isinstance(rep.longrepr, tuple)
         assert rep.longrepr[2] == "Skipped: hello"
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
+    def test_remote_collect_fail(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
         worker.pytester.makepyfile("""aasd qwe""")
         worker.setup()
         ev = worker.popevent("collectionstart")
@@ -156,7 +172,9 @@ class TestWorkerInteractor:
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
+    def test_runtests_all(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
         worker.pytester.makepyfile(
             """
             def test_func(): pass
@@ -175,7 +193,7 @@ class TestWorkerInteractor:
         worker.sendcommand("runtests_all")
         worker.sendcommand("shutdown")
         for func in "::test_func", "::test_func2":
-            for i in range(3):  # setup/call/teardown
+            for _ in range(3):  # setup/call/teardown
                 ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
                 rep = unserialize_report(ev.kwargs["data"])
@@ -216,12 +234,107 @@ class TestWorkerInteractor:
     ) -> None:
         worker.use_callback = True
         worker.setup()
-        worker.slp.process_from_remote(("<nonono>", ()))
+        worker.slp.process_from_remote(("<nonono>", {}))
         out, err = capsys.readouterr()
         assert "INTERNALERROR> ValueError: unknown event: <nonono>" in out
         ev = worker.popevent()
         assert ev.name == "errordown"
 
+    def test_steal_work(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
+            import time
+            def test_func(): time.sleep(1)
+            def test_func2(): pass
+            def test_func3(): pass
+            def test_func4(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 4
+        worker.sendcommand("runtests_all")
+
+        # wait for test_func setup
+        ev = worker.popevent("testreport")
+        rep = unserialize_report(ev.kwargs["data"])
+        assert rep.nodeid.endswith("::test_func")
+        assert rep.when == "setup"
+
+        worker.sendcommand("steal", indices=[1, 2])
+        ev = worker.popevent("unscheduled")
+        # Cannot steal index 1 because it is completed already, so do not steal any.
+        assert ev.kwargs["indices"] == []
+
+        # Index 2 can be stolen, as it is still pending.
+        worker.sendcommand("steal", indices=[2])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == [2]
+
+        reports = [
+            ("test_func", "call"),
+            ("test_func", "teardown"),
+            ("test_func2", "setup"),
+            ("test_func2", "call"),
+            ("test_func2", "teardown"),
+        ]
+
+        for func, when in reports:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith(f"::{func}")
+            assert rep.when == when
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func4")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
+    def test_steal_empty_queue(
+        self, worker: WorkerSetup, unserialize_report: UnserializerReport
+    ) -> None:
+        worker.pytester.makepyfile(
+            """
+            def test_func(): pass
+            def test_func2(): pass
+        """
+        )
+        worker.setup()
+        ev = worker.popevent("collectionfinish")
+        ids = ev.kwargs["ids"]
+        assert len(ids) == 2
+        worker.sendcommand("runtests_all")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func")
+            assert rep.when == when
+
+        worker.sendcommand("steal", indices=[0, 1])
+        ev = worker.popevent("unscheduled")
+        assert ev.kwargs["indices"] == []
+
+        worker.sendcommand("shutdown")
+
+        for when in ["setup", "call", "teardown"]:
+            ev = worker.popevent("testreport")
+            rep = unserialize_report(ev.kwargs["data"])
+            assert rep.nodeid.endswith("::test_func2")
+            assert rep.when == when
+
+        ev = worker.popevent("workerfinished")
+        assert "workeroutput" in ev.kwargs
+
 
 def test_remote_env_vars(pytester: pytest.Pytester) -> None:
     pytester.makepyfile(
@@ -255,20 +368,16 @@ def test_remote_mainargv(pytester: pytest.Pytester) -> None:
     outer_argv = sys.argv
 
     pytester.makepyfile(
-        """
+        f"""
         def test_mainargv(request):
-            assert request.config.workerinput["mainargv"] == {!r}
-        """.format(
-            outer_argv
-        )
+            assert request.config.workerinput["mainargv"] == {outer_argv!r}
+        """
     )
     result = pytester.runpytest("-n1")
     assert result.ret == 0
 
 
-def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
-    if not hasattr(request.config._parser, "prog"):
-        pytest.skip("prog not available in config parser")
+def test_remote_usage_prog(pytester: pytest.Pytester) -> None:
     pytester.makeconftest(
         """
         import pytest

testing/test_workermanage.py

@@ -1,23 +1,35 @@
-import execnet
-import py
-import pytest
+from __future__ import annotations
+
+from pathlib import Path
 import shutil
 import textwrap
-from pathlib import Path
+import warnings
+
+import execnet
+import pytest
+from util import generate_warning
+
 from xdist import workermanage
-from xdist.workermanage import HostRSync, NodeManager
+from xdist._path import visit_path
+from xdist.remote import serialize_warning_message
+from xdist.workermanage import HostRSync
+from xdist.workermanage import NodeManager
+from xdist.workermanage import unserialize_warning_message
+
 
 pytest_plugins = "pytester"
 
 
 @pytest.fixture
-def hookrecorder(request, config, pytester: pytest.Pytester):
+def hookrecorder(
+    config: pytest.Config, pytester: pytest.Pytester
+) -> pytest.HookRecorder:
     hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
     return hookrecorder
 
 
 @pytest.fixture
-def config(pytester: pytest.Pytester):
+def config(pytester: pytest.Pytester) -> pytest.Config:
     return pytester.parseconfig()
 
 
@@ -36,24 +48,23 @@ def dest(tmp_path: Path) -> Path:
 
 
 @pytest.fixture
-def workercontroller(monkeypatch: pytest.MonkeyPatch):
+def workercontroller(monkeypatch: pytest.MonkeyPatch) -> None:
     class MockController:
-        def __init__(self, *args):
+        def __init__(self, *args: object) -> None:
             pass
 
-        def setup(self):
+        def setup(self) -> None:
             pass
 
     monkeypatch.setattr(workermanage, "WorkerController", MockController)
-    return MockController
 
 
 class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, config) -> None:
+    def test_popen_no_default_chdir(self, config: pytest.Config) -> None:
         gm = NodeManager(config, ["popen"])
         assert gm.specs[0].chdir is None
 
-    def test_default_chdir(self, config) -> None:
+    def test_default_chdir(self, config: pytest.Config) -> None:
         specs = ["ssh=noco", "socket=xyz"]
         for spec in NodeManager(config, specs).specs:
             assert spec.chdir == "pyexecnetcache"
@@ -61,53 +72,68 @@ class TestNodeManagerPopen:
             assert spec.chdir == "abc"
 
     def test_popen_makegateway_events(
-        self, config, hookrecorder, workercontroller
+        self,
+        config: pytest.Config,
+        hookrecorder: pytest.HookRecorder,
+        workercontroller: None,
     ) -> None:
         hm = NodeManager(config, ["popen"] * 2)
-        hm.setup_nodes(None)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
         call = hookrecorder.popcall("pytest_xdist_setupnodes")
         assert len(call.specs) == 2
 
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.spec == execnet.XSpec("popen")
-        assert call.gateway.id == "gw0"
-        call = hookrecorder.popcall("pytest_xdist_newgateway")
-        assert call.gateway.id == "gw1"
+        # check expected gateways
+        gw_calls = [
+            hookrecorder.popcall("pytest_xdist_newgateway"),
+            hookrecorder.popcall("pytest_xdist_newgateway"),
+        ]
+        assert {c.gateway.id for c in gw_calls} == {"gw0", "gw1"}
+
+        for c in gw_calls:
+            expected_spec = execnet.XSpec(
+                f"execmodel=main_thread_only//popen//id={c.gateway.id}"
+            )
+            assert c.gateway.spec == expected_spec
+
         assert len(hm.group) == 2
         hm.teardown_nodes()
         assert not len(hm.group)
 
     def test_popens_rsync(
-        self, config, source: Path, dest: Path, workercontroller
+        self,
+        config: pytest.Config,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
     ) -> None:
         hm = NodeManager(config, ["popen"] * 2)
-        hm.setup_nodes(None)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
         assert len(hm.group) == 2
         for gw in hm.group:
 
             class pseudoexec:
                 args = []  # type: ignore[var-annotated]
 
-                def __init__(self, *args):
+                def __init__(self, *args: object) -> None:
                     self.args.extend(args)
 
-                def waitclose(self):
+                def waitclose(self) -> None:
                     pass
 
-            gw.remote_exec = pseudoexec
+            gw.remote_exec = pseudoexec  # type: ignore[assignment]
         notifications = []
         for gw in hm.group:
             hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
         assert not notifications
         hm.teardown_nodes()
         assert not len(hm.group)
-        assert "sys.path.insert" in gw.remote_exec.args[0]
+        assert "sys.path.insert" in gw.remote_exec.args[0]  # type: ignore[attr-defined]
 
     def test_rsync_popen_with_path(
-        self, config, source: Path, dest: Path, workercontroller
+        self, config: pytest.Config, source: Path, dest: Path, workercontroller: None
     ) -> None:
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
-        hm.setup_nodes(None)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
         source.joinpath("dir1", "dir2").mkdir(parents=True)
         source.joinpath("dir1", "dir2", "hello").touch()
         notifications = []
@@ -123,15 +149,15 @@ class TestNodeManagerPopen:
 
     def test_rsync_same_popen_twice(
         self,
-        config,
+        config: pytest.Config,
         source: Path,
         dest: Path,
-        hookrecorder,
-        workercontroller,
+        hookrecorder: pytest.HookRecorder,
+        workercontroller: None,
     ) -> None:
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
         hm.roots = []
-        hm.setup_nodes(None)
+        hm.setup_nodes(None)  # type: ignore[arg-type]
         source.joinpath("dir1", "dir2").mkdir(parents=True)
         source.joinpath("dir1", "dir2", "hello").touch()
         gw = hm.group[0]
@@ -154,15 +180,12 @@ class TestHRSync:
         source.joinpath("somedir").mkdir()
         source.joinpath("somedir", "editfile~").touch()
         syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
-        assert len(files) == 3
-        basenames = [x.basename for x in files]
-        assert "dir" in basenames
-        assert "file.txt" in basenames
-        assert "somedir" in basenames
+        files = list(visit_path(source, recurse=syncer.filter, filter=syncer.filter))
+        names = {x.name for x in files}
+        assert names == {"dir", "file.txt", "somedir"}
 
     def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
-        gw = execnet.makegateway("popen//chdir=%s" % dest)
+        gw = execnet.makegateway("execmodel=main_thread_only//popen//chdir=%s" % dest)
         finished = []
         rsync = HostRSync(source)
         rsync.add_target_host(gw, finished=lambda: finished.append(1))
@@ -191,11 +214,15 @@ class TestNodeManager:
         p = Path(p)
         print("remote curdir", p)
         assert p == dest.joinpath(config.rootpath.name)
-        assert p.joinpath("dir1").check()
-        assert p.joinpath("dir1", "file1").check()
+        assert p.joinpath("dir1").is_dir()
+        assert p.joinpath("dir1", "file1").is_file()
 
     def test_popen_rsync_subdir(
-        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
     ) -> None:
         dir1 = source / "dir1"
         dir1.mkdir()
@@ -209,7 +236,8 @@ class TestNodeManager:
                     "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
                 )
             )
-            nodemanager.setup_nodes(None)  # calls .rsync_roots()
+            # calls .rsync_roots()
+            nodemanager.setup_nodes(None)  # type: ignore[arg-type]
             if rsyncroot == source:
                 dest = dest.joinpath("source")
             assert dest.joinpath("dir1").exists()
@@ -218,14 +246,19 @@ class TestNodeManager:
             nodemanager.teardown_nodes()
 
     @pytest.mark.parametrize(
-        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+        ["flag", "expects_report"],
+        [
+            ("-q", False),
+            ("", False),
+            ("-v", True),
+        ],
     )
     def test_rsync_report(
         self,
         pytester: pytest.Pytester,
         source: Path,
         dest: Path,
-        workercontroller,
+        workercontroller: None,
         capsys: pytest.CaptureFixture[str],
         flag: str,
         expects_report: bool,
@@ -236,7 +269,8 @@ class TestNodeManager:
         if flag:
             args.append(flag)
         nodemanager = NodeManager(pytester.parseconfig(*args))
-        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
         out, _ = capsys.readouterr()
         if expects_report:
             assert "<= pytest/__init__.py" in out
@@ -244,7 +278,11 @@ class TestNodeManager:
             assert "<= pytest/__init__.py" not in out
 
     def test_init_rsync_roots(
-        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
     ) -> None:
         dir2 = source.joinpath("dir1", "dir2")
         dir2.mkdir(parents=True)
@@ -262,13 +300,18 @@ class TestNodeManager:
         )
         config = pytester.parseconfig(source)
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
         assert dest.joinpath("dir2").exists()
         assert not dest.joinpath("dir1").exists()
         assert not dest.joinpath("bogus").exists()
 
     def test_rsyncignore(
-        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
     ) -> None:
         dir2 = source.joinpath("dir1", "dir2")
         dir2.mkdir(parents=True)
@@ -292,7 +335,8 @@ class TestNodeManager:
         config = pytester.parseconfig(source)
         config.option.rsyncignore = ["bar"]
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
-        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        # calls .rsync_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
         assert dest.joinpath("dir1").exists()
         assert not dest.joinpath("dir1", "dir2").exists()
         assert dest.joinpath("dir5", "file").exists()
@@ -301,14 +345,19 @@ class TestNodeManager:
         assert not dest.joinpath("bar").exists()
 
     def test_optimise_popen(
-        self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
+        self,
+        pytester: pytest.Pytester,
+        source: Path,
+        dest: Path,
+        workercontroller: None,
     ) -> None:
         specs = ["popen"] * 3
         source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
         source.joinpath("a").mkdir()
         config = pytester.parseconfig(source)
         nodemanager = NodeManager(config, specs)
-        nodemanager.setup_nodes(None)  # calls .rysnc_roots()
+        # calls .rysnc_roots()
+        nodemanager.setup_nodes(None)  # type: ignore[arg-type]
         for gwspec in nodemanager.specs:
             assert gwspec._samefilesystem()
             assert not gwspec.chdir
@@ -326,3 +375,106 @@ class TestNodeManager:
         )
         (rep,) = reprec.getreports("pytest_runtest_logreport")
         assert rep.passed
+
+    def test_proxy_gateway_setup_nodes(self, pytester: pytest.Pytester) -> None:
+        nodemanager = NodeManager(
+            pytester.parseconfig(
+                "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy"
+            )
+        )
+        nodes = nodemanager.setup_nodes(None)  # type: ignore[arg-type]
+
+        # Proxy gateways are considered as an execnet gateway
+        assert len(nodemanager.group) == 2
+
+        # Proxy gateways do not run workers
+        assert len(nodes) == 1
+
+    def test_proxy_gateway(self, pytester: pytest.Pytester) -> None:
+        pytester.makepyfile(
+            __init__="",
+            test_x="""
+            def test_one():
+                pass
+        """,
+        )
+        reprec = pytester.inline_run(
+            "-d", "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy"
+        )
+        rep = reprec.getreports("pytest_runtest_logreport")
+        assert rep[1].passed
+
+
+class MyWarning(UserWarning):
+    pass
+
+
+@pytest.mark.parametrize(
+    "w_cls",
+    [
+        UserWarning,
+        MyWarning,
+        "Imported",
+        pytest.param(
+            "Nested",
+            marks=pytest.mark.xfail(reason="Nested warning classes are not supported."),
+        ),
+    ],
+)
+def test_unserialize_warning_msg(w_cls: type[Warning] | str) -> None:
+    """Test that warning serialization process works well."""
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        if not isinstance(w_cls, str):
+            warnings.warn("hello", w_cls)
+        elif w_cls == "Imported":
+            generate_warning()
+        elif w_cls == "Nested":
+            # dynamic creation
+            class MyWarning2(UserWarning):
+                pass
+
+            warnings.warn("hello", MyWarning2)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+    w_msg2 = unserialize_warning_message(data)
+
+    # Compare the two objects
+    all_keys = set(vars(w_msg).keys()).union(set(vars(w_msg2).keys()))
+    for k in all_keys:
+        v1 = getattr(w_msg, k)
+        v2 = getattr(w_msg2, k)
+        if k == "message":
+            assert type(v1) is type(v2)
+            assert v1.args == v2.args
+        else:
+            assert v1 == v2
+
+
+class MyWarningUnknown(UserWarning):
+    # Changing the __module__ attribute is only safe if class can be imported
+    # from there
+    __module__ = "unknown"
+
+
+def test_warning_serialization_tweaked_module() -> None:
+    """Test for GH#404."""
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        warnings.warn("hello", MyWarningUnknown)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+
+    # __module__ cannot be found!
+    with pytest.raises(ModuleNotFoundError):
+        unserialize_warning_message(data)

testing/util.py

@@ -0,0 +1,9 @@
+import warnings
+
+
+class MyWarning2(UserWarning):
+    pass
+
+
+def generate_warning() -> None:
+    warnings.warn(MyWarning2("hello"))

tox.ini

@@ -1,33 +1,26 @@
 [tox]
 envlist =
   linting
-  py{36,37,38,39,310}-pytestlatest
-  py38-pytestmain
-  py38-psutil
-  py38-setproctitle
+  py{39,310,311,312,313}-pytestlatest
+  py310-pytestmain
+  py310-psutil
+  py310-setproctitle
+isolated_build = true
 
 [testenv]
-extras = testing
+extras =
+  testing
+  psutil: psutil
+  setproctitle: setproctitle
 deps =
+  pytestmin: pytest==7.0.0
   pytestlatest: pytest
   pytestmain: git+https://github.com/pytest-dev/pytest.git
 commands =
-  pytest {posargs}
-
-[testenv:py38-psutil]
-extras =
-  testing
-  psutil
-commands =
-  pytest {posargs:-k psutil}
-
-[testenv:py38-setproctitle]
-extras =
-  testing
-  setproctitle
-deps = pytest
-commands =
-  pytest {posargs}
+  pytest {posargs:{env:_XDIST_TOX_DEFAULT_POSARGS:}}
+setenv =
+  _XDIST_TOX_DEFAULT_POSARGS={env:_XDIST_TOX_POSARGS_PSUTIL:}
+  psutil: _XDIST_TOX_POSARGS_PSUTIL=-k psutil
 
 [testenv:linting]
 skip_install = True
@@ -39,22 +32,19 @@ commands = pre-commit run --all-files --show-diff-on-failure
 
 [testenv:release]
 changedir =
-decription = do a release, required posarg of the version number
-basepython = python3.7
+description = do a release, required posarg of the version number
 skipsdist = True
 usedevelop = True
 passenv = *
 deps =
   towncrier
 commands =
-  towncrier --version {posargs} --yes
+  towncrier build --version {posargs} --yes
 
-[pytest]
-# pytest-services also defines a worker_id fixture, disable
-# it so they don't conflict with each other (#611).
-addopts = -ra -p no:pytest-services
-testpaths = testing
-
-[flake8]
-max-line-length = 120
-ignore = E203,W503
+[testenv:docs]
+usedevelop = True
+deps =
+    sphinx
+    sphinx_rtd_theme
+commands =
+    sphinx-build -W --keep-going -b html docs docs/_build/html {posargs:}