Update CHANGELOG

fix and update packaging

.github/workflows/main.yml

@@ -40,7 +40,10 @@ jobs:
             python: "3.8"
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
@@ -62,7 +65,10 @@ jobs:
     needs: build
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+      with:
+        # Needed to fetch tags, which are required by setuptools-scm.
+        fetch-depth: 0
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
@@ -70,12 +76,12 @@ jobs:
     - name: Install wheel
       run: |
         python -m pip install --upgrade pip
-        pip install wheel
+        pip install build
     - name: Build package
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
     - name: Publish package to PyPI
-      uses: pypa/gh-action-pypi-publish@master
+      uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.pypi_token }}

.pre-commit-config.yaml

@@ -1,11 +1,21 @@
 repos:
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v1.7.6
+    hooks:
+    -   id: autoflake
+        args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"]
 -   repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.3.0
     hooks:
     -   id: black
         args: [--safe, --quiet, --target-version, py35]
+-   repo: https://github.com/asottile/blacken-docs
+    rev: v1.12.1
+    hooks:
+    -   id: blacken-docs
+        additional_dependencies: [black==20.8b1]
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -16,7 +26,7 @@ repos:
     hooks:
     -   id: flake8
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.1
+    rev: v2.32.1
     hooks:
     -   id: pyupgrade
         args: [--py3-plus]
@@ -29,7 +39,7 @@ repos:
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v0.960
     hooks:
     -   id: mypy
         files: ^(src/|testing/)

CHANGELOG.rst

@@ -1,6 +1,90 @@
+pytest-xdist 3.0.2 (2022-10-25)
+===============================
+
+Bug Fixes
+---------
+
+- `#813 <https://github.com/pytest-dev/pytest-xdist/issues/813>`_: Cancel shutdown when a crashed worker is restarted.
+
+
+Deprecations
+------------
+
+- `#825 <https://github.com/pytest-dev/pytest-xdist/issues/825>`_: The ``--rsyncdir`` command line argument and ``rsyncdirs`` config variable are deprecated.
+
+  The rsync feature will be removed in pytest-xdist 4.0.
+
+- `#826 <https://github.com/pytest-dev/pytest-xdist/issues/826>`_: The ``--looponfail`` command line argument and ``looponfailroots`` config variable are deprecated.
+
+  The loop-on-fail feature will be removed in pytest-xdist 4.0.
+
+
+Improved Documentation
+----------------------
+
+- `#791 <https://github.com/pytest-dev/pytest-xdist/issues/791>`_: Document the ``pytest_xdist_auto_num_workers`` hook.
+
+- `#796 <https://github.com/pytest-dev/pytest-xdist/issues/796>`_: Added known limitations section to documentation.
+
+- `#829 <https://github.com/pytest-dev/pytest-xdist/issues/829>`_: Document the ``-n logical`` option.
+
+
+Features
+--------
+
+- `#792 <https://github.com/pytest-dev/pytest-xdist/issues/792>`_: The environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` can now be used to
+  specify the default for ``-n auto`` and ``-n logical``.
+
+- `#812 <https://github.com/pytest-dev/pytest-xdist/issues/812>`_: Partially restore old initial batch distribution algorithm in ``LoadScheduling``.
+
+  pytest orders tests for optimal sequential execution - i. e. avoiding
+  unnecessary setup and teardown of fixtures. So executing tests in consecutive
+  chunks is important for optimal performance.
+
+  In v1.14, initial test distribution in ``LoadScheduling`` was changed to
+  round-robin, optimized for the corner case, when the number of tests is less
+  than ``2 * number of nodes``. At the same time, it became worse for all other
+  cases.
+
+  For example: if some tests use some "heavy" fixture, and these tests fit into
+  the initial batch, with round-robin distribution the fixture will be created
+  ``min(n_tests, n_workers)`` times, no matter how many other tests there are.
+
+  With the old algorithm (before v1.14), if there are enough tests not using
+  the fixture, the fixture was created only once.
+
+  So restore the old behavior for typical cases where the number of tests is
+  much greater than the number of workers (or, strictly speaking, when there
+  are at least 2 tests for every node).
+
+
+Removals
+--------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command-line option has been removed. If you still need this functionality, install `pytest-forked <https://pypi.org/project/pytest-forked>`__ separately.
+
+
+Trivial Changes
+---------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``py`` dependency has been dropped.
+
+- `#822 <https://github.com/pytest-dev/pytest-xdist/issues/822>`_: Replace internal usage of ``py.log`` with a custom solution (but with the same interface).
+
+- `#823 <https://github.com/pytest-dev/pytest-xdist/issues/823>`_: Remove usage of ``py._pydir`` as an rsync candidate.
+
+- `#824 <https://github.com/pytest-dev/pytest-xdist/issues/824>`_: Replace internal usages of ``py.path.local`` by ``pathlib.Path``.
+
+
 pytest-xdist 2.5.0 (2021-12-10)
 ===============================
 
+Deprecations and Removals
+-------------------------
+
+- `#468 <https://github.com/pytest-dev/pytest-xdist/issues/468>`_: The ``--boxed`` command line argument is deprecated. Install `pytest-forked <https://pypi.org/project/pytest-forked>`__ and use ``--forked`` instead. pytest-xdist 3.0.0 will remove the ``--boxed`` argument and ``pytest-forked`` dependency.
+
+
 Features
 --------
 

README.rst

@@ -25,520 +25,9 @@ tests across multiple CPUs to speed up test execution::
     pytest -n auto
 
 With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
-the tests randomly across them. There is also a number of `distribution modes`_ to choose from.
+the tests randomly across them.
 
-**NOTE**: due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+Documentation
+=============
 
-.. contents:: **Table of Contents**
+Documentation is available at `Read The Docs <https://pytest-xdist.readthedocs.io>`__.
-
-Installation
-------------
-
-Install the plugin with::
-
-    pip install pytest-xdist
-
-
-To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
-
-    pip install pytest-xdist[psutil]
-
-
-Features
---------
-
-* Test run parallelization_: tests can be executed across  multiple CPUs or hosts.
-  This allows to speed up development or to use special resources of `remote machines`_.
-
-* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
-  pytest waits until a file in your project changes and then re-runs
-  the previously failing tests.  This is repeated until all tests pass
-  after which again a full run is performed.
-
-* `Multi-Platform`_ coverage: you can specify different Python interpreters
-  or different platforms and run tests in parallel on all of them.
-
-  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
-  program source code to the remote place.
-  You may specify different Python versions and interpreters. It does not
-  installs/synchronize dependencies however.
-
-  **Note**: this mode exists mostly for backward compatibility, as modern development
-  relies on continuous integration for multi-platform testing.
-
-.. _parallelization:
-
-Running tests across multiple CPUs
-----------------------------------
-
-To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
-
-    pytest -n 8
-
-Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
-can lead to considerable speed ups, especially if your test suite takes a
-noticeable amount of time.
-
-The test distribution algorithm is configured with the ``--dist`` command-line option:
-
-.. _distribution modes:
-
-* ``--dist load`` **(default)**: Sends pending tests to any worker that is
-  available, without any guaranteed order.
-
-* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
-  and by **class** for *test methods*. Groups are distributed to available
-  workers as whole units. This guarantees that all tests in a group run in the
-  same process. This can be useful if you have expensive module-level or
-  class-level fixtures. Grouping by class takes priority over grouping by
-  module.
-
-* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
-  distributed to available workers as whole units. This guarantees that all
-  tests in a file run in the same worker.
-
-* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
-  distributed to available workers as whole units. This guarantees that all
-  tests with same ``xdist_group`` name run in the same worker.
-
-  .. code-block:: python
-
-      @pytest.mark.xdist_group(name="group1")
-      def test1():
-          pass
-
-      class TestA:
-          @pytest.mark.xdist_group("group1")
-          def test2():
-              pass
-
-  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
-  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
-
-* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).
-
-
-Running tests in a Python subprocess
-------------------------------------
-
-To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
-
-    pytest -d --tx popen//python=python3.9
-
-This will start a subprocess which is run with the ``python3.9``
-Python interpreter, found in your system binary lookup path.
-
-If you prefix the --tx option value like this::
-
-    --tx 3*popen//python=python3.9
-
-then three subprocesses would be created and tests
-will be load-balanced across these three processes.
-
-.. _boxed:
-
-Running tests in a boxed subprocess
------------------------------------
-
-This functionality has been moved to the
-`pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ plugin, but the ``--boxed`` option
-is still kept for backward compatibility.
-
-.. _`remote machines`:
-
-Sending tests to remote SSH accounts
-------------------------------------
-
-Suppose you have a package ``mypkg`` which contains some
-tests that you can successfully run locally. And you
-have a ssh-reachable machine ``myhost``.  Then
-you can ad-hoc distribute your tests by typing::
-
-    pytest -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
-
-This will synchronize your :code:`mypkg` package directory
-to a remote ssh account and then locally collect tests
-and send them to remote places for execution.
-
-You can specify multiple :code:`--rsyncdir` directories
-to be sent to the remote side.
-
-.. note::
-
-  For pytest to collect and send tests correctly
-  you not only need to make sure all code and tests
-  directories are rsynced, but that any test (sub) directory
-  also has an :code:`__init__.py` file because internally
-  pytest references tests as a fully qualified python
-  module path.  **You will otherwise get strange errors**
-  during setup of the remote side.
-
-
-You can specify multiple :code:`--rsyncignore` glob patterns
-to be ignored when file are sent to the remote side.
-There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
-Those you cannot override using rsyncignore command-line or
-ini-file option(s).
-
-
-Sending tests to remote Socket Servers
---------------------------------------
-
-Download the single-module `socketserver.py`_ Python program
-and run it like this::
-
-    python socketserver.py
-
-It will tell you that it starts listening on the default
-port.  You can now on your home machine specify this
-new socket host with something like this::
-
-    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
-
-
-.. _`atonce`:
-.. _`Multi-Platform`:
-
-
-Running tests on many platforms at once
----------------------------------------
-
-The basic command to run tests on multiple platforms is::
-
-    pytest --dist=each --tx=spec1 --tx=spec2
-
-If you specify a windows host, an OSX host and a Linux
-environment this command will send each tests to all
-platforms - and report back failures from all platforms
-at once. The specifications strings use the `xspec syntax`_.
-
-.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
-
-.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
-
-.. _`execnet`: https://codespeak.net/execnet
-
-
-When tests crash
-----------------
-
-If a test crashes a worker, pytest-xdist will automatically restart that worker
-and report the test’s failure. You can use the ``--max-worker-restart`` option
-to limit the number of worker restarts that are allowed, or disable restarting
-altogether using ``--max-worker-restart 0``.
-
-
-How-tos
--------
-
-Identifying the worker process during a test
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 1.15.*
-
-If you need to determine the identity of a worker process in
-a test or fixture, you may use the ``worker_id`` fixture to do so:
-
-.. code-block:: python
-
-    @pytest.fixture()
-    def user_account(worker_id):
-        """ use a different account in each xdist worker """
-        return "account_%s" % worker_id
-
-When ``xdist`` is disabled (running with ``-n0`` for example), then
-``worker_id`` will return ``"master"``.
-
-Worker processes also have the following environment variables
-defined:
-
-* ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
-* ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session,
-  e.g., ``"4"`` when ``-n 4`` is given in the command-line.
-
-The information about the worker_id in a test is stored in the ``TestReport`` as
-well, under the ``worker_id`` attribute.
-
-Since version 2.0, the following functions are also available in the ``xdist`` module:
-
-.. code-block:: python
-
-    def is_xdist_worker(request_or_session) -> bool:
-        """Return `True` if this is an xdist worker, `False` otherwise
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-     def is_xdist_controller(request_or_session) -> bool:
-        """Return `True` if this is the xdist controller, `False` otherwise
-
-        Note: this method also returns `False` when distribution has not been
-        activated at all.
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-    def is_xdist_master(request_or_session) -> bool:
-        """Deprecated alias for is_xdist_controller."""
-
-    def get_xdist_worker_id(request_or_session) -> str:
-        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-        if running on the controller node.
-
-        If not distributing tests (for example passing `-n0` or not passing `-n` at all)
-        also return 'master'.
-
-        :param request_or_session: the `pytest` `request` or `session` object
-        """
-
-
-Identifying workers from the system environment
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 2.4*
-
-If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
-update the process title (command line) on its workers to show their current
-state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
-``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
-Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
-pointer regarding the Process Explorer tool.
-
-This is intended purely as an UX enhancement, e.g. to track down issues with
-long-running or CPU intensive tests.  Errors in changing the title are ignored
-silently.  Please try not to rely on the title format or title changes in
-external scripts.
-
-.. _`setproctitle`: https://pypi.org/project/setproctitle/
-
-
-Uniquely identifying the current test run
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-*New in version 1.32.*
-
-If you need to globally distinguish one test run from others in your
-workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
-wanted to create a separate database for each test run:
-
-.. code-block:: python
-
-    import pytest
-    from posix_ipc import Semaphore, O_CREAT
-
-    @pytest.fixture(scope="session", autouse=True)
-    def create_unique_database(testrun_uid):
-        """ create a unique database for this particular test run """
-        database_url = f"psql://myapp-{testrun_uid}"
-
-        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
-            if not database_exists(database_url):
-                create_database(database_url)
-
-    @pytest.fixture()
-    def db(testrun_uid):
-        """ retrieve unique database """
-        database_url = f"psql://myapp-{testrun_uid}"
-        return database_get_instance(database_url)
-
-
-Additionally, during a test run, the following environment variable is defined:
-
-* ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
-
-Accessing ``sys.argv`` from the controller node in workers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To access the ``sys.argv`` passed to the command-line of the controller node, use
-``request.config.workerinput["mainargv"]``.
-
-
-Specifying test exec environments in an ini file
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-You can use pytest's ini file configuration to avoid typing common options.
-You can for example make running with three subprocesses your default like this:
-
-.. code-block:: ini
-
-    [pytest]
-    addopts = -n3
-
-You can also add default environments like this:
-
-.. code-block:: ini
-
-    [pytest]
-    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
-
-and then just type::
-
-    pytest --dist=each
-
-to run tests in each of the environments.
-
-
-Specifying "rsync" dirs in an ini-file
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
-you may specify directories to include or to exclude in synchronisation:
-
-.. code-block:: ini
-
-    [pytest]
-    rsyncdirs = . mypkg helperpkg
-    rsyncignore = .hg
-
-These directory specifications are relative to the directory
-where the configuration file was found.
-
-.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
-.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
-.. _`pytest`: http://pytest.org
-
-
-Making session-scoped fixtures execute only once
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
-a subset of all tests. This means that tests in different processes requesting a high-level
-scoped fixture (for example ``session``) will execute the fixture code more than once, which
-breaks expectations and might be undesired in certain situations.
-
-While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
-executed exactly once, this can be achieved by using a lock file for inter-process communication.
-
-The example below needs to execute the fixture ``session_data`` only once (because it is
-resource intensive, or needs to execute only once to define configuration options, etc), so it makes
-use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
-when the first process requests the fixture, while the other processes will then read
-the data from a file.
-
-Here is the code:
-
-.. code-block:: python
-
-    import json
-
-    import pytest
-    from filelock import FileLock
-
-
-    @pytest.fixture(scope="session")
-    def session_data(tmp_path_factory, worker_id):
-        if worker_id == "master":
-            # not executing in with multiple workers, just produce the data and let
-            # pytest's fixture caching do its job
-            return produce_expensive_data()
-
-        # get the temp directory shared by all workers
-        root_tmp_dir = tmp_path_factory.getbasetemp().parent
-
-        fn = root_tmp_dir / "data.json"
-        with FileLock(str(fn) + ".lock"):
-            if fn.is_file():
-                data = json.loads(fn.read_text())
-            else:
-                data = produce_expensive_data()
-                fn.write_text(json.dumps(data))
-        return data
-
-
-The example above can also be use in cases a fixture needs to execute exactly once per test session, like
-initializing a database service and populating initial tables.
-
-This technique might not work for every case, but should be a starting point for many situations
-where executing a high-scope fixture exactly once is important.
-
-
-How does xdist work?
---------------------
-
-``xdist`` works by spawning one or more **workers**, which are
-controlled by the **controller**. Each **worker** is responsible for
-performing a full test collection and afterwards running tests as
-dictated by the **controller**.
-
-The execution flow is:
-
-1. **controller** spawns one or more **workers** at the beginning of the
-   test session. The communication between **controller** and **worker**
-   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
-   its
-   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
-   The actual interpreters executing the code for the **workers** might
-   be remote or local.
-
-2. Each **worker** itself is a mini pytest runner. **workers** at this
-   point perform a full test collection, sending back the collected
-   test-ids back to the **controller** which does not perform any
-   collection itself.
-
-3. The **controller** receives the result of the collection from all
-   nodes. At this point the **controller** performs some sanity check to
-   ensure that all **workers** collected the same tests (including
-   order), bailing out otherwise. If all is well, it converts the list
-   of test-ids into a list of simple indexes, where each index
-   corresponds to the position of that test in the original collection
-   list. This works because all nodes have the same collection list, and
-   saves bandwidth because the **controller** can now tell one of the
-   workers to just *execute test index 3* index of passing the full test
-   id.
-
-4. If **dist-mode** is **each**: the **controller** just sends the full
-   list of test indexes to each node at this moment.
-
-5. If **dist-mode** is **load**: the **controller** takes around 25% of
-   the tests and sends them one by one to each **worker** in a round
-   robin fashion. The rest of the tests will be distributed later as
-   **workers** finish tests (see below).
-
-6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
-   implement custom tests distribution logic.
-
-7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
-   implementation basically loops over all collected items in the
-   ``session`` object and executes the ``pytest_runtest_protocol`` for
-   each test item, but in xdist **workers** sit idly waiting for
-   **controller** to send tests for execution. As tests are received by
-   **workers**, ``pytest_runtest_protocol`` is executed for each test.
-   Here it worth noting an implementation detail: **workers** always
-   must keep at least one test item on their queue due to how the
-   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
-   to pass the ``nextitem`` to the hook, the worker must wait for more
-   instructions from controller before executing that remaining test. If
-   it receives more tests, then it can safely call
-   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
-   parameter will be. If it receives a “shutdown” signal, then it can
-   execute the hook passing ``nextitem`` as ``None``.
-
-8. As tests are started and completed at the **workers**, the results
-   are sent back to the **controller**, which then just forwards the
-   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
-   and ``pytest_runtest_logreport``. This way other plugins (for example
-   ``junitxml``) can work normally. The **controller** (when in
-   dist-mode **load**) decides to send more tests to a node when a test
-   completes, using some heuristics such as test durations and how many
-   tests each **worker** still has to run.
-
-9. When the **controller** has no more pending tests it will send a
-   “shutdown” signal to all **workers**, which will then run their
-   remaining tests to completion and shut down. At this point the
-   **controller** will sit waiting for **workers** to shut down, still
-   processing events such as ``pytest_runtest_logreport``.
-
-FAQ
----
-
-**Question**: Why does each worker do its own collection, as opposed to having the
-controller collect once and distribute from that collection to the
-workers?
-
-If collection was performed by controller then it would have to
-serialize collected items to send them through the wire, as workers live
-in another process. The problem is that test items are not easily
-(impossible?) to serialize, as they contain references to the test
-functions, fixture managers, config objects, etc. Even if one manages to
-serialize it, it seems it would be very hard to get it right and easy to
-break by any small change in pytest.

changelog/468.deprecation.rst

@@ -1,3 +0,0 @@
-The ``--boxed`` commmand line argument is deprecated.
-Install pytest-forked and use ``--forked`` instead.
-pytest-xdist 3.0.0 will remove the ``--boxed`` argument and pytest-forked dependency.

docs/.gitignore

@@ -0,0 +1 @@
+_build/

docs/changelog.rst

@@ -0,0 +1,5 @@
+=========
+Changelog
+=========
+
+.. include:: ../CHANGELOG.rst

docs/conf.py

@@ -0,0 +1,55 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = "pytest-xdist"
+copyright = "2022, holger krekel and contributors"
+author = "holger krekel and contributors"
+
+master_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx_rtd_theme",
+    "sphinx.ext.autodoc",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = ['_static']

docs/crash.rst

@@ -0,0 +1,7 @@
+When tests crash
+================
+
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.

docs/distribution.rst

@@ -0,0 +1,85 @@
+.. _parallelization:
+
+Running tests across multiple CPUs
+==================================
+
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
+
+    pytest -n auto
+
+This can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
+
+With ``-n auto``, pytest-xdist will use as many processes as your computer
+has CPU cores.
+
+Use ``-n logical`` to use the number of *logical* CPU cores rather than
+physical ones. This currently requires the ``psutils`` package to be installed;
+if it is not, pytest-xdist will fall back to ``-n auto`` behavior.
+
+Pass a number, e.g. ``-n 8``, to specify the number of processes explicitly.
+
+To specify a different meaning for ``-n auto`` and ``-n logical`` for your
+tests, you can:
+
+* Set the environment variable ``PYTEST_XDIST_AUTO_NUM_WORKERS`` to the
+  desired number of processes.
+
+* Implement the ``pytest_xdist_auto_num_workers``
+  `pytest hook <https://docs.pytest.org/en/latest/how-to/writing_plugins.html>`__
+  (a ``pytest_xdist_auto_num_workers(config)`` function in e.g. ``conftest.py``)
+  that returns the number of processes to use.
+  The hook can use ``config.option.numprocesses`` to determine if the user
+  asked for ``"auto"`` or ``"logical"``, and it can return ``None`` to fall
+  back to the default.
+
+If both the hook and environment variable are specified, the hook takes
+priority.
+
+
+Parallelization can be configured further with these options:
+
+* ``--maxprocesses=maxprocesses``: limit the maximum number of workers to
+  process the tests.
+
+* ``--max-worker-restart``: maximum number of workers that can be restarted
+  when crashed (set to zero to disable this feature).
+
+The test distribution algorithm is configured with the ``--dist`` command-line option:
+
+.. _distribution modes:
+
+* ``--dist load`` **(default)**: Sends pending tests to any worker that is
+  available, without any guaranteed order.
+
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
+
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+* ``--dist loadgroup``: Tests are grouped by the ``xdist_group`` mark. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests with same ``xdist_group`` name run in the same worker.
+
+  .. code-block:: python
+
+      @pytest.mark.xdist_group(name="group1")
+      def test1():
+          pass
+
+
+      class TestA:
+          @pytest.mark.xdist_group("group1")
+          def test2():
+              pass
+
+  This will make sure ``test1`` and ``TestA::test2`` will run in the same worker.
+  Tests without the ``xdist_group`` mark are distributed normally as in the ``--dist=load`` mode.
+
+* ``--dist no``: The normal pytest execution mode, runs one test at a time (no distribution at all).

docs/how-it-works.rst

@@ -0,0 +1,90 @@
+How it works?
+=============
+
+``xdist`` works by spawning one or more **workers**, which are
+controlled by the **controller**. Each **worker** is responsible for
+performing a full test collection and afterwards running tests as
+dictated by the **controller**.
+
+The execution flow is:
+
+1. **controller** spawns one or more **workers** at the beginning of the
+   test session. The communication between **controller** and **worker**
+   nodes makes use of `execnet <https://codespeak.net/execnet/>`__ and
+   its
+   `gateways <https://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters>`__.
+   The actual interpreters executing the code for the **workers** might
+   be remote or local.
+
+2. Each **worker** itself is a mini pytest runner. **workers** at this
+   point perform a full test collection, sending back the collected
+   test-ids back to the **controller** which does not perform any
+   collection itself.
+
+3. The **controller** receives the result of the collection from all
+   nodes. At this point the **controller** performs some sanity check to
+   ensure that all **workers** collected the same tests (including
+   order), bailing out otherwise. If all is well, it converts the list
+   of test-ids into a list of simple indexes, where each index
+   corresponds to the position of that test in the original collection
+   list. This works because all nodes have the same collection list, and
+   saves bandwidth because the **controller** can now tell one of the
+   workers to just *execute test index 3* index of passing the full test
+   id.
+
+4. If **dist-mode** is **each**: the **controller** just sends the full
+   list of test indexes to each node at this moment.
+
+5. If **dist-mode** is **load**: the **controller** takes around 25% of
+   the tests and sends them one by one to each **worker** in a round
+   robin fashion. The rest of the tests will be distributed later as
+   **workers** finish tests (see below).
+
+6. Note that ``pytest_xdist_make_scheduler`` hook can be used to
+   implement custom tests distribution logic.
+
+7. **workers** re-implement ``pytest_runtestloop``: pytest’s default
+   implementation basically loops over all collected items in the
+   ``session`` object and executes the ``pytest_runtest_protocol`` for
+   each test item, but in xdist **workers** sit idly waiting for
+   **controller** to send tests for execution. As tests are received by
+   **workers**, ``pytest_runtest_protocol`` is executed for each test.
+   Here it worth noting an implementation detail: **workers** always
+   must keep at least one test item on their queue due to how the
+   ``pytest_runtest_protocol(item, nextitem)`` hook is defined: in order
+   to pass the ``nextitem`` to the hook, the worker must wait for more
+   instructions from controller before executing that remaining test. If
+   it receives more tests, then it can safely call
+   ``pytest_runtest_protocol`` because it knows what the ``nextitem``
+   parameter will be. If it receives a “shutdown” signal, then it can
+   execute the hook passing ``nextitem`` as ``None``.
+
+8. As tests are started and completed at the **workers**, the results
+   are sent back to the **controller**, which then just forwards the
+   results to the appropriate pytest hooks: ``pytest_runtest_logstart``
+   and ``pytest_runtest_logreport``. This way other plugins (for example
+   ``junitxml``) can work normally. The **controller** (when in
+   dist-mode **load**) decides to send more tests to a node when a test
+   completes, using some heuristics such as test durations and how many
+   tests each **worker** still has to run.
+
+9. When the **controller** has no more pending tests it will send a
+   “shutdown” signal to all **workers**, which will then run their
+   remaining tests to completion and shut down. At this point the
+   **controller** will sit waiting for **workers** to shut down, still
+   processing events such as ``pytest_runtest_logreport``.
+
+FAQ
+---
+
+**Question**: Why does each worker do its own collection, as opposed to having the
+controller collect once and distribute from that collection to the
+workers?
+
+If collection was performed by controller then it would have to
+serialize collected items to send them through the wire, as workers live
+in another process. The problem is that test items are not easily
+(impossible?) to serialize, as they contain references to the test
+functions, fixture managers, config objects, etc. Even if one manages to
+serialize it, it seems it would be very hard to get it right and easy to
+break by any small change in pytest.

docs/how-to.rst

@@ -0,0 +1,233 @@
+How-tos
+-------
+
+This section show cases how to accomplish some specialized tasks with ``pytest-xdist``.
+
+Identifying the worker process during a test
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.15.*
+
+If you need to determine the identity of a worker process in
+a test or fixture, you may use the ``worker_id`` fixture to do so:
+
+.. code-block:: python
+
+    @pytest.fixture()
+    def user_account(worker_id):
+        """ use a different account in each xdist worker """
+        return "account_%s" % worker_id
+
+When ``xdist`` is disabled (running with ``-n0`` for example), then
+``worker_id`` will return ``"master"``.
+
+Worker processes also have the following environment variables
+defined:
+
+.. envvar:: PYTEST_XDIST_WORKER
+
+The name of the worker, e.g., ``"gw2"``.
+
+.. envvar:: PYTEST_XDIST_WORKER_COUNT
+
+The total number of workers in this session, e.g., ``"4"`` when ``-n 4`` is given in the command-line.
+
+The information about the worker_id in a test is stored in the ``TestReport`` as
+well, under the ``worker_id`` attribute.
+
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+
+.. autofunction:: xdist.is_xdist_worker
+.. autofunction:: xdist.is_xdist_controller
+.. autofunction:: xdist.is_xdist_master
+.. autofunction:: xdist.get_xdist_worker_id
+
+Identifying workers from the system environment
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 2.4*
+
+If the `setproctitle`_ package is installed, ``pytest-xdist`` will use it to
+update the process title (command line) on its workers to show their current
+state.  The titles used are ``[pytest-xdist running] file.py/node::id`` and
+``[pytest-xdist idle]``, visible in standard tools like ``ps`` and ``top`` on
+Linux, Mac OS X and BSD systems.  For Windows, please follow `setproctitle`_'s
+pointer regarding the Process Explorer tool.
+
+This is intended purely as an UX enhancement, e.g. to track down issues with
+long-running or CPU intensive tests.  Errors in changing the title are ignored
+silently.  Please try not to rely on the title format or title changes in
+external scripts.
+
+.. _`setproctitle`: https://pypi.org/project/setproctitle/
+
+
+Uniquely identifying the current test run
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """ create a unique database for this particular test run """
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """ retrieve unique database """
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+.. envvar:: PYTEST_XDIST_TESTRUNUID
+
+The unique id of the test run.
+
+Accessing ``sys.argv`` from the controller node in workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To access the ``sys.argv`` passed to the command-line of the controller node, use
+``request.config.workerinput["mainargv"]``.
+
+
+Specifying test exec environments in an ini file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can use pytest's ini file configuration to avoid typing common options.
+You can for example make running with three subprocesses your default like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = -n3
+
+You can also add default environments like this:
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = --tx ssh=myhost//python=python3.9 --tx ssh=myhost//python=python3.6
+
+and then just type::
+
+    pytest --dist=each
+
+to run tests in each of the environments.
+
+
+Specifying "rsync" dirs in an ini-file
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
+you may specify directories to include or to exclude in synchronisation:
+
+.. code-block:: ini
+
+    [pytest]
+    rsyncdirs = . mypkg helperpkg
+    rsyncignore = .hg
+
+These directory specifications are relative to the directory
+where the configuration file was found.
+
+.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
+.. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
+.. _`pytest`: http://pytest.org
+
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
+
+
+Creating one log file for each worker
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create one log file for each worker with ``pytest-xdist``, you can leverage :envvar:`PYTEST_XDIST_WORKER`
+to generate a unique filename for each worker.
+
+Example:
+
+.. code-block:: python
+
+    # content of conftest.py
+    def pytest_configure(config):
+        worker_id = os.environ.get("PYTEST_XDIST_WORKER")
+        if worker_id is not None:
+            log_file = config.getini("worker_log_file")
+            logging.basicConfig(
+                format=config.getini("log_file_format"),
+                filename=f"tests_{worker_id}.log",
+                level=config.getini("log_file_level"),
+            )
+
+
+When running the tests with ``-n3``, for example, three files will be created in the current directory:
+``tests_gw0.log``, ``tests_gw1.log`` and ``tests_gw2.log``.

docs/index.rst

@@ -0,0 +1,63 @@
+pytest-xdist
+============
+
+The `pytest-xdist`_ plugin extends pytest with new test execution modes, the most used being distributing
+tests across multiple CPUs to speed up test execution::
+
+    pytest -n auto
+
+With this call, pytest will spawn a number of workers processes equal to the number of available CPUs, and distribute
+the tests randomly across them.
+
+.. note::
+    Due to how pytest-xdist is implemented, the ``-s/--capture=no`` option does not work.
+
+
+Installation
+------------
+
+Install the plugin with::
+
+    pip install pytest-xdist
+
+
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
+Features
+--------
+
+* Test run :ref:`parallelization`: tests can be executed across  multiple CPUs or hosts.
+  This allows to speed up development or to use special resources of :ref:`remote machines`.
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
+  pytest waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed (DEPRECATED).
+
+* :ref:`Multi-Platform` coverage: you can specify different Python interpreters
+  or different platforms and run tests in parallel on all of them.
+
+  Before running tests remotely, ``pytest`` efficiently "rsyncs" your
+  program source code to the remote place.
+  You may specify different Python versions and interpreters. It does not
+  installs/synchronize dependencies however.
+
+  **Note**: this mode exists mostly for backward compatibility, as modern development
+  relies on continuous integration for multi-platform testing.
+
+
+
+.. toctree::
+    :maxdepth: 2
+    :caption: Contents:
+
+    distribution
+    subprocess
+    remote
+    crash
+    how-to
+    how-it-works
+    known-limitations
+    changelog

docs/known-limitations.rst

@@ -0,0 +1,64 @@
+Known limitations
+=================
+
+pytest-xdist has some limitations that may be supported in pytest but can't be supported in pytest-xdist.
+
+Order and amount of test must be consistent
+-------------------------------------------
+
+Is is not possible to have tests that differ in order or their amount across workers.
+
+This is especially true with ``pytest.mark.parametrize``, when values are produced with sets or other unordered iterables/generators.
+
+
+Example:
+
+.. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", {"a", "b"})
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+In the example above, the fact that ``set`` are not necessarily ordered can cause different workers
+to collect tests in different order, which will throw an error.
+
+Workarounds
+~~~~~~~~~~~
+
+A solution to this is to guarantee that the parametrized values have the same order.
+
+Some solutions:
+
+* Convert your sequence to a ``list``.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", ["a", "b"])
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+* Sort your sequence, guaranteeing order.
+
+  .. code-block:: python
+
+    import pytest
+
+
+    @pytest.mark.parametrize("param", sorted({"a", "b"}))
+    def test_pytest_parametrize_unordered(param):
+        pass
+
+Output (stdout and stderr) from workers
+---------------------------------------
+
+The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time.
+
+However this option does not work with ``pytest-xdist`` because `execnet <https://github.com/pytest-dev/execnet>`__ the underlying library used for communication between master and workers, does not support transferring stdout/stderr from workers.
+
+Currenlty there are no plans ot support this in ``pytest-xdist``.

docs/remote.rst

@@ -0,0 +1,83 @@
+
+.. _`Multi-Platform`:
+.. _`remote machines`:
+
+Sending tests to remote SSH accounts
+====================================
+
+.. deprecated:: 3.0
+
+.. warning::
+
+    This feature is deprecated because the support for ``rsync`` is faulty
+    in terms of reproducing the development environment in the remote
+    worker, and there is no clear solution moving forward.
+
+    This feature is scheduled to be removed in release 4.0, to let the team
+    focus on a smaller set of features.
+
+Suppose you have a package ``mypkg`` which contains some
+tests that you can successfully run locally. And you
+have a ssh-reachable machine ``myhost``.  Then
+you can ad-hoc distribute your tests by typing::
+
+    pytest -d  --rsyncdir mypkg --tx ssh=myhostpopen mypkg/tests/unit/test_something.py
+
+This will synchronize your :code:`mypkg` package directory
+to a remote ssh account and then locally collect tests
+and send them to remote places for execution.
+
+You can specify multiple :code:`--rsyncdir` directories
+to be sent to the remote side.
+
+.. note::
+
+  For pytest to collect and send tests correctly
+  you not only need to make sure all code and tests
+  directories are rsynced, but that any test (sub) directory
+  also has an :code:`__init__.py` file because internally
+  pytest references tests as a fully qualified python
+  module path.  **You will otherwise get strange errors**
+  during setup of the remote side.
+
+
+You can specify multiple :code:`--rsyncignore` glob patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: :code:`.*, *.pyc, *.pyo, *~`
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
+Sending tests to remote Socket Servers
+--------------------------------------
+
+Download the single-module `socketserver.py`_ Python program
+and run it like this::
+
+    python socketserver.py
+
+It will tell you that it starts listening on the default
+port.  You can now on your home machine specify this
+new socket host with something like this::
+
+    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg
+
+
+
+Running tests on many platforms at once
+---------------------------------------
+
+The basic command to run tests on multiple platforms is::
+
+    pytest --dist=each --tx=spec1 --tx=spec2
+
+If you specify a windows host, an OSX host and a Linux
+environment this command will send each tests to all
+platforms - and report back failures from all platforms
+at once. The specifications strings use the `xspec syntax`_.
+
+.. _`xspec syntax`: https://codespeak.net/execnet/basics.html#xspec
+
+.. _`execnet`: https://codespeak.net/execnet
+
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py

docs/subprocess.rst

@@ -0,0 +1,16 @@
+Running tests in a Python subprocess
+====================================
+
+To instantiate a ``python3.9`` subprocess and send tests to it, you may type::
+
+    pytest -d --tx popen//python=python3.9
+
+This will start a subprocess which is run with the ``python3.9``
+Python interpreter, found in your system binary lookup path.
+
+If you prefix the --tx option value like this::
+
+    --tx 3*popen//python=python3.9
+
+then three subprocesses would be created and tests
+will be load-balanced across these three processes.

pyproject.toml

@@ -1,6 +1,5 @@
 [build-system]
 requires = [
-  # sync with setup.py until we discard non-pep-517/518
   "setuptools>=45.0",
   "setuptools-scm[toml]>=6.2.3",
   "wheel",
@@ -17,32 +16,23 @@ directory = "changelog/"
 title_format = "pytest-xdist {version} ({project_date})"
 template = "changelog/_template.rst"
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.removal]
-  directory = "removal"
+name = "Removals"
-  name = "Deprecations and Removals"
-  showcontent = true
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.deprecation]
-  directory = "feature"
+name = "Deprecations"
+
+[tool.towncrier.fragment.feature]
 name = "Features"
-  showcontent = true
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.bugfix]
-  directory = "bugfix"
 name = "Bug Fixes"
-  showcontent = true
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.vendor]
-  directory = "vendor"
 name = "Vendored Libraries"
-  showcontent = true
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.doc]
-  directory = "doc"
 name = "Improved Documentation"
-  showcontent = true
 
-  [[tool.towncrier.type]]
+[tool.towncrier.fragment.trivial]
-  directory = "trivial"
 name = "Trivial Changes"
-  showcontent = true

setup.cfg

@@ -30,6 +30,11 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
 license_file = LICENSE
+project_urls =
+    Documentation=https://pytest-xdist.readthedocs.io/en/latest
+    Changelog=https://pytest-xdist.readthedocs.io/en/latest/changelog.html
+    Source=https://github.com/pytest-dev/pytest-xdist
+    Tracker=https://github.com/pytest-dev/pytest-xdist/issues
 
 [options]
 packages = find:
@@ -39,8 +44,6 @@ python_requires = >=3.6
 install_requires =
     execnet>=1.1
     pytest>=6.2.0
-    pytest-forked
-setup_requires = setuptools_scm>=6.0
 
 [options.packages.find]
 where = src

setup.py

@@ -1,4 +0,0 @@
-from setuptools import setup
-
-if __name__ == "__main__":
-    setup()

src/xdist/_path.py

@@ -0,0 +1,19 @@
+import os
+from itertools import chain
+from pathlib import Path
+from typing import Callable, Iterator
+
+
+def visit_path(
+    path: Path, *, filter: Callable[[Path], bool], recurse: Callable[[Path], bool]
+) -> Iterator[Path]:
+    """
+    Implements the interface of ``py.path.local.visit()`` for Path objects,
+    to simplify porting the code over from ``py.path.local``.
+    """
+    for dirpath, dirnames, filenames in os.walk(path):
+        dirnames[:] = [x for x in dirnames if recurse(Path(dirpath, x))]
+        for name in chain(dirnames, filenames):
+            p = Path(dirpath, name)
+            if filter(p):
+                yield p

src/xdist/dsession.py

@@ -1,6 +1,6 @@
-import py
 import pytest
 
+from xdist.remote import Producer
 from xdist.workermanage import NodeManager
 from xdist.scheduler import (
     EachScheduling,
@@ -34,9 +34,7 @@ class DSession:
 
     def __init__(self, config):
         self.config = config
-        self.log = py.log.Producer("dsession")
+        self.log = Producer("dsession", enabled=config.option.debug)
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
         self.nodemanager = None
         self.sched = None
         self.shuttingdown = False
@@ -225,6 +223,7 @@ class DSession:
             self.triggershutdown()
         else:
             self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
+            self.shuttingdown = False
             self._clone_node(node)
         self._active_nodes.remove(node)
 

src/xdist/looponfail.py

@@ -6,11 +6,17 @@
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-import py
+import os
+from pathlib import Path
+from typing import Dict, Sequence
+
 import pytest
 import sys
 import time
 import execnet
+from _pytest._io import TerminalWriter
+
+from xdist._path import visit_path
 
 
 @pytest.hookimpl
@@ -38,9 +44,12 @@ def pytest_cmdline_main(config):
         return 2  # looponfail only can get stop with ctrl-C anyway
 
 
-def looponfail_main(config):
+def looponfail_main(config: "pytest.Config") -> None:
     remotecontrol = RemoteControl(config)
-    rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")]
+    config_roots = config.getini("looponfailroots")
+    if not config_roots:
+        config_roots = [Path.cwd()]
+    rootdirs = [Path(root) for root in config_roots]
     statrecorder = StatRecorder(rootdirs)
     try:
         while 1:
@@ -71,7 +80,7 @@ class RemoteControl:
 
     def setup(self, out=None):
         if out is None:
-            out = py.io.TerminalWriter()
+            out = TerminalWriter()
         if hasattr(self, "gateway"):
             raise ValueError("already have gateway %r" % self.gateway)
         self.trace("setting up worker session")
@@ -129,7 +138,7 @@ class RemoteControl:
 
 
 def repr_pytest_looponfailinfo(failreports, rootdirs):
-    tr = py.io.TerminalWriter()
+    tr = TerminalWriter()
     if failreports:
         tr.sep("#", "LOOPONFAILING", bold=True)
         for report in failreports:
@@ -225,16 +234,16 @@ class WorkerFailSession:
 
 
 class StatRecorder:
-    def __init__(self, rootdirlist):
+    def __init__(self, rootdirlist: Sequence[Path]) -> None:
         self.rootdirlist = rootdirlist
-        self.statcache = {}
+        self.statcache: Dict[Path, os.stat_result] = {}
         self.check()  # snapshot state
 
-    def fil(self, p):
+    def fil(self, p: Path) -> bool:
-        return p.check(file=1, dotfile=0) and p.ext != ".pyc"
+        return p.is_file() and not p.name.startswith(".") and p.suffix != ".pyc"
 
-    def rec(self, p):
+    def rec(self, p: Path) -> bool:
-        return p.check(dotfile=0)
+        return not p.name.startswith(".") and p.exists()
 
     def waitonchange(self, checkinterval=1.0):
         while 1:
@@ -243,34 +252,34 @@ class StatRecorder:
                 return
             time.sleep(checkinterval)
 
-    def check(self, removepycfiles=True):  # noqa, too complex
+    def check(self, removepycfiles: bool = True) -> bool:  # noqa, too complex
         changed = False
-        statcache = self.statcache
+        newstat: Dict[Path, os.stat_result] = {}
-        newstat = {}
         for rootdir in self.rootdirlist:
-            for path in rootdir.visit(self.fil, self.rec):
+            for path in visit_path(rootdir, filter=self.fil, recurse=self.rec):
-                oldstat = statcache.pop(path, None)
+                oldstat = self.statcache.pop(path, None)
                 try:
-                    newstat[path] = curstat = path.stat()
+                    curstat = path.stat()
-                except py.error.ENOENT:
+                except OSError:
                     if oldstat:
                         changed = True
                 else:
-                    if oldstat:
+                    newstat[path] = curstat
+                    if oldstat is not None:
                         if (
-                            oldstat.mtime != curstat.mtime
+                            oldstat.st_mtime != curstat.st_mtime
-                            or oldstat.size != curstat.size
+                            or oldstat.st_size != curstat.st_size
                         ):
                             changed = True
                             print("# MODIFIED", path)
-                            if removepycfiles and path.ext == ".py":
+                            if removepycfiles and path.suffix == ".py":
-                                pycfile = path + "c"
+                                pycfile = path.with_suffix(".pyc")
-                                if pycfile.check():
+                                if pycfile.is_file():
-                                    pycfile.remove()
+                                    os.unlink(pycfile)
 
                     else:
                         changed = True
-        if statcache:
+        if self.statcache:
             changed = True
         self.statcache = newstat
         return changed

src/xdist/newhooks.py

@@ -24,12 +24,16 @@ def pytest_xdist_newgateway(gateway):
     """called on new raw gateway creation."""
 
 
-@pytest.hookspec()
+@pytest.hookspec(
+    warn_on_impl="rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+)
 def pytest_xdist_rsyncstart(source, gateways):
     """called before rsyncing a directory to remote gateways takes place."""
 
 
-@pytest.hookspec()
+@pytest.hookspec(
+    warn_on_impl="rsync feature is deprecated and will be removed in pytest-xdist 4.0"
+)
 def pytest_xdist_rsyncfinish(source, gateways):
     """called after rsyncing a directory to remote gateways takes place."""
 

src/xdist/plugin.py

@@ -1,9 +1,8 @@
 import os
 import uuid
 import sys
-from pathlib import Path
+import warnings
 
-import py
 import pytest
 
 
@@ -14,6 +13,15 @@ _sys_path = list(sys.path)  # freeze a copy of sys.path at interpreter startup
 
 @pytest.hookimpl
 def pytest_xdist_auto_num_workers(config):
+    env_var = os.environ.get("PYTEST_XDIST_AUTO_NUM_WORKERS")
+    if env_var:
+        try:
+            return int(env_var)
+        except ValueError:
+            warnings.warn(
+                "PYTEST_XDIST_AUTO_NUM_WORKERS is not a number: {env_var!r}. Ignoring it."
+            )
+
     try:
         import psutil
     except ImportError:
@@ -135,11 +143,6 @@ def pytest_addoption(parser):
         metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.",
     )
-    group.addoption(
-        "--boxed",
-        action="store_true",
-        help="backward compatibility alias for pytest-forked --forked",
-    )
     group.addoption(
         "--testrunuid",
         action="store",
@@ -164,8 +167,7 @@ def pytest_addoption(parser):
     parser.addini(
         "looponfailroots",
         type="paths" if PYTEST_GTE_7 else "pathlist",
-        help="directories to check for changes",
+        help="directories to check for changes. Default: current directory.",
-        default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()],
     )
 
 
@@ -196,14 +198,20 @@ def pytest_configure(config):
         tr = config.pluginmanager.getplugin("terminalreporter")
         if tr:
             tr.showfspath = False
-    if config.getoption("boxed"):
+
+    if config.getoption("looponfail", None) or config.getini("looponfailroots"):
         warning = DeprecationWarning(
-            "The --boxed commmand line argument is deprecated. "
+            "The --looponfail command line argument and looponfailroots config variable are deprecated.\n"
-            "Install pytest-forked and use --forked instead. "
+            "The loop-on-fail feature will be removed in pytest-xdist 4.0."
-            "pytest-xdist 3.0.0 will remove the --boxed argument and pytest-forked dependency."
+        )
+        config.issue_config_time_warning(warning, 2)
+
+    if config.getoption("rsyncdir", None) or config.getini("rsyncdirs"):
+        warning = DeprecationWarning(
+            "The --rsyncdir command line argument and rsyncdirs config variable are deprecated.\n"
+            "The rsync feature will be removed in pytest-xdist 4.0."
         )
         config.issue_config_time_warning(warning, 2)
-        config.option.forked = True
 
     config_line = (
         "xdist_group: specify group for tests should run in same session."

src/xdist/remote.py

@@ -9,8 +9,8 @@
 import sys
 import os
 import time
+from typing import Any
 
-import py
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
@@ -24,6 +24,29 @@ except ImportError:
         pass
 
 
+class Producer:
+    """
+    Simplified implementation of the same interface as py.log, for backward compatibility
+    since we dropped the dependency on pylib.
+    Note: this is defined here because this module can't depend on xdist, so we need
+    to have the other way around.
+    """
+
+    def __init__(self, name: str, *, enabled: bool = True):
+        self.name = name
+        self.enabled = enabled
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}({self.name!r}, enabled={self.enabled})"
+
+    def __call__(self, *a: Any, **k: Any) -> None:
+        if self.enabled:
+            print(f"[{self.name}]", *a, **k, file=sys.stderr)
+
+    def __getattr__(self, name: str) -> "Producer":
+        return type(self)(name, enabled=self.enabled)
+
+
 def worker_title(title):
     try:
         setproctitle(title)
@@ -37,9 +60,7 @@ class WorkerInteractor:
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
         self.testrunuid = config.workerinput["testrunuid"]
-        self.log = py.log.Producer("worker-%s" % self.workerid)
+        self.log = Producer(f"worker-{self.workerid}", enabled=config.option.debug)
-        if not config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
         self.channel = channel
         config.pluginmanager.register(self)
 

src/xdist/report.py

@@ -14,7 +14,8 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
     error_message = (
         "Different tests were collected between {from_id} and {to_id}. "
         "The difference is:\n"
-        "{diff}"
+        "{diff}\n"
+        "To see why this happens see Known limitations in documentation"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
     msg = "\n".join(x.rstrip() for x in error_message.split("\n"))
     return msg

src/xdist/scheduler/each.py

@@ -1,5 +1,4 @@
-from py.log import Producer
+from xdist.remote import Producer
-
 from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 

src/xdist/scheduler/load.py

@@ -1,8 +1,8 @@
 from itertools import cycle
 
-from py.log import Producer
 from _pytest.runner import CollectReport
 
+from xdist.remote import Producer
 from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
@@ -248,13 +248,27 @@ class LoadScheduling:
         # Send a batch of tests to run. If we don't have at least two
         # tests per node, we have to send them all so that we can send
         # shutdown signals and get all nodes working.
-        initial_batch = max(len(self.pending) // 4, 2 * len(self.nodes))
+        if len(self.pending) < 2 * len(self.nodes):
-
+            # Distribute tests round-robin. Try to load all nodes if there are
-        # distribute tests round-robin up to the batch size
+            # enough tests. The other branch tries sends at least 2 tests
-        # (or until we run out)
+            # to each node - which is suboptimal when you have less than
+            # 2 * len(nodes) tests.
             nodes = cycle(self.nodes)
-        for i in range(initial_batch):
+            for i in range(len(self.pending)):
                 self._send_tests(next(nodes), 1)
+        else:
+            # Send batches of consecutive tests. By default, pytest sorts tests
+            # in order for optimal single-threaded execution, minimizing the
+            # number of necessary fixture setup/teardown. Try to keep that
+            # optimal order for every worker.
+
+            # how many items per node do we have about?
+            items_per_node = len(self.collection) // len(self.node2pending)
+            # take a fraction of tests for initial distribution
+            node_chunksize = max(items_per_node // 4, 2)
+            # and initialize each node with a chunk of tests
+            for node in self.nodes:
+                self._send_tests(node, node_chunksize)
 
         if not self.pending:
             # initial distribution sent all tests, start node shutdown

src/xdist/scheduler/loadfile.py

@@ -1,5 +1,5 @@
 from .loadscope import LoadScopeScheduling
-from py.log import Producer
+from xdist.remote import Producer
 
 
 class LoadFileScheduling(LoadScopeScheduling):

src/xdist/scheduler/loadgroup.py

@@ -1,5 +1,5 @@
 from .loadscope import LoadScopeScheduling
-from py.log import Producer
+from xdist.remote import Producer
 
 
 class LoadGroupScheduling(LoadScopeScheduling):

src/xdist/scheduler/loadscope.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 from _pytest.runner import CollectReport
-from py.log import Producer
+from xdist.remote import Producer
 from xdist.report import report_collection_diff
 from xdist.workermanage import parse_spec_config
 
@@ -361,12 +361,12 @@ class LoadScopeScheduling:
         extra_nodes = len(self.nodes) - len(self.workqueue)
 
         if extra_nodes > 0:
-            self.log("Shuting down {} nodes".format(extra_nodes))
+            self.log("Shutting down {} nodes".format(extra_nodes))
 
             for _ in range(extra_nodes):
                 unused_node, assigned = self.assigned_work.popitem(last=True)
 
-                self.log("Shuting down unused node {}".format(unused_node))
+                self.log("Shutting down unused node {}".format(unused_node))
                 unused_node.shutdown()
 
         # Assign initial workload

src/xdist/workermanage.py

@@ -3,12 +3,14 @@ import os
 import re
 import sys
 import uuid
+from pathlib import Path
+from typing import List, Union, Sequence, Optional, Any, Tuple, Set
 
-import py
 import pytest
 import execnet
 
 import xdist.remote
+from xdist.remote import Producer
 from xdist.plugin import _sys_path
 
 
@@ -33,7 +35,7 @@ class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
-    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
+    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache") -> None:
         self.config = config
         self.trace = self.config.trace.get("nodemanager")
         self.testrunuid = self.config.getoption("testrunuid")
@@ -52,7 +54,7 @@ class NodeManager:
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
         self.rsyncoptions = self._getrsyncoptions()
-        self._rsynced_specs = set()
+        self._rsynced_specs: Set[Tuple[Any, Any]] = set()
 
     def rsync_roots(self, gateway):
         """Rsync the set of roots to the node's gateway cwd."""
@@ -81,7 +83,7 @@ class NodeManager:
     def _getxspecs(self):
         return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
 
-    def _getrsyncdirs(self):
+    def _getrsyncdirs(self) -> List[Path]:
         for spec in self.specs:
             if not spec.popen or spec.chdir:
                 break
@@ -101,15 +103,15 @@ class NodeManager:
         pytestpath = get_dir(pytest.__file__)
         pytestdir = get_dir(_pytest.__file__)
         config = self.config
-        candidates = [py._pydir, pytestpath, pytestdir]
+        candidates = [pytestpath, pytestdir]
         candidates += config.option.rsyncdir
         rsyncroots = config.getini("rsyncdirs")
         if rsyncroots:
             candidates.extend(rsyncroots)
         roots = []
         for root in candidates:
-            root = py.path.local(root).realpath()
+            root = Path(root).resolve()
-            if not root.check():
+            if not root.exists():
                 raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root))
             if root not in roots:
                 roots.append(root)
@@ -160,18 +162,24 @@ class NodeManager:
 class HostRSync(execnet.RSync):
     """RSyncer that filters out common files"""
 
-    def __init__(self, sourcedir, *args, **kwargs):
+    PathLike = Union[str, "os.PathLike[str]"]
-        self._synced = {}
-        ignores = kwargs.pop("ignores", None) or []
-        self._ignores = [
-            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
-        ]
-        super().__init__(sourcedir=sourcedir, **kwargs)
 
-    def filter(self, path):
+    def __init__(
-        path = py.path.local(path)
+        self,
+        sourcedir: PathLike,
+        *,
+        ignores: Optional[Sequence[PathLike]] = None,
+        **kwargs: object
+    ) -> None:
+        if ignores is None:
+            ignores = []
+        self._ignores = [re.compile(fnmatch.translate(os.fspath(x))) for x in ignores]
+        super().__init__(sourcedir=Path(sourcedir), **kwargs)
+
+    def filter(self, path: PathLike) -> bool:
+        path = Path(path)
         for cre in self._ignores:
-            if cre.match(path.basename) or cre.match(path.strpath):
+            if cre.match(path.name) or cre.match(str(path)):
                 return False
         else:
             return True
@@ -187,20 +195,28 @@ class HostRSync(execnet.RSync):
             print("{}:{} <= {}".format(gateway.spec, remotepath, path))
 
 
-def make_reltoroot(roots, args):
+def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]:
     # XXX introduce/use public API for splitting pytest args
     splitcode = "::"
     result = []
     for arg in args:
         parts = arg.split(splitcode)
-        fspath = py.path.local(parts[0])
+        fspath = Path(parts[0])
-        if not fspath.exists():
+        try:
+            exists = fspath.exists()
+        except OSError:
+            exists = False
+        if not exists:
             result.append(arg)
             continue
         for root in roots:
-            x = fspath.relto(root)
+            x: Optional[Path]
+            try:
+                x = fspath.relative_to(root)
+            except ValueError:
+                x = None
             if x or fspath == root:
-                parts[0] = root.basename + "/" + x
+                parts[0] = root.name + "/" + str(x)
                 break
         else:
             raise ValueError("arg {} not relative to an rsync root".format(arg))
@@ -230,9 +246,7 @@ class WorkerController:
         }
         self._down = False
         self._shutdown_sent = False
-        self.log = py.log.Producer("workerctl-%s" % gateway.id)
+        self.log = Producer(f"workerctl-{gateway.id}", enabled=config.option.debug)
-        if not self.config.option.debug:
-            py.log.setconsumer(self.log._keywords, None)
 
     def __repr__(self):
         return "<{} {}>".format(self.__class__.__name__, self.gateway.id)

testing/acceptance_test.py

@@ -154,7 +154,6 @@ class TestDistribution:
         result.stdout.fnmatch_lines(["*2*Python*", "*2 failed, 1 passed, 1 skipped*"])
         assert result.ret == 1
 
-    @pytest.mark.xfail("sys.platform.startswith('java')", run=False)
     def test_dist_tests_with_crash(self, pytester: pytest.Pytester) -> None:
         if not hasattr(os, "kill"):
             pytest.skip("no os.kill")
@@ -603,11 +602,10 @@ def test_config_initialization(
 def test_crashing_item(pytester, when) -> None:
     """Ensure crashing item is correctly reported during all testing stages"""
     code = dict(setup="", call="", teardown="")
-    code[when] = "py.process.kill(os.getpid())"
+    code[when] = "os._exit(1)"
     p = pytester.makepyfile(
         """
         import os
-        import py
         import pytest
 
         @pytest.fixture
@@ -742,7 +740,7 @@ def test_sub_plugins_disabled(pytester, plugin) -> None:
             pass
     """
     )
-    result = pytester.runpytest(p1, "-n1", "-p", "no:%s" % plugin)
+    result = pytester.runpytest(p1, "-n1", "-p", f"no:{plugin}")
     assert result.ret == 0
     result.stdout.fnmatch_lines("*1 passed*")
 
@@ -1245,22 +1243,14 @@ class TestFileScope:
             "test_b.py::TestB", result.outlines
         )
 
-        assert (
+        assert test_a_workers_and_test_count in (
-            test_a_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
 
     def test_by_class(self, pytester: pytest.Pytester) -> None:
         pytester.makepyfile(
@@ -1285,22 +1275,14 @@ class TestFileScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert (
+        assert test_a_workers_and_test_count in (
-            test_a_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
 
     def test_module_single_start(self, pytester: pytest.Pytester) -> None:
         """Fix test suite never finishing in case all workers start with a single test (#277)."""
@@ -1347,22 +1329,14 @@ class TestGroupScope:
             "test_b.py::TestA", result.outlines
         )
 
-        assert (
+        assert test_a_workers_and_test_count in (
-            test_a_workers_and_test_count
-            in (
             {"gw0": 5},
             {"gw1": 0},
-            )
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
+        assert test_b_workers_and_test_count in (
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
             {"gw0": 5},
             {"gw1": 0},
-            )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 5})
-        )
         assert (
             test_a_workers_and_test_count.items()
             == test_b_workers_and_test_count.items()
@@ -1392,22 +1366,14 @@ class TestGroupScope:
             "test_a.py::TestB", result.outlines
         )
 
-        assert (
+        assert test_a_workers_and_test_count in (
-            test_a_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_a_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
+        assert test_b_workers_and_test_count in (
-        )
-        assert (
-            test_b_workers_and_test_count
-            in (
             {"gw0": 10},
             {"gw1": 0},
-            )
+        ) or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-            or test_b_workers_and_test_count in ({"gw0": 0}, {"gw1": 10})
-        )
         assert (
             test_a_workers_and_test_count.items()
             == test_b_workers_and_test_count.items()
@@ -1576,3 +1542,22 @@ class TestAPI:
         assert xdist.get_xdist_worker_id(fake_request) == "gw5"
         del fake_request.config.workerinput
         assert xdist.get_xdist_worker_id(fake_request) == "master"
+
+
+def test_collection_crash(testdir):
+    p1 = testdir.makepyfile(
+        """
+        assert 0
+    """
+    )
+    result = testdir.runpytest(p1, "-n1")
+    assert result.ret == 1
+    result.stdout.fnmatch_lines(
+        [
+            "gw0 I",
+            "gw0 [[]0[]]",
+            "*_ ERROR collecting test_collection_crash.py _*",
+            "E   assert 0",
+            "*= 1 error in *",
+        ]
+    )

testing/test_dsession.py

@@ -115,18 +115,18 @@ class TestLoadScheduling:
         # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
-        assert sent1 == [0, 2]
+        assert sent1 == [0, 1]
-        assert sent2 == [1, 3]
+        assert sent2 == [2, 3]
         assert sched.pending == [4, 5]
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
         assert len(sched.pending) == 2
         sched.mark_test_complete(node1, 0)
-        assert node1.sent == [0, 2, 4]
+        assert node1.sent == [0, 1, 4]
         assert sched.pending == [5]
-        assert node2.sent == [1, 3]
+        assert node2.sent == [2, 3]
-        sched.mark_test_complete(node1, 2)
+        sched.mark_test_complete(node1, 1)
-        assert node1.sent == [0, 2, 4, 5]
+        assert node1.sent == [0, 1, 4, 5]
         assert not sched.pending
 
     def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
@@ -290,7 +290,8 @@ def test_report_collection_diff_different() -> None:
         " bbb\n"
         "+XXX\n"
         " ccc\n"
-        "-YYY"
+        "-YYY\n"
+        "To see why this happens see Known limitations in documentation"
     )
 
     msg = report_collection_diff(from_collection, to_collection, "1", "2")

testing/test_looponfail.py

@@ -1,4 +1,6 @@
-import py
+import unittest.mock
+from typing import List
+
 import pytest
 import shutil
 import textwrap
@@ -16,7 +18,7 @@ class TestStatRecorder:
         tmp = tmp_path
         hello = tmp / "hello.py"
         hello.touch()
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
@@ -56,15 +58,12 @@ class TestStatRecorder:
         tmp = tmp_path
         tmp.joinpath("dir").mkdir()
         tmp.joinpath("dir", "hello.py").touch()
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
-        assert not sd.fil(py.path.local(tmp / "dir"))
+        assert not sd.fil(tmp / "dir")
 
-    def test_filechange_deletion_race(
+    def test_filechange_deletion_race(self, tmp_path: Path) -> None:
-        self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
-    ) -> None:
         tmp = tmp_path
-        pytmp = py.path.local(tmp)
+        sd = StatRecorder([tmp])
-        sd = StatRecorder([pytmp])
         changed = sd.check()
         assert not changed
 
@@ -76,8 +75,12 @@ class TestStatRecorder:
         p.unlink()
         # make check()'s visit() call return our just removed
         # path as if we were in a race condition
-        monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
+        dirname = str(tmp)
-
+        dirnames: List[str] = []
+        filenames = [str(p)]
+        with unittest.mock.patch(
+            "os.walk", return_value=[(dirname, dirnames, filenames)], autospec=True
+        ):
             changed = sd.check()
         assert changed
 
@@ -85,7 +88,7 @@ class TestStatRecorder:
         tmp = tmp_path
         hello = tmp / "hello.py"
         hello.touch()
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
         changed = sd.check()
         assert not changed
 
@@ -100,7 +103,7 @@ class TestStatRecorder:
         self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
     ) -> None:
         tmp = tmp_path
-        sd = StatRecorder([py.path.local(tmp)])
+        sd = StatRecorder([tmp])
 
         ret_values = [True, False]
         monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -123,7 +126,7 @@ class TestRemoteControl:
         failures = control.runsession()
         assert failures
         control.setup()
-        item_path = item.path if PYTEST_GTE_7 else Path(item.fspath)  # type: ignore[attr-defined]
+        item_path = item.path if PYTEST_GTE_7 else Path(str(item.fspath))  # type: ignore[attr-defined]
         item_path.write_text("def test_func():\n assert 1\n")
         removepyc(item_path)
         topdir, failures = control.runsession()[:2]
@@ -141,7 +144,11 @@ class TestRemoteControl:
         control = RemoteControl(modcol.config)
         control.loop_once()
         assert control.failures
-        modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)  # type: ignore[attr-defined]
+        if PYTEST_GTE_7:
+            modcol_path = modcol.path  # type:ignore[attr-defined]
+        else:
+            modcol_path = Path(str(modcol.fspath))
+
         modcol_path.write_text(
             textwrap.dedent(
                 """

testing/test_newhooks.py

@@ -94,3 +94,28 @@ class TestCrashItem:
         res = pytester.runpytest("-n2", "-s")
         res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
         res.stdout.fnmatch_lines(["*3 passed*"])
+
+    def test_handlecrashitem_one(self, pytester: pytest.Pytester) -> None:
+        """Test pytest_handlecrashitem hook with just one test."""
+        pytester.makeconftest(
+            """
+            test_runs = 0
+
+            def pytest_handlecrashitem(crashitem, report, sched):
+                global test_runs
+
+                if test_runs == 0:
+                    sched.mark_test_pending(crashitem)
+                    test_runs = 1
+                else:
+                    print("HOOK: pytest_handlecrashitem")
+        """
+        )
+        res = pytester.runpytest("-n1", "-s", "-k", "test_b")
+        res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
+        res.stdout.fnmatch_lines(
+            [
+                "FAILED test_handlecrashitem_one.py::test_b",
+                "FAILED test_handlecrashitem_one.py::test_b",
+            ]
+        )

testing/test_plugin.py

@@ -1,5 +1,7 @@
 from contextlib import suppress
 from pathlib import Path
+import sys
+import os
 
 import execnet
 from xdist.workermanage import NodeManager
@@ -7,6 +9,15 @@ from xdist.workermanage import NodeManager
 import pytest
 
 
+@pytest.fixture
+def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch):
+    """Make pytest-xdist believe the system has 3 CPUs"""
+    # block import
+    monkeypatch.setitem(sys.modules, "psutil", None)  # type: ignore
+    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
+    monkeypatch.setattr(os, "cpu_count", lambda: 3)
+
+
 def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
     result = pytester.runpytest("--pdb", "--looponfail")
     assert result.ret != 0
@@ -41,7 +52,6 @@ def test_dist_options(pytester: pytest.Pytester) -> None:
 def test_auto_detect_cpus(
     pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    import os
     from xdist.plugin import pytest_cmdline_main as check_options
 
     with suppress(ImportError):
@@ -102,6 +112,20 @@ def test_auto_detect_cpus_psutil(
     assert config.getoption("numprocesses") == 84
 
 
+def test_auto_detect_cpus_os(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
 def test_hook_auto_num_workers(
     pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
 ) -> None:
@@ -122,20 +146,103 @@ def test_hook_auto_num_workers(
     assert config.getoption("numprocesses") == 42
 
 
-def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
+def test_hook_auto_num_workers_arg(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    # config.option.numprocesses is a pytest feature,
+    # but we document it so let's test it.
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    config = pytester.parseconfigure("-n1", "--boxed")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers(config):
+            if config.option.numprocesses == 'auto':
+                return 42
+            if config.option.numprocesses == 'logical':
+                return 8
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
     check_options(config)
-    assert config.option.forked
+    assert config.getoption("numprocesses") == 42
 
-    config = pytester.parseconfigure("-n1", "--collect-only")
+    config = pytester.parseconfigure("-nlogical")
     check_options(config)
-    assert not config.option.forked
+    assert config.getoption("numprocesses") == 8
 
-    config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
+
+def test_hook_auto_num_workers_none(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    # Returning None from a hook to skip it is pytest behavior,
+    # but we document it so let's test it.
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return None
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
     check_options(config)
-    assert config.option.forked
+    assert config.getoption("numprocesses") == 3
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "5")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 5
+
+
+def test_envvar_auto_num_workers(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "7")
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+    config = pytester.parseconfigure("-nlogical")
+    check_options(config)
+    assert config.getoption("numprocesses") == 7
+
+
+def test_envvar_auto_num_workers_warn(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "fourscore")
+
+    config = pytester.parseconfigure("-nauto")
+    with pytest.warns(UserWarning):
+        check_options(config)
+    assert config.getoption("numprocesses") == 3
+
+
+def test_auto_num_workers_hook_overrides_envvar(
+    pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, monkeypatch_3_cpus
+) -> None:
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    monkeypatch.setenv("PYTEST_XDIST_AUTO_NUM_WORKERS", "987")
+    pytester.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 2
+    """
+    )
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
+
+    config = pytester.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 2
 
 
 def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:

testing/test_remote.py

@@ -1,5 +1,4 @@
 import pprint
-import py
 import pytest
 import sys
 import uuid
@@ -108,7 +107,7 @@ class TestWorkerInteractor:
         assert ev.name == "collectionstart"
         assert not ev.kwargs
         ev = worker.popevent("collectionfinish")
-        assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
+        assert ev.kwargs["topdir"] == str(worker.pytester.path)
         ids = ev.kwargs["ids"]
         assert len(ids) == 1
         worker.sendcommand("runtests", indices=list(range(len(ids))))

testing/test_workermanage.py

@@ -1,11 +1,14 @@
 import execnet
-import py
 import pytest
 import shutil
 import textwrap
+import warnings
 from pathlib import Path
+from util import generate_warning
 from xdist import workermanage
-from xdist.workermanage import HostRSync, NodeManager
+from xdist._path import visit_path
+from xdist.remote import serialize_warning_message
+from xdist.workermanage import HostRSync, NodeManager, unserialize_warning_message
 
 pytest_plugins = "pytester"
 
@@ -154,12 +157,9 @@ class TestHRSync:
         source.joinpath("somedir").mkdir()
         source.joinpath("somedir", "editfile~").touch()
         syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
+        files = list(visit_path(source, recurse=syncer.filter, filter=syncer.filter))
-        assert len(files) == 3
+        names = {x.name for x in files}
-        basenames = [x.basename for x in files]
+        assert names == {"dir", "file.txt", "somedir"}
-        assert "dir" in basenames
-        assert "file.txt" in basenames
-        assert "somedir" in basenames
 
     def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
         gw = execnet.makegateway("popen//chdir=%s" % dest)
@@ -326,3 +326,80 @@ class TestNodeManager:
         )
         (rep,) = reprec.getreports("pytest_runtest_logreport")
         assert rep.passed
+
+
+class MyWarning(UserWarning):
+    pass
+
+
+@pytest.mark.parametrize(
+    "w_cls",
+    [
+        UserWarning,
+        MyWarning,
+        "Imported",
+        pytest.param(
+            "Nested",
+            marks=pytest.mark.xfail(reason="Nested warning classes are not supported."),
+        ),
+    ],
+)
+def test_unserialize_warning_msg(w_cls):
+    """Test that warning serialization process works well"""
+
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        if not isinstance(w_cls, str):
+            warnings.warn("hello", w_cls)
+        elif w_cls == "Imported":
+            generate_warning()
+        elif w_cls == "Nested":
+            # dynamic creation
+            class MyWarning2(UserWarning):
+                pass
+
+            warnings.warn("hello", MyWarning2)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+    w_msg2 = unserialize_warning_message(data)
+
+    # Compare the two objects
+    all_keys = set(vars(w_msg).keys()).union(set(vars(w_msg2).keys()))
+    for k in all_keys:
+        v1 = getattr(w_msg, k)
+        v2 = getattr(w_msg2, k)
+        if k == "message":
+            assert type(v1) == type(v2)
+            assert v1.args == v2.args
+        else:
+            assert v1 == v2
+
+
+class MyWarningUnknown(UserWarning):
+    # Changing the __module__ attribute is only safe if class can be imported
+    # from there
+    __module__ = "unknown"
+
+
+def test_warning_serialization_tweaked_module():
+    """Test for GH#404"""
+
+    # Create a test warning message
+    with pytest.warns(UserWarning) as w:
+        warnings.warn("hello", MyWarningUnknown)
+
+    # Unpack
+    assert len(w) == 1
+    w_msg = w[0]
+
+    # Serialize and deserialize
+    data = serialize_warning_message(w_msg)
+
+    # __module__ cannot be found!
+    with pytest.raises(ModuleNotFoundError):
+        unserialize_warning_message(data)

testing/util.py

@@ -0,0 +1,9 @@
+import warnings
+
+
+class MyWarning2(UserWarning):
+    pass
+
+
+def generate_warning():
+    warnings.warn(MyWarning2("hello"))

tox.ini

@@ -5,7 +5,7 @@ envlist=
   py38-pytestmain
   py38-psutil
   py38-setproctitle
-
+isolated_build = true
 [testenv]
 extras = testing
 deps =
@@ -39,7 +39,7 @@ commands = pre-commit run --all-files --show-diff-on-failure
 
 [testenv:release]
 changedir=
-decription = do a release, required posarg of the version number
+description = do a release, required posarg of the version number
 basepython = python3.7
 skipsdist = True
 usedevelop = True
@@ -47,7 +47,16 @@ passenv = *
 deps =
   towncrier
 commands =
-  towncrier --version {posargs} --yes
+  towncrier build --version {posargs} --yes
+
+[testenv:docs]
+basepython = python3
+usedevelop = True
+deps =
+    sphinx
+    sphinx_rtd_theme
+commands =
+    sphinx-build -W --keep-going -b html docs docs/_build/html {posargs:}
 
 [pytest]
 # pytest-services also defines a worker_id fixture, disable