release 2.2.1

Add python 3.9 to classifiers

.github/workflows/main.yml

@@ -0,0 +1,99 @@
+name: build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        tox_env:
+          - "py35-pytestlatest"
+          - "py36-pytestlatest"
+          - "py37-pytestlatest"
+          - "py38-pytestlatest"
+          - "py39-pytestlatest"
+          - "py38-pytestmaster"
+          - "py38-psutil"
+          - "linting"
+
+        os: [ubuntu-latest, windows-latest]
+        include:
+          - tox_env: "py35-pytestlatest"
+            python: "3.5"
+          - tox_env: "py36-pytestlatest"
+            python: "3.6"
+          - tox_env: "py37-pytestlatest"
+            python: "3.7"
+          - tox_env: "py38-pytestlatest"
+            python: "3.8"
+          - tox_env: "py39-pytestlatest"
+            python: "3.9"
+          - tox_env: "py38-pytestmaster"
+            python: "3.8"
+          - tox_env: "py38-psutil"
+            python: "3.8"
+          - tox_env: "linting"
+            python: "3.7"
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Test
+      run: |
+        tox -e ${{ matrix.tox_env }}
+
+  linting:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: "3.7"
+    - name: Install tox
+      run: |
+        python -m pip install --upgrade pip
+        pip install tox
+    - name: Linting
+      run: |
+        tox -e linting
+
+  deploy:
+
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+
+    runs-on: ubuntu-latest
+
+    needs: [build, linting]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: "3.7"
+    - name: Install wheel
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+    - name: Build package
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Publish package to PyPI
+      uses: pypa/gh-action-pypi-publish@master
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_token }}

.gitignore

@@ -23,7 +23,7 @@ include/
 lib/
 bin/
 env/
-xdist/_version.py*
+src/xdist/_version.py*
 pytest_xdist.egg-info
 issue/
 3rdparty/

.pre-commit-config.yaml

@@ -1,18 +1,23 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 18.6b4
+    rev: 19.10b0
     hooks:
     -   id: black
-        args: [--safe, --quiet]
+        args: [--safe, --quiet, --target-version, py35]
-        language_version: python3.6
+        language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+    rev: v2.4.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: debug-statements
     -   id: flake8
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
+    hooks:
+    -   id: pyupgrade
+        args: [--py3-plus]
 -   repo: local
     hooks:
     -   id: rst
@@ -21,4 +26,4 @@ repos:
         files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
-        python_version: python3.6
+        language_version: python3.7

.travis.yml

@@ -1,67 +0,0 @@
-sudo: false
-language: python
-
-notifications:
-  irc:
-    channels:
-    - 'chat.freenode.net#pytest'
-    on_success: change
-    on_failure: change
-    skip_join: true
-  email:
-  - pytest-commit@python.org
-
-install:
-  - pip install -U pip
-  - pip install tox setuptools_scm
-script: tox
-
-stages:
-- baseline
-- test
-- name: deploy
-  if: repo = pytest-dev/pytest-xdist AND tag IS present
-
-jobs:
-  include:
-    - stage: baseline
-      python: '3.6'
-      env: TOXENV=linting
-    - python: '3.6'
-      env: TOXENV=py36-pytestlatest
-    - python: '2.7'
-      env: TOXENV=py27-pytestlatest
-
-    - stage: test
-      python: "3.4"
-      env: TOXENV=py34-pytestlatest
-    - python: "3.5"
-      env: TOXENV=py35-pytestlatest
-    - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-      sudo: required
-      dist: xenial
-    - python: "2.7"
-      env: TOXENV=py27-pytestmaster
-    - python: "2.7"
-      env: TOXENV=py27-pytestfeatures
-    - python: "3.6"
-      env: TOXENV=py36-pytestmaster
-    - python: "3.6"
-      env: TOXENV=py36-pytestfeatures
-
-    - stage: deploy
-      python: '3.6'
-      env:
-      install: pip install -U setuptools setuptools_scm
-      script: skip
-      deploy:
-        provider: pypi
-        user: ronny
-        distributions: sdist bdist_wheel
-        skip_upload_docs: true
-        password:
-          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-        on:
-          tags: true
-          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,230 @@
+pytest-xdist 2.2.1 (2021-02-09)
+===============================
+
+Bug Fixes
+---------
+
+- `#623 <https://github.com/pytest-dev/pytest-xdist/issues/623>`_: Gracefully handle the pending deprecation of Node.fspath by using config.rootpath for topdir.
+
+
+pytest-xdist 2.2.0 (2020-12-14)
+===============================
+
+Features
+--------
+
+- `#608 <https://github.com/pytest-dev/pytest-xdist/issues/608>`_: Internal errors in workers are now propagated to the master node.
+
+
+pytest-xdist 2.1.0 (2020-08-25)
+===============================
+
+Features
+--------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: New ``pytest_xdist_auto_num_workers`` hook can be implemented by plugins or ``conftest.py`` files to control the number of workers when ``--numprocesses=auto`` is given in the command-line.
+
+
+Trivial Changes
+---------------
+
+- `#585 <https://github.com/pytest-dev/pytest-xdist/issues/585>`_: ``psutil`` has proven to make ``pytest-xdist`` installation in certain platforms and containers problematic, so to use it for automatic number of CPUs detection users need to install the ``psutil`` extra::
+
+      pip install pytest-xdist[psutil]
+
+
+pytest-xdist 2.0.0 (2020-08-12)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: Drop backward-compatibility "slave" aliases related to worker nodes.  We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.
+
+- `#569 <https://github.com/pytest-dev/pytest-xdist/issues/569>`_: ``pytest-xdist`` no longer supports Python 2.7.
+
+
+Features
+--------
+
+- `#504 <https://github.com/pytest-dev/pytest-xdist/issues/504>`_: New functions ``xdist.is_xdist_worker``, ``xdist.is_xdist_master``, ``xdist.get_xdist_worker_id``, to easily identify the current node.
+
+
+Bug Fixes
+---------
+
+- `#471 <https://github.com/pytest-dev/pytest-xdist/issues/471>`_: Fix issue with Rsync reporting in quiet mode.
+
+- `#553 <https://github.com/pytest-dev/pytest-xdist/issues/553>`_: When using ``-n auto``, count the number of physical CPU cores instead of logical ones.
+
+
+Trivial Changes
+---------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: ``pytest-xdist`` now requires ``pytest>=6.0``.
+
+
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
+pytest-xdist 1.32.0 (2020-05-03)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- `#475 <https://github.com/pytest-dev/pytest-xdist/issues/475>`_: Drop support for EOL Python 3.4.
+
+
+Features
+--------
+
+- `#524 <https://github.com/pytest-dev/pytest-xdist/issues/524>`_: Add `testrun_uid` fixture. This is a shared value that uniquely identifies a test run among all workers.
+  This also adds a `PYTEST_XDIST_TESTRUNUID` environment variable that is accessible within a test as well as a command line option `--testrunuid` to manually set the value from outside.
+
+
+pytest-xdist 1.31.0 (2019-12-19)
+================================
+
+Features
+--------
+
+- `#486 <https://github.com/pytest-dev/pytest-xdist/issues/486>`_: Add support for Python 3.8.
+
+
+Bug Fixes
+---------
+
+- `#491 <https://github.com/pytest-dev/pytest-xdist/issues/491>`_: Fix regression that caused custom plugin command-line arguments to be discarded when using ``--tx`` mode.
+
+
+
+pytest-xdist 1.30.0 (2019-10-01)
+================================
+
+Features
+--------
+
+- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
+  long-standing issues related to startup with the ``-c`` option.
+
+  Issues:
+
+  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
+  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
+
+  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
+
+
+Bug Fixes
+---------
+
+- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
+
+
+pytest-xdist 1.29.0 (2019-06-14)
+================================
+
+Features
+--------
+
+- `#226 <https://github.com/pytest-dev/pytest-xdist/issues/226>`_: ``--max-worker-restart`` now assumes a more reasonable value (4 times the number of
+  nodes) when not given explicitly. This prevents test suites from running forever when the suite crashes during collection.
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: When the test session is interrupted due to running out of workers, the reason is shown in the test summary
+  for easier viewing.
+
+- `#442 <https://github.com/pytest-dev/pytest-xdist/issues/442>`_: Compatibility fix for upcoming pytest 5.0: ``session.exitstatus`` is now an ``IntEnum`` object.
+
+
+Bug Fixes
+---------
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: No longer show an internal error when we run out of workers due to crashes.
+
+
+pytest-xdist 1.28.0 (2019-04-02)
+================================
+
+Features
+--------
+
+- `#426 <https://github.com/pytest-dev/pytest-xdist/issues/426>`_: ``pytest-xdist`` now uses the new ``pytest_report_to_serializable`` and ``pytest_report_from_serializable``
+  hooks from ``pytest 4.4`` (still experimental). This will make report serialization more reliable and
+  extensible.
+
+  This also means that ``pytest-xdist`` now requires ``pytest>=4.4``.
+
+
+pytest-xdist 1.27.0 (2019-02-15)
+================================
+
+Features
+--------
+
+- `#374 <https://github.com/pytest-dev/pytest-xdist/issues/374>`_: The new ``pytest_xdist_getremotemodule`` hook allows overriding the module run on remote nodes.
+
+- `#415 <https://github.com/pytest-dev/pytest-xdist/issues/415>`_: Improve behavior of ``--numprocesses=auto`` to work well with ``--pdb`` option.
+
+
+pytest-xdist 1.26.1 (2019-01-28)
+================================
+
+Bug Fixes
+---------
+
+- `#406 <https://github.com/pytest-dev/pytest-xdist/issues/406>`_: Do not implement deprecated ``pytest_logwarning`` hook in pytest versions where it is deprecated.
+
+
+pytest-xdist 1.26.0 (2019-01-11)
+================================
+
+Features
+--------
+
+- `#376 <https://github.com/pytest-dev/pytest-xdist/issues/376>`_: The current directory is no longer added ``sys.path`` for local workers, only for remote connections.
+
+  This behavior is surprising because it makes xdist runs and non-xdist runs to potentially behave differently.
+
+
+Bug Fixes
+---------
+
+- `#379 <https://github.com/pytest-dev/pytest-xdist/issues/379>`_: Warning attributes are checked to make sure they can be dumped prior to
+  serializing the warning for submission to the master node.
+
+
 pytest-xdist 1.25.0 (2018-12-12)
 ================================
 
@@ -329,17 +556,6 @@ Improved Documentation
 - Added ``HOWTORELEASE`` documentation. (#155)
 
 
-..
-    You should *NOT* be adding new change log entries to this file, this
-    file is managed by towncrier. You *may* edit previous change logs to
-    fix problems like typo corrections or such.
-    To add a new change log entry, please see
-    https://pip.pypa.io/en/latest/development/#adding-a-news-entry
-    We named the news folder ``changelog``
-
-.. towncrier release notes start
-
-
 1.17.0
 ------
 

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no pytest plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

README.rst

@@ -11,13 +11,8 @@
     :alt: Python versions
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
+.. image:: https://github.com/pytest-dev/pytest-xdist/workflows/build/badge.svg
-    :alt: Travis CI build status
+    :target: https://github.com/pytest-dev/pytest-xdist/actions
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
-
-.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
-    :alt: AppVeyor build status
-    :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black
@@ -57,43 +52,99 @@ Install the plugin with::
 
     pip install pytest-xdist
 
-or use the package in develop/in-place mode with
-a checkout of the `pytest-xdist repository`_ ::
 
-    pip install --editable .
+To use ``psutil`` for detection of the number of CPUs available, install the ``psutil`` extra::
+
+    pip install pytest-xdist[psutil]
+
 
 .. _parallelization:
 
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, type::
+To send tests to multiple CPUs, use the ``-n`` (or ``--numprocesses``) option::
 
-    pytest -n NUM
+    pytest -n NUMCPUS
 
-Especially for longer running tests or tests requiring
+Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
-a lot of I/O this can lead to considerable speed ups. This option can
+can lead to considerable speed ups, especially if your test suite takes a
-also be set to ``auto`` for automatic detection of the number of CPUs.
+noticeable amount of time.
 
-If a test crashes the interpreter, pytest-xdist will automatically restart
+If a test crashes a worker, pytest-xdist will automatically restart that worker
-that worker and report the failure as usual. You can use the
+and report the test’s failure. You can use the ``--max-worker-restart`` option
-``--max-worker-restart`` option to limit the number of workers that can
+to limit the number of worker restarts that are allowed, or disable restarting
-be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+altogether using ``--max-worker-restart 0``.
 
-By default, the ``-n`` option will send pending tests to any worker that is available, without
+By default, using ``--numprocesses`` will send pending tests to any worker that
-any guaranteed order, but you can control this with these options:
+is available, without any guaranteed order. You can change the test
+distribution algorithm this with the ``--dist`` option. It takes these values:
 
-* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
+* ``--dist no``: The default algorithm, distributing one test at a time.
-  by **class** for *test methods*, then each group will be sent to an available worker,
-  guaranteeing that all tests in a group run in the same process. This can be useful if you have
-  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
-  with grouping by class takes priority over grouping by module.
-  This feature was added in version ``1.19``.
 
-* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
-  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
+  and by **class** for *test methods*. Groups are distributed to available
-  in version ``1.21``.
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
 
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if worker_id == "master":
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
 
 Running tests in a Python subprocess
 ------------------------------------
@@ -191,7 +242,7 @@ at once. The specifications strings use the `xspec syntax`_.
 
 .. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
 
-.. _`socketserver.py`: http://bitbucket.org/hpk42/execnet/raw/2af991418160/execnet/script/socketserver.py
+.. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py
 
 .. _`execnet`: http://codespeak.net/execnet
 
@@ -213,7 +264,7 @@ a test or fixture, you may use the ``worker_id`` fixture to do so:
 When ``xdist`` is disabled (running with ``-n0`` for example), then
 ``worker_id`` will return ``"master"``.
 
-Additionally, worker processes have the following environment variables
+Worker processes also have the following environment variables
 defined:
 
 * ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
@@ -223,6 +274,75 @@ defined:
 The information about the worker_id in a test is stored in the ``TestReport`` as
 well, under the ``worker_id`` attribute.
 
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+.. code-block:: python
+
+    def is_xdist_worker(request_or_session) -> bool:
+        """Return `True` if this is an xdist worker, `False` otherwise
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def is_xdist_master(request_or_session) -> bool:
+        """Return `True` if this is the xdist master, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def get_xdist_worker_id(request_or_session) -> str:
+        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+        if running on the 'master' node.
+
+        If not distributing tests (for example passing `-n0` or not passing `-n` at all) also return 'master'.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+
+Uniquely identifying the current test run
+-----------------------------------------
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """ create a unique database for this particular test run """
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """ retrieve unique database """
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+* ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
+
+Accessing ``sys.argv`` from the master node in workers
+------------------------------------------------------
+
+To access the ``sys.argv`` passed to the command-line of the master node, use
+``request.config.workerinput["mainargv"]``.
+
 
 Specifying test exec environments in an ini file
 ------------------------------------------------

appveyor.yml

@@ -1,24 +0,0 @@
-environment:
-  matrix:
-  - TOXENV: "py27-pytestlatest"
-  - TOXENV: "py34-pytestlatest"
-  - TOXENV: "py35-pytestlatest"
-  - TOXENV: "py36-pytestlatest"
-  - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py27-pytestmaster"
-  - TOXENV: "py36-pytestmaster"
-  - TOXENV: "py27-pytestfeatures"
-  - TOXENV: "py36-pytestfeatures"
-
-install:
-  - C:\Python37\python -m pip install -U pip
-  - C:\Python37\python -m pip install -U tox setuptools_scm
-
-build: false  # Not a C# project, build stuff at the test step instead.
-
-test_script:
-  - C:\Python37\python -m tox
-
-# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
-# might as well save resources
-skip_tags: true

setup.cfg

@@ -1,6 +1,3 @@
-[bdist_wheel]
-universal = 1
-
 [metadata]
 license_file = LICENSE
 

setup.py

@@ -1,25 +1,29 @@
 from setuptools import setup, find_packages
 
-install_requires = ["execnet>=1.1", "pytest>=3.6.0", "pytest-forked", "six"]
+install_requires = ["execnet>=1.1", "pytest>=6.0.0", "pytest-forked"]
 
 
+with open("README.rst") as f:
+    long_description = f.read()
+
 setup(
     name="pytest-xdist",
-    use_scm_version={"write_to": "xdist/_version.py"},
+    use_scm_version={"write_to": "src/xdist/_version.py"},
-    description="pytest xdist plugin for distributed testing"
+    description="pytest xdist plugin for distributed testing and loop-on-failing modes",
-    " and loop-on-failing modes",
+    long_description=long_description,
-    long_description=open("README.rst").read(),
     license="MIT",
     author="holger krekel and contributors",
     author_email="pytest-dev@python.org,holger@merlinux.eu",
     url="https://github.com/pytest-dev/pytest-xdist",
     platforms=["linux", "osx", "win32"],
-    packages=find_packages(exclude=["testing", "example"]),
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    extras_require={"testing": ["filelock"], "psutil": ["psutil>=3.0"]},
     entry_points={
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
+    python_requires=">=3.5",
     install_requires=install_requires,
     setup_requires=["setuptools_scm"],
     classifiers=[
@@ -34,12 +38,12 @@ setup(
         "Topic :: Software Development :: Quality Assurance",
         "Topic :: Utilities",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
     ],
 )

src/xdist/__init__.py

@@ -0,0 +1,4 @@
+from xdist.plugin import is_xdist_worker, is_xdist_master, get_xdist_worker_id
+from xdist._version import version as __version__
+
+__all__ = ["__version__", "is_xdist_worker", "is_xdist_master", "get_xdist_worker_id"]

src/xdist/dsession.py

@@ -10,14 +10,14 @@ from xdist.scheduler import (
 )
 
 
-from six.moves.queue import Empty, Queue
+from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
 
 
-class DSession(object):
+class DSession:
     """A pytest plugin which runs a distributed test session
 
     At the beginning of the test session this creates a NodeManager
@@ -46,14 +46,11 @@ class DSession(object):
         self._failed_collection_errors = {}
         self._active_nodes = set()
         self._failed_nodes_count = 0
-        self._max_worker_restart = self.config.option.maxworkerrestart
+        self._max_worker_restart = get_default_max_worker_restart(self.config)
-        if self._max_worker_restart is not None:
+        # summary message to print at the end of the session
-            self._max_worker_restart = int(self._max_worker_restart)
+        self._summary_report = None
-        try:
         self.terminal = config.pluginmanager.getplugin("terminalreporter")
-        except KeyError:
+        if self.terminal:
-            self.terminal = None
-        else:
             self.trdist = TerminalDistReporter(config)
             config.pluginmanager.register(self.trdist, "terminaldistreporter")
 
@@ -153,9 +150,6 @@ class DSession(object):
         node.workerinfo["id"] = node.gateway.id
         node.workerinfo["spec"] = node.gateway.spec
 
-        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
-        node.slaveinfo = node.workerinfo
-
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
@@ -172,7 +166,7 @@ class DSession(object):
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
         if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
-            self.shouldstop = "%s received keyboard-interrupt" % (node,)
+            self.shouldstop = "{} received keyboard-interrupt".format(node)
             self.worker_errordown(node, "keyboard-interrupt")
             return
         if node in self.sched.nodes:
@@ -180,6 +174,24 @@ class DSession(object):
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
+    def worker_internal_error(self, node, formatted_error):
+        """
+        pytest_internalerror() was called on the worker.
+
+        pytest_internalerror() arguments are an excinfo and an excrepr, which can't
+        be serialized, so we go with a poor man's solution of raising an exception
+        here ourselves using the formatted message.
+        """
+        self._active_nodes.remove(node)
+        try:
+            assert False, formatted_error
+        except AssertionError:
+            from _pytest._code import ExceptionInfo
+
+            excinfo = ExceptionInfo.from_current()
+            excrepr = excinfo.getrepr()
+            self.config.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo)
+
     def worker_errordown(self, node, error):
         """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
@@ -198,15 +210,23 @@ class DSession(object):
         )
         if maximum_reached:
             if self._max_worker_restart == 0:
-                msg = "Worker restarting disabled"
+                msg = "worker {} crashed and worker restarting disabled".format(
+                    node.gateway.id
+                )
             else:
-                msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
+                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
-            self.report_line(msg)
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
         else:
-            self.report_line("Replacing crashed worker %s" % node.gateway.id)
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
+    def pytest_terminal_summary(self, terminalreporter):
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
+
     def worker_collectionfinish(self, node, ids):
         """worker has finished test collection.
 
@@ -266,16 +286,18 @@ class DSession(object):
         assert not rep.passed
         self._failed_worker_collectreport(node, rep)
 
-    def worker_logwarning(self, message, code, nodeid, fslocation):
-        """Emitted when a node calls the pytest_logwarning hook."""
-        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
-        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
-
     def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_logwarning hook."""
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 
@@ -316,7 +338,7 @@ class DSession(object):
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
+        msg = "worker {!r} crashed while running {!r}".format(worker.gateway.id, nodeid)
         rep = runner.TestReport(
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
@@ -324,7 +346,7 @@ class DSession(object):
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
-class TerminalDistReporter(object):
+class TerminalDistReporter:
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")
@@ -346,7 +368,9 @@ class TerminalDistReporter(object):
 
     def getstatus(self):
         if self.config.option.verbose >= 0:
-            parts = ["%s %s" % (spec.id, self._status[spec.id]) for spec in self._specs]
+            parts = [
+                "{} {}".format(spec.id, self._status[spec.id]) for spec in self._specs
+            ]
             return " / ".join(parts)
         else:
             return "bringing up nodes..."
@@ -381,19 +405,27 @@ class TerminalDistReporter(object):
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
             d = node.workerinfo
-            infoline = "[%s] Python %s" % (d["id"], d["version"].replace("\n", " -- "))
+            infoline = "[{}] Python {}".format(
+                d["id"], d["version"].replace("\n", " -- ")
+            )
             self.rewrite(infoline, newline=True)
         self.setstatus(node.gateway.spec, "ok")
 
     def pytest_testnodedown(self, node, error):
         if not error:
             return
-        self.write_line("[%s] node down: %s" % (node.gateway.id, error))
+        self.write_line("[{}] node down: {}".format(node.gateway.id, error))
 
-    # def pytest_xdist_rsyncstart(self, source, gateways):
+
-    #    targets = ",".join([gw.id for gw in gateways])
+def get_default_max_worker_restart(config):
-    #    msg = "[%s] rsyncing: %s" %(targets, source)
+    """gets the default value of --max-worker-restart option if it is not provided.
-    #    self.write_line(msg)
+
-    # def pytest_xdist_rsyncfinish(self, source, gateways):
+    Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
-    #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
+    """
-    #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
+    result = config.option.maxworkerrestart
+    if result is not None:
+        result = int(result)
+    elif config.option.numprocesses:
+        # if --max-worker-restart was not provided, use a reasonable default (#226)
+        result = config.option.numprocesses * 4
+    return result

src/xdist/looponfail.py

@@ -6,7 +6,6 @@
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-from __future__ import print_function
 import py
 import pytest
 import sys
@@ -30,7 +29,7 @@ def pytest_addoption(parser):
 def pytest_cmdline_main(config):
 
     if config.getoption("looponfail"):
-        usepdb = config.getoption("usepdb")  # a core option
+        usepdb = config.getoption("usepdb", False)  # a core option
         if usepdb:
             raise pytest.UsageError("--pdb is incompatible with --looponfail.")
         looponfail_main(config)
@@ -55,7 +54,7 @@ def looponfail_main(config):
         print()
 
 
-class RemoteControl(object):
+class RemoteControl:
     def __init__(self, config):
         self.config = config
         self.failures = []
@@ -136,7 +135,7 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
                 tr.line(report, red=True)
     tr.sep("#", "waiting for changes", bold=True)
     for rootdir in rootdirs:
-        tr.line("### Watching:   %s" % (rootdir,), bold=True)
+        tr.line("### Watching:   {}".format(rootdir), bold=True)
 
 
 def init_worker_session(channel, args, option_dict):
@@ -165,7 +164,7 @@ def init_worker_session(channel, args, option_dict):
     WorkerFailSession(config, channel).main()
 
 
-class WorkerFailSession(object):
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -220,7 +219,7 @@ class WorkerFailSession(object):
         self.channel.send((trails, failreports, self.collection_failed))
 
 
-class StatRecorder(object):
+class StatRecorder:
     def __init__(self, rootdirlist):
         self.rootdirlist = rootdirlist
         self.statcache = {}

src/xdist/newhooks.py

@@ -30,6 +30,11 @@ def pytest_xdist_rsyncfinish(source, gateways):
     """ called after rsyncing a directory to remote gateways takes place. """
 
 
+@pytest.mark.firstresult
+def pytest_xdist_getremotemodule():
+    """ called when creating remote node"""
+
+
 def pytest_configure_node(node):
     """ configure node information before it gets instantiated. """
 
@@ -50,3 +55,13 @@ def pytest_xdist_node_collection_finished(node, ids):
 @pytest.mark.firstresult
 def pytest_xdist_make_scheduler(config, log):
     """ return a node scheduler implementation """
+
+
+@pytest.mark.firstresult
+def pytest_xdist_auto_num_workers(config):
+    """
+    Return the number of workers to spawn when ``--numprocesses=auto`` is given in the
+    command-line.
+
+    .. versionadded:: 2.1
+    """

src/xdist/plugin.py

@@ -1,12 +1,25 @@
 import os
+import uuid
 
 import py
 import pytest
 
 
-def auto_detect_cpus():
+def pytest_xdist_auto_num_workers():
+    try:
+        import psutil
+    except ImportError:
+        pass
+    else:
+        count = psutil.cpu_count(logical=False) or psutil.cpu_count()
+        if count:
+            return count
     try:
         from os import sched_getaffinity
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
     except ImportError:
         if os.environ.get("TRAVIS") == "true":
             # workaround https://bitbucket.org/pypy/pypy/issues/2375
@@ -15,11 +28,6 @@ def auto_detect_cpus():
             from os import cpu_count
         except ImportError:
             from multiprocessing import cpu_count
-    else:
-
-        def cpu_count():
-            return len(sched_getaffinity(0))
-
     try:
         n = cpu_count()
     except NotImplementedError:
@@ -29,8 +37,8 @@ def auto_detect_cpus():
 
 def parse_numprocesses(s):
     if s == "auto":
-        return auto_detect_cpus()
+        return "auto"
-    else:
+    elif s is not None:
         return int(s)
 
 
@@ -45,7 +53,7 @@ def pytest_addoption(parser):
         type=parse_numprocesses,
         help="shortcut for '--dist=load --tx=NUM*popen', "
         "you can use 'auto' here for auto detection CPUs number on "
-        "host system",
+        "host system and it will be 0 when used with --pdb",
     )
     group.addoption(
         "--maxprocesses",
@@ -57,14 +65,11 @@ def pytest_addoption(parser):
     )
     group.addoption(
         "--max-worker-restart",
-        "--max-slave-restart",
         action="store",
         default=None,
         dest="maxworkerrestart",
         help="maximum number of workers that can be restarted "
-        "when crashed (set to zero to disable this feature)\n"
+        "when crashed (set to zero to disable this feature)",
-        "'--max-slave-restart' option is deprecated and will be removed in "
-        "a future release",
     )
     group.addoption(
         "--dist",
@@ -118,12 +123,22 @@ def pytest_addoption(parser):
         metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.",
     )
-
     group.addoption(
         "--boxed",
         action="store_true",
         help="backward compatibility alias for pytest-forked --forked",
     )
+    group.addoption(
+        "--testrunuid",
+        action="store",
+        help=(
+            "provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture,\n\n,"
+            "if not provided, 'testrun_uid' is filled with a new unique string "
+            "on every test run."
+        ),
+    )
+
     parser.addini(
         "rsyncdirs",
         "list of (relative) paths to be rsynced for remote distributed testing.",
@@ -150,11 +165,7 @@ def pytest_addoption(parser):
 def pytest_addhooks(pluginmanager):
     from xdist import newhooks
 
-    # avoid warnings with pytest-2.8
+    pluginmanager.add_hookspecs(newhooks)
-    method = getattr(pluginmanager, "add_hookspecs", None)
-    if method is None:
-        method = pluginmanager.addhooks
-    method(newhooks)
 
 
 # -------------------------------------------------------------------------
@@ -170,6 +181,7 @@ def pytest_configure(config):
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
         tr = config.pluginmanager.getplugin("terminalreporter")
+        if tr:
             tr.showfspath = False
     if config.getoption("boxed"):
         config.option.forked = True
@@ -177,6 +189,15 @@ def pytest_configure(config):
 
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
+    usepdb = config.getoption("usepdb", False)  # a core option
+    if config.option.numprocesses == "auto":
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            auto_num_cpus = config.hook.pytest_xdist_auto_num_workers(config=config)
+            config.option.numprocesses = auto_num_cpus
+
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
@@ -187,26 +208,66 @@ def pytest_cmdline_main(config):
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue
-    if not val("collectonly"):
+    if not val("collectonly") and val("dist") != "no" and usepdb:
-        usepdb = config.getoption("usepdb")  # a core option
-        if val("dist") != "no":
-            if usepdb:
         raise pytest.UsageError(
-                    "--pdb is incompatible with distributing tests; try using -n0."
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
         )  # noqa: E501
 
 
 # -------------------------------------------------------------------------
-# fixtures
+# fixtures and API to easily know the role of current node
 # -------------------------------------------------------------------------
 
 
+def is_xdist_worker(request_or_session) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_master(request_or_session) -> bool:
+    """Return `True` if this is the xdist master, `False` otherwise
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+def get_xdist_worker_id(request_or_session) -> str:
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the 'master' node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        return request_or_session.config.workerinput["workerid"]
+    else:
+        return "master"
+
+
 @pytest.fixture(scope="session")
 def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
+    return get_xdist_worker_id(request)
+
+
+@pytest.fixture(scope="session")
+def testrun_uid(request):
+    """Return the unique id of the current test."""
     if hasattr(request.config, "workerinput"):
-        return request.config.workerinput["workerid"]
+        return request.config.workerinput["testrunuid"]
     else:
-        return "master"
+        return uuid.uuid4().hex

src/xdist/remote.py

@@ -10,15 +10,18 @@ import sys
 import os
 import time
 
-import _pytest.hookspec
+import py
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
+from _pytest.config import _prepareconfig, Config
 
-class WorkerInteractor(object):
+
+class WorkerInteractor:
     def __init__(self, config, channel):
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
+        self.testrunuid = config.workerinput["testrunuid"]
         self.log = py.log.Producer("worker-%s" % self.workerid)
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
@@ -30,8 +33,10 @@ class WorkerInteractor(object):
         self.channel.send((name, kwargs))
 
     def pytest_internalerror(self, excrepr):
-        for line in str(excrepr).split("\n"):
+        formatted_error = str(excrepr)
+        for line in formatted_error.split("\n"):
             self.log("IERROR>", line)
+        interactor.sendevent("internal_error", formatted_error=formatted_error)
 
     def pytest_sessionstart(self, session):
         self.session = session
@@ -40,7 +45,8 @@ class WorkerInteractor(object):
 
     @pytest.hookimpl(hookwrapper=True)
     def pytest_sessionfinish(self, exitstatus):
-        self.config.workeroutput["exitstatus"] = exitstatus
+        # in pytest 5.0+, exitstatus is an IntEnum object
+        self.config.workeroutput["exitstatus"] = int(exitstatus)
         yield
         self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
@@ -87,100 +93,51 @@ class WorkerInteractor(object):
         )
 
     def pytest_collection_finish(self, session):
+        try:
+            topdir = str(self.config.rootpath)
+        except AttributeError:  # pytest <= 6.1.0
+            topdir = str(self.config.rootdir)
+
         self.sendevent(
             "collectionfinish",
-            topdir=str(session.fspath),
+            topdir=topdir,
             ids=[item.nodeid for item in session.items],
         )
 
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
-    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
-    if hasattr(_pytest.hookspec, "pytest_runtest_logfinish"):
-
     def pytest_runtest_logfinish(self, nodeid, location):
         self.sendevent("logfinish", nodeid=nodeid, location=location)
 
     def pytest_runtest_logreport(self, report):
-        data = serialize_report(report)
+        data = self.config.hook.pytest_report_to_serializable(
+            config=self.config, report=report
+        )
         data["item_index"] = self.item_index
         data["worker_id"] = self.workerid
+        data["testrun_uid"] = self.testrunuid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
         # send only reports that have not passed to master as optimization (#330)
         if not report.passed:
-            data = serialize_report(report)
+            data = self.config.hook.pytest_report_to_serializable(
+                config=self.config, report=report
+            )
             self.sendevent("collectreport", data=data)
 
-    # the pytest_logwarning hook was removed in pytest 4.1
+    def pytest_warning_recorded(self, warning_message, when, nodeid, location):
-    if hasattr(_pytest.hookspec, "pytest_logwarning"):
-
-        def pytest_logwarning(self, message, code, nodeid, fslocation):
         self.sendevent(
-                "logwarning",
+            "warning_recorded",
-                message=message,
-                code=code,
-                nodeid=nodeid,
-                fslocation=str(fslocation),
-            )
-
-    # the pytest_warning_captured hook was introduced in pytest 3.8
-    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
-
-        def pytest_warning_captured(self, warning_message, when, item):
-            self.sendevent(
-                "warning_captured",
             warning_message_data=serialize_warning_message(warning_message),
             when=when,
-                # item cannot be serialized and will always be None when used with xdist
+            nodeid=nodeid,
-                item=None,
+            location=location,
         )
 
 
-def serialize_report(rep):
-    def disassembled_report(rep):
-        reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
-        reprcrash = rep.longrepr.reprcrash.__dict__.copy()
-
-        new_entries = []
-        for entry in reprtraceback["reprentries"]:
-            entry_data = {"type": type(entry).__name__, "data": entry.__dict__.copy()}
-            for key, value in entry_data["data"].items():
-                if hasattr(value, "__dict__"):
-                    entry_data["data"][key] = value.__dict__.copy()
-            new_entries.append(entry_data)
-
-        reprtraceback["reprentries"] = new_entries
-
-        return {
-            "reprcrash": reprcrash,
-            "reprtraceback": reprtraceback,
-            "sections": rep.longrepr.sections,
-        }
-
-    import py
-
-    d = rep.__dict__.copy()
-    if hasattr(rep.longrepr, "toterminal"):
-        if hasattr(rep.longrepr, "reprtraceback") and hasattr(
-            rep.longrepr, "reprcrash"
-        ):
-            d["longrepr"] = disassembled_report(rep)
-        else:
-            d["longrepr"] = str(rep.longrepr)
-    else:
-        d["longrepr"] = rep.longrepr
-    for name in d:
-        if isinstance(d[name], py.path.local):
-            d[name] = str(d[name])
-        elif name == "result":
-            d[name] = None  # for now
-    return d
-
-
 def serialize_warning_message(warning_message):
     if isinstance(warning_message.message, Warning):
         message_module = type(warning_message.message).__module__
@@ -218,7 +175,15 @@ def serialize_warning_message(warning_message):
     for attr_name in warning_message._WARNING_DETAILS:
         if attr_name in ("message", "category"):
             continue
-        result[attr_name] = getattr(warning_message, attr_name)
+        attr = getattr(warning_message, attr_name)
+        # Check if we can serialize the warning detail, marking `None` otherwise
+        # Note that we need to define the attr (even as `None`) to allow deserializing
+        try:
+            dumps(attr)
+        except DumpError:
+            result[attr_name] = repr(attr)
+        else:
+            result[attr_name] = attr
     return result
 
 
@@ -236,39 +201,44 @@ def getinfodict():
 
 
 def remote_initconfig(option_dict, args):
-    from _pytest.config import Config
-
     option_dict["plugins"].append("no:terminal")
-    config = Config.fromdictargs(option_dict, args)
+    return Config.fromdictargs(option_dict, args)
+
+
+def setup_config(config, basetemp):
     config.option.looponfail = False
     config.option.usepdb = False
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
     config.option.maxprocesses = None
-    config.args = args
+    config.option.basetemp = basetemp
-    return config
 
 
 if __name__ == "__channelexec__":
     channel = channel  # noqa
-    workerinput, args, option_dict = channel.receive()
+    workerinput, args, option_dict, change_sys_path = channel.receive()
+
+    if change_sys_path:
         importpath = os.getcwd()
-    sys.path.insert(0, importpath)  # XXX only for remote situations
+        sys.path.insert(0, importpath)
         os.environ["PYTHONPATH"] = (
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
+
+    os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
     os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
-    # os.environ['PYTHONPATH'] = importpath
-    import py
 
+    if hasattr(Config, "InvocationParams"):
+        config = _prepareconfig(args, None)
+    else:
         config = remote_initconfig(option_dict, args)
+        config.args = args
+
+    setup_config(config, option_dict.get("basetemp"))
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
     config.workerinput = workerinput
     config.workeroutput = {}
-    # TODO: deprecated name, backward compatibility only. Remove it in future
-    config.slaveinput = config.workerinput
-    config.slaveoutput = config.workeroutput
     interactor = WorkerInteractor(config, channel)
     config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 from difflib import unified_diff
 
 
@@ -13,9 +12,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
 
     diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
     error_message = (
-        u"Different tests were collected between {from_id} and {to_id}. "
+        "Different tests were collected between {from_id} and {to_id}. "
-        u"The difference is:\n"
+        "The difference is:\n"
-        u"{diff}"
+        "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
     msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
     return msg

src/xdist/scheduler/each.py

@@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class EachScheduling(object):
+class EachScheduling:
     """Implement scheduling of test items on all nodes
 
     If a node gets added after the test run is started then it is

src/xdist/scheduler/load.py

@@ -7,7 +7,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class LoadScheduling(object):
+class LoadScheduling:
     """Implement load scheduling across nodes.
 
     This distributes the tests collected across all nodes so each test

src/xdist/scheduler/loadfile.py

@@ -21,7 +21,7 @@ class LoadFileScheduling(LoadScopeScheduling):
     """
 
     def __init__(self, config, log=None):
-        super(LoadFileScheduling, self).__init__(config, log)
+        super().__init__(config, log)
         if log is None:
             self.log = Producer("loadfilesched")
         else:

src/xdist/scheduler/loadscope.py

@@ -6,7 +6,7 @@ from xdist.report import report_collection_diff
 from xdist.workermanage import parse_spec_config
 
 
-class LoadScopeScheduling(object):
+class LoadScopeScheduling:
     """Implement load scheduling across nodes, but grouping test by scope.
 
     This distributes the tests collected across all nodes so each test is run
@@ -358,12 +358,12 @@ class LoadScopeScheduling(object):
         extra_nodes = len(self.nodes) - len(self.workqueue)
 
         if extra_nodes > 0:
-            self.log("Shuting down {0} nodes".format(extra_nodes))
+            self.log("Shuting down {} nodes".format(extra_nodes))
 
             for _ in range(extra_nodes):
                 unused_node, assigned = self.assigned_work.popitem(last=True)
 
-                self.log("Shuting down unused node {0}".format(unused_node))
+                self.log("Shuting down unused node {}".format(unused_node))
                 unused_node.shutdown()
 
         # Assign initial workload

src/xdist/workermanage.py

@@ -1,9 +1,8 @@
-from __future__ import print_function
 import fnmatch
 import os
 import re
 import sys
-import threading
+import uuid
 
 import py
 import pytest
@@ -11,8 +10,6 @@ import execnet
 
 import xdist.remote
 
-from _pytest import runner  # XXX load dynamically
-
 
 def parse_spec_config(config):
     xspeclist = []
@@ -31,14 +28,16 @@ def parse_spec_config(config):
     return xspeclist
 
 
-class NodeManager(object):
+class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
-        self._nodesready = threading.Event()
         self.trace = self.config.trace.get("nodemanager")
+        self.testrunuid = self.config.getoption("testrunuid")
+        if self.testrunuid is None:
+            self.testrunuid = uuid.uuid4().hex
         self.group = execnet.Group()
         if specs is None:
             specs = self._getxspecs()
@@ -63,10 +62,7 @@ class NodeManager(object):
     def setup_nodes(self, putevent):
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        nodes = []
+        return [self.setup_node(spec, putevent) for spec in self.specs]
-        for spec in self.specs:
-            nodes.append(self.setup_node(spec, putevent))
-        return nodes
 
     def setup_node(self, spec, putevent):
         gw = self.group.makegateway(spec)
@@ -93,8 +89,16 @@ class NodeManager(object):
         import pytest
         import _pytest
 
-        pytestpath = pytest.__file__.rstrip("co")
+        def get_dir(p):
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
         config = self.config
         candidates = [py._pydir, pytestpath, pytestdir]
         candidates += config.option.rsyncdir
@@ -105,7 +109,7 @@ class NodeManager(object):
         for root in candidates:
             root = py.path.local(root).realpath()
             if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: %r" % (root,))
+                raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root))
             if root not in roots:
                 roots.append(root)
         return roots
@@ -116,7 +120,10 @@ class NodeManager(object):
         ignores += self.config.option.rsyncignore
         ignores += self.config.getini("rsyncignore")
 
-        return {"ignores": ignores, "verbose": self.config.option.verbose}
+        return {
+            "ignores": ignores,
+            "verbose": getattr(self.config.option, "verbose", 0),
+        }
 
     def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
         """Perform rsync to remote hosts for node."""
@@ -155,12 +162,11 @@ class HostRSync(execnet.RSync):
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
-        self._ignores = []
         ignores = kwargs.pop("ignores", None) or []
-        for x in ignores:
+        self._ignores = [
-            x = getattr(x, "strpath", x)
+            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
-            self._ignores.append(re.compile(fnmatch.translate(x)))
+        ]
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
+        super().__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
         path = py.path.local(path)
@@ -172,15 +178,13 @@ class HostRSync(execnet.RSync):
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
-            gateway, remotepath, finishedcallback=finished, delete=True
-        )
 
     def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
+        if self._verbose > 0:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
-            print("%s:%s <= %s" % (gateway.spec, remotepath, path))
+            print("{}:{} <= {}".format(gateway.spec, remotepath, path))
 
 
 def make_reltoroot(roots, args):
@@ -190,21 +194,30 @@ def make_reltoroot(roots, args):
     for arg in args:
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
+        if not fspath.exists():
+            result.append(arg)
+            continue
         for root in roots:
             x = fspath.relto(root)
             if x or fspath == root:
                 parts[0] = root.basename + "/" + x
                 break
         else:
-            raise ValueError("arg %s not relative to an rsync root" % (arg,))
+            raise ValueError("arg {} not relative to an rsync root".format(arg))
         result.append(splitcode.join(parts))
     return result
 
 
-class WorkerController(object):
+class WorkerController:
     ENDMARK = -1
 
+    class RemoteHook:
+        @pytest.mark.trylast
+        def pytest_xdist_getremotemodule(self):
+            return xdist.remote
+
     def __init__(self, nodemanager, gateway, config, putevent):
+        config.pluginmanager.register(self.RemoteHook())
         self.nodemanager = nodemanager
         self.putevent = putevent
         self.gateway = gateway
@@ -212,12 +225,9 @@ class WorkerController(object):
         self.workerinput = {
             "workerid": gateway.id,
             "workercount": len(nodemanager.specs),
-            "slaveid": gateway.id,
+            "testrunuid": nodemanager.testrunuid,
-            "slavecount": len(nodemanager.specs),
             "mainargv": sys.argv,
         }
-        # TODO: deprecated name, backward compatibility only. Remove it in future
-        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
         self.log = py.log.Producer("workerctl-%s" % gateway.id)
@@ -225,7 +235,7 @@ class WorkerController(object):
             py.log.setconsumer(self.log._keywords, None)
 
     def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__, self.gateway.id)
+        return "<{} {}>".format(self.__class__.__name__, self.gateway.id)
 
     @property
     def shutting_down(self):
@@ -234,18 +244,27 @@ class WorkerController(object):
     def setup(self):
         self.log("setting up worker session")
         spec = self.gateway.spec
+        if hasattr(self.config, "invocation_params"):
+            args = [str(x) for x in self.config.invocation_params.args or ()]
+            option_dict = {}
+        else:
             args = self.config.args
+            option_dict = vars(self.config.option)
         if not spec.popen or spec.chdir:
             args = make_reltoroot(self.nodemanager.roots, args)
-        option_dict = vars(self.config.option)
         if spec.popen:
             name = "popen-%s" % self.gateway.id
             if hasattr(self.config, "_tmpdirhandler"):
                 basetemp = self.config._tmpdirhandler.getbasetemp()
                 option_dict["basetemp"] = str(basetemp.join(name))
         self.config.hook.pytest_configure_node(node=self)
-        self.channel = self.gateway.remote_exec(xdist.remote)
+
-        self.channel.send((self.workerinput, args, option_dict))
+        remote_module = self.config.hook.pytest_xdist_getremotemodule()
+        self.channel = self.gateway.remote_exec(remote_module)
+        # change sys.path only for remote workers
+        change_sys_path = not self.gateway.spec.popen
+        self.channel.send((self.workerinput, args, option_dict, change_sys_path))
+
         if self.putevent:
             self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
 
@@ -270,17 +289,17 @@ class WorkerController(object):
         if not self._down:
             try:
                 self.sendcommand("shutdown")
-            except (IOError, OSError):
+            except OSError:
                 pass
             self._shutdown_sent = True
 
     def sendcommand(self, name, **kwargs):
         """ send a named parametrized command to the other side. """
-        self.log("sending command %s(**%s)" % (name, kwargs))
+        self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
     def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing %s(**%s)" % (eventname, kwargs))
+        self.log("queuing {}(**{})".format(eventname, kwargs))
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
@@ -302,18 +321,22 @@ class WorkerController(object):
                 return
             eventname, kwargs = eventcall
             if eventname in ("collectionstart",):
-                self.log("ignoring %s(%s)" % (eventname, kwargs))
+                self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "internal_error":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
-                self.workeroutput = self.slaveoutput = kwargs["workeroutput"]
+                self.workeroutput = kwargs["workeroutput"]
                 self.notify_inproc("workerfinished", node=self)
             elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in ("testreport", "collectreport", "teardownreport"):
                 item_index = kwargs.pop("item_index", None)
-                rep = unserialize_report(eventname, kwargs["data"])
+                rep = self.config.hook.pytest_report_from_serializable(
+                    config=self.config, data=kwargs["data"]
+                )
                 if item_index is not None:
                     rep.item_index = item_index
                 self.notify_inproc(eventname, node=self, rep=rep)
@@ -339,92 +362,32 @@ class WorkerController(object):
                     when=kwargs["when"],
                     item=kwargs["item"],
                 )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
             else:
-                raise ValueError("unknown event: %s" % (eventname,))
+                raise ValueError("unknown event: {}".format(eventname))
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise
         except:  # noqa
             from _pytest._code import ExceptionInfo
 
-            # ExceptionInfo API changed in pytest 4.1
-            if hasattr(ExceptionInfo, "from_current"):
             excinfo = ExceptionInfo.from_current()
-            else:
-                excinfo = ExceptionInfo()
             print("!" * 20, excinfo)
             self.config.notify_exception(excinfo)
             self.shutdown()
             self.notify_inproc("errordown", node=self, error=excinfo)
 
 
-def unserialize_report(name, reportdict):
-    def assembled_report(reportdict):
-        from _pytest._code.code import (
-            ReprEntry,
-            ReprEntryNative,
-            ReprExceptionInfo,
-            ReprFileLocation,
-            ReprFuncArgs,
-            ReprLocals,
-            ReprTraceback,
-        )
-
-        if reportdict["longrepr"]:
-            if (
-                "reprcrash" in reportdict["longrepr"]
-                and "reprtraceback" in reportdict["longrepr"]
-            ):
-
-                reprtraceback = reportdict["longrepr"]["reprtraceback"]
-                reprcrash = reportdict["longrepr"]["reprcrash"]
-
-                unserialized_entries = []
-                reprentry = None
-                for entry_data in reprtraceback["reprentries"]:
-                    data = entry_data["data"]
-                    entry_type = entry_data["type"]
-                    if entry_type == "ReprEntry":
-                        reprfuncargs = None
-                        reprfileloc = None
-                        reprlocals = None
-                        if data["reprfuncargs"]:
-                            reprfuncargs = ReprFuncArgs(**data["reprfuncargs"])
-                        if data["reprfileloc"]:
-                            reprfileloc = ReprFileLocation(**data["reprfileloc"])
-                        if data["reprlocals"]:
-                            reprlocals = ReprLocals(data["reprlocals"]["lines"])
-
-                        reprentry = ReprEntry(
-                            lines=data["lines"],
-                            reprfuncargs=reprfuncargs,
-                            reprlocals=reprlocals,
-                            filelocrepr=reprfileloc,
-                            style=data["style"],
-                        )
-                    elif entry_type == "ReprEntryNative":
-                        reprentry = ReprEntryNative(data["lines"])
-                    else:
-                        report_unserialization_failure(entry_type, name, reportdict)
-                    unserialized_entries.append(reprentry)
-                reprtraceback["reprentries"] = unserialized_entries
-
-                exception_info = ReprExceptionInfo(
-                    reprtraceback=ReprTraceback(**reprtraceback),
-                    reprcrash=ReprFileLocation(**reprcrash),
-                )
-
-                for section in reportdict["longrepr"]["sections"]:
-                    exception_info.addsection(*section)
-                reportdict["longrepr"] = exception_info
-        return reportdict
-
-    if name == "testreport":
-        return runner.TestReport(**assembled_report(reportdict))
-    elif name == "collectreport":
-        return runner.CollectReport(**assembled_report(reportdict))
-
-
 def unserialize_warning_message(data):
     import warnings
     import importlib
@@ -465,17 +428,3 @@ def unserialize_warning_message(data):
         kwargs[attr_name] = data[attr_name]
 
     return warnings.WarningMessage(**kwargs)
-
-
-def report_unserialization_failure(type_name, report_name, reportdict):
-    from pprint import pprint
-
-    url = "https://github.com/pytest-dev/pytest-xdist/issues"
-    stream = py.io.TextIO()
-    pprint("-" * 100, stream=stream)
-    pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream)
-    pprint("report_name: %s" % report_name, stream=stream)
-    pprint(reportdict, stream=stream)
-    pprint("Please report this bug at %s" % url, stream=stream)
-    pprint("-" * 100, stream=stream)
-    assert 0, stream.getvalue()

testing/acceptance_test.py

@@ -1,9 +1,9 @@
 import os
 import re
-import textwrap
 
 import py
 import pytest
+import xdist
 
 
 class TestDistribution:
@@ -185,16 +185,46 @@ class TestDistribution:
         )
         assert result.ret == 1
 
-    def test_distribution_rsyncdirs_example(self, testdir):
+    def test_distribution_rsyncdirs_example(self, testdir, monkeypatch):
+        # use a custom plugin that has a custom command-line option to ensure
+        # this is propagated to workers (see #491)
+        testdir.makepyfile(
+            **{
+                "myplugin/src/foobarplugin.py": """
+            from __future__ import print_function
+
+            import os
+            import sys
+            import pytest
+
+            def pytest_addoption(parser):
+                parser.addoption("--foobar", action="store", dest="foobar_opt")
+
+            @pytest.mark.tryfirst
+            def pytest_load_initial_conftests(early_config):
+                opt = early_config.known_args_namespace.foobar_opt
+                print("--foobar=%s active! [%s]" % (opt, os.getpid()), file=sys.stderr)
+            """
+            }
+        )
+        assert (testdir.tmpdir / "myplugin/src/foobarplugin.py").check(file=1)
+        monkeypatch.setenv(
+            "PYTHONPATH", str(testdir.tmpdir / "myplugin/src"), prepend=os.pathsep
+        )
+
         source = testdir.mkdir("source")
         dest = testdir.mkdir("dest")
         subdir = source.mkdir("example_pkg")
         subdir.ensure("__init__.py")
         p = subdir.join("test_one.py")
         p.write("def test_5():\n  assert not __file__.startswith(%r)" % str(p))
-        result = testdir.runpytest(
+        result = testdir.runpytest_subprocess(
             "-v",
             "-d",
+            "-s",
+            "-pfoobarplugin",
+            "--foobar=123",
+            "--dist=load",
             "--rsyncdir=%(subdir)s" % locals(),
             "--tx=popen//chdir=%(dest)s" % locals(),
             p,
@@ -208,24 +238,9 @@ class TestDistribution:
                 "*1 passed*",
             ]
         )
+        result.stderr.fnmatch_lines(["--foobar=123 active! *"])
         assert dest.join(subdir.basename).check(dir=1)
 
-    def test_backward_compatibility_worker_terminology(self, testdir):
-        """Ensure that we still support "config.slaveinput" for backward compatibility (#234).
-
-        Keep in mind that removing this compatibility will break a ton of plugins and user code.
-        """
-        testdir.makepyfile(
-            """
-            def test(pytestconfig):
-                assert hasattr(pytestconfig, 'slaveinput')
-                assert hasattr(pytestconfig, 'workerinput')
-        """
-        )
-        result = testdir.runpytest("-n1")
-        result.stdout.fnmatch_lines("*1 passed*")
-        assert result.ret == 0
-
     def test_data_exchange(self, testdir):
         testdir.makeconftest(
             """
@@ -296,7 +311,7 @@ class TestDistribution:
                 time.sleep(10)
         """
         )
-        child = testdir.spawn_pytest("-n1 -v")
+        child = testdir.spawn_pytest("-n1 -v", expect_timeout=30.0)
         child.expect(".*test_sleep.*")
         child.kill(2)  # keyboard interrupt
         child.expect(".*KeyboardInterrupt.*")
@@ -489,9 +504,8 @@ def test_session_hooks(testdir):
                 name = "worker"
             else:
                 name = "master"
-            f = open(name, "w")
+            with open(name, "w") as f:
                 f.write("xy")
-            f.close()
             # let's fail on the worker
             if name == "worker":
                 raise ValueError(42)
@@ -559,6 +573,29 @@ def test_fixture_teardown_failure(testdir):
     assert result.ret
 
 
+def test_config_initialization(testdir, monkeypatch, pytestconfig):
+    """Ensure workers and master are initialized consistently. Integration test for #445"""
+    testdir.makepyfile(
+        **{
+            "dir_a/test_foo.py": """
+                def test_1(request):
+                    assert request.config.option.verbose == 2
+        """
+        }
+    )
+    testdir.makefile(
+        ".ini",
+        myconfig="""
+        [pytest]
+        testpaths=dir_a
+    """,
+    )
+    monkeypatch.setenv("PYTEST_ADDOPTS", "-v")
+    result = testdir.runpytest("-n2", "-c", "myconfig.ini", "-v")
+    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*", "*= 1 passed in *"])
+    assert result.ret == 0
+
+
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
 def test_crashing_item(testdir, when):
     """Ensure crashing item is correctly reported during all testing stages"""
@@ -634,32 +671,6 @@ def test_skipping(testdir):
     result.stdout.fnmatch_lines(["*hello*", "*1 skipped*"])
 
 
-def test_issue34_pluginloading_in_subprocess(testdir):
-    import _pytest.hookspec
-
-    if not hasattr(_pytest.hookspec, "pytest_namespace"):
-        pytest.skip("this pytest version no longer supports pytest_namespace()")
-
-    testdir.tmpdir.join("plugin123.py").write(
-        textwrap.dedent(
-            """
-        def pytest_namespace():
-            return {'sample_variable': 'testing'}
-    """
-        )
-    )
-    testdir.makepyfile(
-        """
-        import pytest
-        def test_hello():
-            assert pytest.sample_variable == "testing"
-    """
-    )
-    result = testdir.runpytest_subprocess("-n1", "-p", "plugin123")
-    assert result.ret == 0
-    result.stdout.fnmatch_lines(["*1 passed*"])
-
-
 def test_fixture_scope_caching_issue503(testdir):
     p1 = testdir.makepyfile(
         """
@@ -737,28 +748,48 @@ def test_sub_plugins_disabled(testdir, plugin):
 
 class TestWarnings:
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
+    def test_warnings(self, testdir, n):
-    def test_warnings(self, testdir, n, warn_type):
-        if warn_type == "builtin":
-            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
-        elif warn_type == "pytest":
-            warn_code = """request.config.warn('', 'this is a warning',
-                           fslocation=py.path.local())"""
-        else:
-            assert False
         testdir.makepyfile(
             """
             import warnings, py, pytest
 
             @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
             def test_func(request):
-                {warn_code}
+                warnings.warn(UserWarning('this is a warning'))
-        """.format(
+            """
-                warn_code=warn_code
-            )
         )
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warning*"])
+
+    def test_warning_captured_deprecated_in_pytest_6(self, testdir):
+        """
+        Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
+        """
+        import _pytest.hookspec
+
+        if not hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+            pytest.skip("test requires pytest 6.0+")
+
+        testdir.makeconftest(
+            """
+            def pytest_warning_captured(warning_message):
+                if warning_message == "my custom worker warning":
+                    assert False, (
+                        "this hook should not be called from workers "
+                        "in this version: {}"
+                    ).format(warning_message)
+        """
+        )
+        testdir.makepyfile(
+            """
+            import warnings
+            def test():
+                warnings.warn("my custom worker warning")
+        """
+        )
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines(["*1 passed*"])
+        result.stdout.no_fnmatch_line("*this hook should not be called in this version")
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_custom_subclass(self, testdir, n):
@@ -782,7 +813,7 @@ class TestWarnings:
         )
         testdir.syspathinsert()
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warning*"])
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_unserializable_arguments(self, testdir, n):
@@ -799,7 +830,37 @@ class TestWarnings:
         )
         testdir.syspathinsert()
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warning*"])
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_unserializable_warning_details(self, testdir, n):
+        """Check that warnings with unserializable _WARNING_DETAILS are
+        handled correctly (#379).
+        """
+        testdir.makepyfile(
+            """
+            import warnings, pytest
+            import socket
+            import gc
+            def abuse_socket():
+                s = socket.socket()
+                del s
+
+            # Deliberately provoke a ResourceWarning for an unclosed socket.
+            # The socket itself will end up attached as a value in
+            # _WARNING_DETAIL. We need to test that it is not serialized
+            # (it can't be, so the test will fail if we try to).
+            @pytest.mark.filterwarnings('always')
+            def test_func(tmpdir):
+                abuse_socket()
+                gc.collect()
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(
+            ["*ResourceWarning*unclosed*", "*1 passed, 1 warning*"]
+        )
 
 
 class TestNodeFailure:
@@ -814,8 +875,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -833,8 +894,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n2")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*3 passed*",
             ]
         )
@@ -850,8 +911,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "--dist=each", "--tx=popen")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -887,14 +948,35 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker*",
-                "*Maximum crashed workers reached: 1*",
+                "maximum crashed workers reached: 1*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*2 failed*2 passed*",
             ]
         )
 
+    def test_max_worker_restart_tests_queued(self, testdir):
+        f = testdir.makepyfile(
+            """
+            import os, pytest
+            @pytest.mark.parametrize('i', range(10))
+            def test(i): os._exit(1)
+        """
+        )
+        res = testdir.runpytest(f, "-n2", "--max-worker-restart=3")
+        res.stdout.fnmatch_lines(
+            [
+                "replacing crashed worker*",
+                "maximum crashed workers reached: 3*",
+                "worker*crashed while running*",
+                "worker*crashed while running*",
+                "* xdist: maximum crashed workers reached: 3 *",
+                "* 4 failed in *",
+            ]
+        )
+        assert "INTERNALERROR" not in res.stdout.str()
+
     def test_max_worker_restart_die(self, testdir):
         f = testdir.makepyfile(
             """
@@ -904,7 +986,10 @@ class TestNodeFailure:
         )
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
-            ["*Unexpectedly no active workers*", "*INTERNALERROR*"]
+            [
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* no tests ran in *",
+            ]
         )
 
     def test_disable_restart(self, testdir):
@@ -919,9 +1004,10 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
             [
-                "*Worker restarting disabled*",
+                "worker gw* crashed and worker restarting disabled",
-                "*Worker*crashed while running*",
+                "*worker*crashed while running*",
-                "*1 failed*2 passed*",
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* 1 failed, 2 passed in *",
             ]
         )
 
@@ -951,6 +1037,29 @@ def test_worker_id_fixture(testdir, n):
         assert worker_ids == {"gw0", "gw1"}
 
 
+@pytest.mark.parametrize("n", [0, 2])
+def test_testrun_uid_fixture(testdir, n):
+    import glob
+
+    f = testdir.makepyfile(
+        """
+        import pytest
+        @pytest.mark.parametrize("run_num", range(2))
+        def test_testrun_uid1(testrun_uid, run_num):
+            with open("testrun_uid%s.txt" % run_num, "w") as f:
+                f.write(testrun_uid)
+    """
+    )
+    result = testdir.runpytest(f, "-n%d" % n)
+    result.stdout.fnmatch_lines("* 2 passed in *")
+    testrun_uids = set()
+    for fname in glob.glob(str(testdir.tmpdir.join("*.txt"))):
+        with open(fname) as f:
+            testrun_uids.add(f.read().strip())
+    assert len(testrun_uids) == 1
+    assert len(testrun_uids.pop()) == 32
+
+
 @pytest.mark.parametrize("tb", ["auto", "long", "short", "no", "line", "native"])
 def test_error_report_styles(testdir, tb):
     testdir.makepyfile(
@@ -990,6 +1099,22 @@ def test_color_yes_collection_on_non_atty(testdir, request):
     assert "gw0 C / gw1 C" not in result.stdout.str()
 
 
+def test_without_terminal_plugin(testdir, request):
+    """
+    No output when terminal plugin is disabled
+    """
+    testdir.makepyfile(
+        """
+        def test_1():
+            pass
+    """
+    )
+    result = testdir.runpytest("-p", "no:terminal", "-n2")
+    assert result.stdout.str() == ""
+    assert result.stderr.str() == ""
+    assert result.ret == 0
+
+
 def test_internal_error_with_maxfail(testdir):
     """
     Internal error when using --maxfail option (#62, #65).
@@ -1013,6 +1138,18 @@ def test_internal_error_with_maxfail(testdir):
     assert "INTERNALERROR" not in result.stderr.str()
 
 
+def test_internal_errors_propagate_to_master(testdir):
+    testdir.makeconftest(
+        """
+        def pytest_collection_modifyitems():
+            raise RuntimeError("Some runtime error")
+        """
+    )
+    testdir.makepyfile("def test(): pass")
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines(["*RuntimeError: Some runtime error*"])
+
+
 class TestLoadScope:
     def test_by_module(self, testdir):
         test_file = """
@@ -1227,7 +1364,7 @@ def parse_tests_and_workers_from_output(lines):
             r"""
             \[(gw\d)\]  # worker
             \s*
-            (?:\[\s*\d+%\])? # progress indicator (pytest >=3.3)
+            (?:\[\s*\d+%\])? # progress indicator
             \s(.*?)     # status string ("PASSED")
             \s(.*::.*)  # nodeid
         """,
@@ -1246,3 +1383,39 @@ def get_workers_and_test_count_by_prefix(prefix, lines, expected_status="PASSED"
         if expected_status == status and nodeid.startswith(prefix):
             result[worker] = result.get(worker, 0) + 1
     return result
+
+
+class TestAPI:
+    @pytest.fixture
+    def fake_request(self):
+        class FakeOption:
+            def __init__(self):
+                self.dist = "load"
+
+        class FakeConfig:
+            def __init__(self):
+                self.workerinput = {"workerid": "gw5"}
+                self.option = FakeOption()
+
+        class FakeRequest:
+            def __init__(self):
+                self.config = FakeConfig()
+
+        return FakeRequest()
+
+    def test_is_xdist_worker(self, fake_request):
+        assert xdist.is_xdist_worker(fake_request)
+        del fake_request.config.workerinput
+        assert not xdist.is_xdist_worker(fake_request)
+
+    def test_is_xdist_master(self, fake_request):
+        assert not xdist.is_xdist_master(fake_request)
+        del fake_request.config.workerinput
+        assert xdist.is_xdist_master(fake_request)
+        fake_request.config.option.dist = "no"
+        assert not xdist.is_xdist_master(fake_request)
+
+    def test_get_xdist_worker_id(self, fake_request):
+        assert xdist.get_xdist_worker_id(fake_request) == "gw5"
+        del fake_request.config.workerinput
+        assert xdist.get_xdist_worker_id(fake_request) == "master"

testing/conftest.py

@@ -2,20 +2,8 @@ import py
 import pytest
 import execnet
 
-
-@pytest.fixture(scope="session", autouse=True)
-def _ensure_imports():
-    # we import some modules because pytest-2.8's testdir fixture
-    # will unload all modules after each test and this cause
-    # (unknown) problems with execnet.Group()
-    execnet.Group
-    execnet.makegateway
-
-
 pytest_plugins = "pytester"
 
-# rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
-
 
 @pytest.fixture(autouse=True)
 def _divert_atexit(request, monkeypatch):
@@ -23,12 +11,16 @@ def _divert_atexit(request, monkeypatch):
 
     finalizers = []
 
-    def finish():
+    def fake_register(func, *args, **kwargs):
-        while finalizers:
+        finalizers.append((func, args, kwargs))
-            finalizers.pop()()
 
-    monkeypatch.setattr(atexit, "register", finalizers.append)
+    monkeypatch.setattr(atexit, "register", fake_register)
-    request.addfinalizer(finish)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
 
 
 def pytest_addoption(parser):
@@ -45,14 +37,6 @@ def specssh(request):
     return getspecssh(request.config)
 
 
-@pytest.fixture
-def testdir(testdir):
-    # pytest before 2.8 did not have a runpytest_subprocess
-    if not hasattr(testdir, "runpytest_subprocess"):
-        testdir.runpytest_subprocess = testdir.runpytest
-    return testdir
-
-
 # configuration information for tests
 def getgspecs(config):
     return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]

testing/test_dsession.py

@@ -1,4 +1,4 @@
-from xdist.dsession import DSession
+from xdist.dsession import DSession, get_default_max_worker_restart
 from xdist.report import report_collection_diff
 from xdist.scheduler import EachScheduling, LoadScheduling
 
@@ -203,7 +203,7 @@ class TestLoadScheduling:
         different test ids are collected by workers.
         """
 
-        class CollectHook(object):
+        class CollectHook:
             """
             Dummy hook that stores collection reports.
             """
@@ -268,6 +268,24 @@ def test_report_collection_diff_equal():
     assert report_collection_diff(from_collection, to_collection, 1, 2) is None
 
 
+def test_default_max_worker_restart():
+    class config:
+        class option:
+            maxworkerrestart = None
+            numprocesses = 0
+
+    assert get_default_max_worker_restart(config) is None
+
+    config.option.numprocesses = 2
+    assert get_default_max_worker_restart(config) == 8
+
+    config.option.maxworkerrestart = "1"
+    assert get_default_max_worker_restart(config) == 1
+
+    config.option.maxworkerrestart = "0"
+    assert get_default_max_worker_restart(config) == 0
+
+
 def test_report_collection_diff_different():
     """Test reporting of different collections."""
     from_collection = ["aaa", "bbb", "ccc", "YYY"]

testing/test_looponfail.py

@@ -284,7 +284,7 @@ class TestFunctional:
         )
         # p = testdir.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+        child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
@@ -311,7 +311,7 @@ class TestFunctional:
                 pass
         """
         )
-        child = testdir.spawn_pytest("-f %s" % p)
+        child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")

testing/test_newhooks.py

@@ -15,7 +15,7 @@ class TestHooks:
 
     def test_runtest_logreport(self, testdir):
         """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain "node", "nodeid" and "worker_id" attributes. (#8)
+        with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
         """
         testdir.makeconftest(
             """
@@ -23,12 +23,16 @@ class TestHooks:
                 if hasattr(report, 'node'):
                     if report.when == "call":
                         workerid = report.node.workerinput['workerid']
+                        testrunuid = report.node.workerinput['testrunuid']
                         if workerid != report.worker_id:
                             print("HOOK: Worker id mismatch: %s %s"
                                    % (workerid, report.worker_id))
+                        elif testrunuid != report.testrun_uid:
+                            print("HOOK: Testrun uid mismatch: %s %s"
+                                   % (testrunuid, report.testrun_uid))
                         else:
-                            print("HOOK: %s %s"
+                            print("HOOK: %s %s %s"
-                                   % (report.nodeid, report.worker_id))
+                                   % (report.nodeid, report.worker_id, report.testrun_uid))
         """
         )
         res = testdir.runpytest("-n1", "-s")

testing/test_plugin.py

@@ -1,7 +1,11 @@
+from contextlib import suppress
+
 import py
 import execnet
 from xdist.workermanage import NodeManager
 
+import pytest
+
 
 def test_dist_incompatibility_messages(testdir):
     result = testdir.runpytest("--pdb", "--looponfail")
@@ -36,6 +40,12 @@ def test_dist_options(testdir):
 
 def test_auto_detect_cpus(testdir, monkeypatch):
     import os
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    with suppress(ImportError):
+        import psutil
+
+        monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
 
     if hasattr(os, "sched_getaffinity"):
         monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
@@ -50,14 +60,48 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption("numprocesses") == 2
 
     config = testdir.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 99
 
+    config = testdir.parseconfigure("-nauto", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
+
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
     config = testdir.parseconfigure("-nauto")
+    check_options(config)
     assert config.getoption("numprocesses") == 2
 
 
+def test_auto_detect_cpus_psutil(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    psutil = pytest.importorskip("psutil")
+
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 42)
+
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
+def test_hook_auto_num_workers(testdir, monkeypatch):
+    from xdist.plugin import pytest_cmdline_main as check_options
+
+    testdir.makeconftest(
+        """
+        def pytest_xdist_auto_num_workers():
+            return 42
+    """
+    )
+    config = testdir.parseconfigure("-nauto")
+    check_options(config)
+    assert config.getoption("numprocesses") == 42
+
+
 def test_boxed_with_collect_only(testdir):
     from xdist.plugin import pytest_cmdline_main as check_options
 
@@ -89,6 +133,18 @@ def test_dsession_with_collect_only(testdir):
     assert not config.pluginmanager.hasplugin("dsession")
 
 
+def test_testrunuid_provided(testdir):
+    config = testdir.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    nm = NodeManager(config)
+    assert nm.testrunuid == "test123"
+
+
+def test_testrunuid_generated(testdir):
+    config = testdir.parseconfigure("--tx=popen")
+    nm = NodeManager(config)
+    assert len(nm.testrunuid) == 32
+
+
 class TestDistOptions:
     def test_getxspecs(self, testdir):
         config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")

testing/test_remote.py

@@ -2,13 +2,13 @@ import py
 import pprint
 import pytest
 import sys
+import uuid
 
-from xdist.workermanage import WorkerController, unserialize_report
+from xdist.workermanage import WorkerController
-from xdist.remote import serialize_report
 import execnet
 import marshal
 
-from six.moves.queue import Queue
+from queue import Queue
 
 WAIT_TIMEOUT = 10.0
 
@@ -26,7 +26,7 @@ class EventCall:
         self.name, self.kwargs = eventcall
 
     def __str__(self):
-        return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
+        return "<EventCall {}(**{})>".format(self.name, self.kwargs)
 
 
 class WorkerSetup:
@@ -45,6 +45,7 @@ class WorkerSetup:
         putevent = self.use_callback and self.events.put or None
 
         class DummyMananger:
+            testrunuid = uuid.uuid4().hex
             specs = [0, 1]
 
         self.slp = WorkerController(DummyMananger, self.gateway, config, putevent)
@@ -60,7 +61,7 @@ class WorkerSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping %s" % (ev,))
+            print("skipping {}".format(ev))
 
     def sendcommand(self, name, **kwargs):
         self.slp.sendcommand(name, **kwargs)
@@ -81,188 +82,17 @@ def test_remoteinitconfig(testdir):
     assert config2.pluginmanager.getplugin("terminal") in (-1, None)
 
 
-class TestReportSerialization:
-    def test_xdist_longrepr_to_str_issue_241(self, testdir):
-        testdir.makepyfile(
-            """
-            import os
-            def test_a(): assert False
-            def test_b(): pass
-        """
-        )
-        testdir.makeconftest(
-            """
-            def pytest_runtest_logreport(report):
-                print(report.longrepr)
-        """
-        )
-        res = testdir.runpytest("-n1", "-s")
-        res.stdout.fnmatch_lines(["*1 failed, 1 passed *"])
-
-    def test_xdist_report_longrepr_reprcrash_130(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-                    import py
-                    def test_fail(): assert False, 'Expected Message'
-                """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        added_section = ("Failure Metadata", str("metadata metadata"), "*")
-        rep.longrepr.sections.append(added_section)
-        d = serialize_report(rep)
-        check_marshallable(d)
-        a = unserialize_report("testreport", d)
-        # Check assembled == rep
-        assert a.__dict__.keys() == rep.__dict__.keys()
-        for key in rep.__dict__.keys():
-            if key != "longrepr":
-                assert getattr(a, key) == getattr(rep, key)
-        assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno
-        assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message
-        assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path
-        assert rep.longrepr.reprtraceback.entrysep == a.longrepr.reprtraceback.entrysep
-        assert (
-            rep.longrepr.reprtraceback.extraline == a.longrepr.reprtraceback.extraline
-        )
-        assert rep.longrepr.reprtraceback.style == a.longrepr.reprtraceback.style
-        assert rep.longrepr.sections == a.longrepr.sections
-        # Missing section attribute PR171
-        assert added_section in a.longrepr.sections
-
-    def test_reprentries_serialization_170(self, testdir):
-        from _pytest._code.code import ReprEntry
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry():
-                                x = 0
-                                assert x
-                        """,
-            "--showlocals",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntry)
-            assert rep_entries[i].lines == a_entries[i].lines
-            assert rep_entries[i].localssep == a_entries[i].localssep
-            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
-            assert (
-                rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
-            )
-            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
-            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
-            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
-            assert rep_entries[i].style == a_entries[i].style
-
-    def test_reprentries_serialization_196(self, testdir):
-        from _pytest._code.code import ReprEntryNative
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry_native():
-                                x = 0
-                                assert x
-                        """,
-            "--tb=native",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntryNative)
-            assert rep_entries[i].lines == a_entries[i].lines
-
-    def test_itemreport_outcomes(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-            import py
-            def test_pass(): pass
-            def test_fail(): 0/0
-            @py.test.mark.skipif("True")
-            def test_skip(): pass
-            def test_skip_imperative():
-                py.test.skip("hello")
-            @py.test.mark.xfail("True")
-            def test_xfail(): 0/0
-            def test_xfail_imperative():
-                py.test.xfail("hello")
-        """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 17  # with setup/teardown "passed" reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("testreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if newrep.skipped and not hasattr(newrep, "wasxfail"):
-                assert len(newrep.longrepr) == 3
-            assert newrep.outcome == rep.outcome
-            assert newrep.when == rep.when
-            assert newrep.keywords == rep.keywords
-            if rep.failed:
-                assert newrep.longreprtext == rep.longreprtext
-
-    def test_collectreport_passed(self, testdir):
-        reprec = testdir.inline_runsource("def test_func(): pass")
-        reports = reprec.getreports("pytest_collectreport")
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-
-    def test_collectreport_fail(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-    def test_extended_report_deserialization(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            rep.extra = True
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.extra
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-
 class TestWorkerInteractor:
-    def test_basic_collect_and_runtests(self, worker):
+    @pytest.fixture
+    def unserialize_report(self, pytestconfig):
+        def unserialize(data):
+            return pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
+
+        return unserialize
+
+    def test_basic_collect_and_runtests(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             def test_func():
@@ -287,14 +117,14 @@ class TestWorkerInteractor:
         ev = worker.popevent("testreport")  # setup
         ev = worker.popevent("testreport")
         assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, worker):
+    def test_remote_collect_skip(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             import pytest
@@ -306,25 +136,25 @@ class TestWorkerInteractor:
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
         assert rep.longrepr[2] == "Skipped: hello"
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, worker):
+    def test_remote_collect_fail(self, worker, unserialize_report):
         worker.testdir.makepyfile("""aasd qwe""")
         worker.setup()
         ev = worker.popevent("collectionstart")
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.failed
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, worker):
+    def test_runtests_all(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             def test_func(): pass
@@ -346,7 +176,7 @@ class TestWorkerInteractor:
             for i in range(3):  # setup/call/teardown
                 ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
-                rep = unserialize_report(ev.name, ev.kwargs["data"])
+                rep = unserialize_report(ev.kwargs["data"])
                 assert rep.nodeid.endswith(func)
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
@@ -392,6 +222,7 @@ def test_remote_env_vars(testdir):
         """
         import os
         def test():
+            assert len(os.environ['PYTEST_XDIST_TESTRUNUID']) == 32
             assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     """
@@ -458,6 +289,4 @@ def test_remote_usage_prog(testdir, request):
 
     result = testdir.runpytest_subprocess("-n1")
     assert result.ret == 1
-    result.stdout.fnmatch_lines(
+    result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
-        ["usage: pytest.py *", "pytest.py: error: my_usage_error"]
-    )

testing/test_workermanage.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import py
 import pytest
 import textwrap
@@ -35,7 +34,7 @@ def mysetup(tmpdir):
 
 @pytest.fixture
 def workercontroller(monkeypatch):
-    class MockController(object):
+    class MockController:
         def __init__(self, *args):
             pass
 
@@ -168,7 +167,7 @@ class TestNodeManager:
         # assert nodemanager.config.topdir == source == config.topdir
         nodemanager.makegateways()
         nodemanager.rsync_roots()
-        p, = nodemanager.gwmanager.multi_exec(
+        (p,) = nodemanager.gwmanager.multi_exec(
             "import os ; channel.send(os.getcwd())"
         ).receive_each()
         p = py.path.local(p)
@@ -197,6 +196,25 @@ class TestNodeManager:
             assert dest.join("dir1", "dir2", "hello").check()
             nodemanager.teardown_nodes()
 
+    @pytest.mark.parametrize(
+        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+    )
+    def test_rsync_report(
+        self, testdir, mysetup, workercontroller, capsys, flag, expects_report
+    ):
+        source, dest = mysetup.source, mysetup.dest
+        dir1 = mysetup.source.mkdir("dir1")
+        args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source
+        if flag:
+            args += (flag,)
+        nodemanager = NodeManager(testdir.parseconfig(*args))
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
     def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)
@@ -269,5 +287,5 @@ class TestNodeManager:
         reprec = testdir.inline_run(
             "-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir
         )
-        rep, = reprec.getreports("pytest_runtest_logreport")
+        (rep,) = reprec.getreports("pytest_runtest_logreport")
         assert rep.passed

tox.ini

@@ -1,23 +1,30 @@
 [tox]
 envlist=
   linting
-  py{27,34,35,36,37}-pytestlatest
+  py{35,36,37,38,39}-pytestlatest
-  py{27,36}-pytest{master,features}
+  py38-pytestmaster
+  py38-psutil
 
 [testenv]
-changedir=testing
+extras = testing
-passenv = USER USERNAME
 deps =
   pytestlatest: pytest
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
-  pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
-  filelock
 commands=
   pytest {posargs}
 
+[testenv:py38-psutil]
+extras =
+  testing
+  psutil
+deps = pytest
+commands =
+  pytest {posargs:-k psutil}
+
 [testenv:linting]
-skipsdist = True
+skip_install = True
 usedevelop = True
+passenv = PRE_COMMIT_HOME
 deps =
   pre-commit
 commands = pre-commit run --all-files --show-diff-on-failure
@@ -25,7 +32,7 @@ commands = pre-commit run --all-files --show-diff-on-failure
 [testenv:release]
 changedir=
 decription = do a release, required posarg of the version number
-basepython = python3.6
+basepython = python3.7
 skipsdist = True
 usedevelop = True
 passenv = *
@@ -35,7 +42,10 @@ commands =
   towncrier --version {posargs} --yes
 
 [pytest]
-addopts = -ra
+# pytest-services also defines a worker_id fixture, disable
+# it so they don't conflict with each other (#611).
+addopts = -ra -p no:pytest-services
+testpaths = testing
 
 [flake8]
 max-line-length = 120

xdist/__init__.py

@@ -1,3 +0,0 @@
-from xdist._version import version as __version__
-
-__all__ = ["__version__"]