Release 1.34.0

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>

.appveyor.yml

@@ -1,21 +1,20 @@
 environment:
   matrix:
   - TOXENV: "py27-pytestlatest"
-  - TOXENV: "py34-pytestlatest"
   - TOXENV: "py35-pytestlatest"
   - TOXENV: "py36-pytestlatest"
   - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py37-pytestmaster"
-  - TOXENV: "py37-pytestfeatures"
+  - TOXENV: "py38-pytestlatest"
+  - TOXENV: "py38-pytestmaster"
 
 install:
-  - C:\Python37\python -m pip install -U pip setuptools
-  - C:\Python37\python -m pip install -U tox setuptools_scm
+  - C:\Python38\python -m pip install -U pip setuptools virtualenv
+  - C:\Python38\python -m pip install -U tox setuptools_scm
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
-  - C:\Python37\python -m tox
+  - C:\Python38\python -m tox
 
 # We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
 # might as well save resources

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: 18.6b4
+    rev: 19.10b0
     hooks:
     -   id: black
         args: [--safe, --quiet]
         language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.3.0
+    rev: v2.4.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
@@ -21,4 +21,4 @@ repos:
         files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
-        python_version: python3.7
+        language_version: python3.7

.travis.yml

@@ -30,25 +30,23 @@ jobs:
       cache:
         directories:
           - $HOME/.cache/pre-commit
-    - python: '3.7'
-      env: TOXENV=py37-pytestlatest
+    - python: '3.8'
+      env: TOXENV=py38-pytestlatest
     - python: '2.7'
       env: TOXENV=py27-pytestlatest
 
     - stage: test
-      python: "3.4"
-      env: TOXENV=py34-pytestlatest
-    - python: "3.5"
+      python: "3.5"
       env: TOXENV=py35-pytestlatest
     - python: "3.6"
       env: TOXENV=py36-pytestlatest
     - python: "3.7"
-      env: TOXENV=py37-pytestmaster
-    - python: "3.7"
-      env: TOXENV=py37-pytestfeatures
+      env: TOXENV=py37-pytestlatest
+    - python: "3.8"
+      env: TOXENV=py38-pytestmaster
 
     - stage: deploy
-      python: '3.7'
+      python: '3.8'
       env:
       install: pip install -U setuptools setuptools_scm
       script: skip

CHANGELOG.rst

@@ -1,3 +1,92 @@
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
+pytest-xdist 1.32.0 (2020-05-03)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- `#475 <https://github.com/pytest-dev/pytest-xdist/issues/475>`_: Drop support for EOL Python 3.4.
+
+
+Features
+--------
+
+- `#524 <https://github.com/pytest-dev/pytest-xdist/issues/524>`_: Add `testrun_uid` fixture. This is a shared value that uniquely identifies a test run among all workers.
+  This also adds a `PYTEST_XDIST_TESTRUNUID` environment variable that is accessible within a test as well as a command line option `--testrunuid` to manually set the value from outside.
+
+
+pytest-xdist 1.31.0 (2019-12-19)
+================================
+
+Features
+--------
+
+- `#486 <https://github.com/pytest-dev/pytest-xdist/issues/486>`_: Add support for Python 3.8.
+
+
+Bug Fixes
+---------
+
+- `#491 <https://github.com/pytest-dev/pytest-xdist/issues/491>`_: Fix regression that caused custom plugin command-line arguments to be discarded when using ``--tx`` mode.
+
+
+
+pytest-xdist 1.30.0 (2019-10-01)
+================================
+
+Features
+--------
+
+- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
+  long-standing issues related to startup with the ``-c`` option.
+
+  Issues:
+
+  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
+  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
+
+  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
+
+
+Bug Fixes
+---------
+
+- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
+
+
 pytest-xdist 1.29.0 (2019-06-14)
 ================================
 
@@ -401,39 +490,6 @@ Improved Documentation
 - Added ``HOWTORELEASE`` documentation. (#155)
 
 
-..
-    You should *NOT* be adding new change log entries to this file, this
-    file is managed by towncrier. You *may* edit previous change logs to
-    fix problems like typo corrections or such.
-    To add a new change log entry, please see
-    https://pip.pypa.io/en/latest/development/#adding-a-news-entry
-    We named the news folder ``changelog``
-
-.. towncrier release notes start
-
-pytest-xdist 1.30.0 (2019-10-01)
-================================
-
-Features
---------
-
-- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
-  long-standing issues related to startup with the ``-c`` option.
-
-  Issues:
-
-  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
-  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
-
-  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
-
-
-Bug Fixes
----------
-
-- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
-
-
 1.17.0
 ------
 

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

README.rst

@@ -67,33 +67,88 @@ a checkout of the `pytest-xdist repository`_ ::
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, type::
+To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
 
-    pytest -n NUM
+    pytest -n NUMCPUS
 
-Especially for longer running tests or tests requiring
-a lot of I/O this can lead to considerable speed ups. This option can
-also be set to ``auto`` for automatic detection of the number of CPUs.
+Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
+can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
 
-If a test crashes the interpreter, pytest-xdist will automatically restart
-that worker and report the failure as usual. You can use the
-``--max-worker-restart`` option to limit the number of workers that can
-be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.
 
-By default, the ``-n`` option will send pending tests to any worker that is available, without
-any guaranteed order, but you can control this with these options:
+By default, using ``--numprocesses`` will send pending tests to any worker that
+is available, without any guaranteed order. You can change the test
+distribution algorithm this with the ``--dist`` option. It takes these values:
 
-* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
-  by **class** for *test methods*, then each group will be sent to an available worker,
-  guaranteeing that all tests in a group run in the same process. This can be useful if you have
-  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
-  with grouping by class takes priority over grouping by module.
-  This feature was added in version ``1.19``.
+* ``--dist no``: The default algorithm, distributing one test at a time.
 
-* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
-  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
-  in version ``1.21``.
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
 
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
+
+Making session-scoped fixtures execute only once
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``pytest-xdist`` is designed so that each worker process will perform its own collection and execute
+a subset of all tests. This means that tests in different processes requesting a high-level
+scoped fixture (for example ``session``) will execute the fixture code more than once, which
+breaks expectations and might be undesired in certain situations.
+
+While ``pytest-xdist`` does not have a builtin support for ensuring a session-scoped fixture is
+executed exactly once, this can be achieved by using a lock file for inter-process communication.
+
+The example below needs to execute the fixture ``session_data`` only once (because it is
+resource intensive, or needs to execute only once to define configuration options, etc), so it makes
+use of a `FileLock <https://pypi.org/project/filelock/>`_ to produce the fixture data only once
+when the first process requests the fixture, while the other processes will then read
+the data from a file.
+
+Here is the code:
+
+.. code-block:: python
+
+    import json
+
+    import pytest
+    from filelock import FileLock
+
+
+    @pytest.fixture(scope="session")
+    def session_data(tmp_path_factory, worker_id):
+        if not worker_id:
+            # not executing in with multiple workers, just produce the data and let
+            # pytest's fixture caching do its job
+            return produce_expensive_data()
+
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
+
+        fn = root_tmp_dir / "data.json"
+        with FileLock(str(fn) + ".lock"):
+            if fn.is_file():
+                data = json.loads(fn.read_text())
+            else:
+                data = produce_expensive_data()
+                fn.write_text(json.dumps(data))
+        return data
+
+
+The example above can also be use in cases a fixture needs to execute exactly once per test session, like
+initializing a database service and populating initial tables.
+
+This technique might not work for every case, but should be a starting point for many situations
+where executing a high-scope fixture exactly once is important.
 
 Running tests in a Python subprocess
 ------------------------------------
@@ -223,8 +278,43 @@ defined:
 The information about the worker_id in a test is stored in the ``TestReport`` as
 well, under the ``worker_id`` attribute.
 
-Acessing ``sys.argv`` from the master node in workers
------------------------------------------------------
+
+Uniquely identifying the current test run
+-----------------------------------------
+
+*New in version 1.32.*
+
+If you need to globally distinguish one test run from others in your
+workers, you can use the ``testrun_uid`` fixture. For instance, let's say you
+wanted to create a separate database for each test run:
+
+.. code-block:: python
+
+    import pytest
+    from posix_ipc import Semaphore, O_CREAT
+
+    @pytest.fixture(scope="session", autouse=True)
+    def create_unique_database(testrun_uid):
+        """ create a unique database for this particular test run """
+        database_url = f"psql://myapp-{testrun_uid}"
+
+        with Semaphore(f"/{testrun_uid}-lock", flags=O_CREAT, initial_value=1):
+            if not database_exists(database_url):
+                create_database(database_url)
+
+    @pytest.fixture()
+    def db(testrun_uid):
+        """ retrieve unique database """
+        database_url = f"psql://myapp-{testrun_uid}"
+        return database_get_instance(database_url)
+
+
+Additionally, during a test run, the following environment variable is defined:
+
+* ``PYTEST_XDIST_TESTRUNUID``: the unique id of the test run.
+
+Accessing ``sys.argv`` from the master node in workers
+------------------------------------------------------
 
 To access the ``sys.argv`` passed to the command-line of the master node, use
 ``request.config.workerinput["mainargv"]``.

setup.py

@@ -24,7 +24,7 @@ setup(
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
+    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
     install_requires=install_requires,
     setup_requires=["setuptools_scm"],
     classifiers=[
@@ -42,9 +42,9 @@ setup(
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
 )

src/xdist/dsession.py

@@ -169,7 +169,7 @@ class DSession(object):
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
         if node.workeroutput["exitstatus"] == 2:  # keyboard-interrupt
-            self.shouldstop = "%s received keyboard-interrupt" % (node,)
+            self.shouldstop = "{} received keyboard-interrupt".format(node)
             self.worker_errordown(node, "keyboard-interrupt")
             return
         if node in self.sched.nodes:
@@ -277,10 +277,17 @@ class DSession(object):
         self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
     def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_logwarning hook."""
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 
@@ -321,7 +328,7 @@ class DSession(object):
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "worker %r crashed while running %r" % (worker.gateway.id, nodeid)
+        msg = "worker {!r} crashed while running {!r}".format(worker.gateway.id, nodeid)
         rep = runner.TestReport(
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
@@ -351,7 +358,9 @@ class TerminalDistReporter(object):
 
     def getstatus(self):
         if self.config.option.verbose >= 0:
-            parts = ["%s %s" % (spec.id, self._status[spec.id]) for spec in self._specs]
+            parts = [
+                "{} {}".format(spec.id, self._status[spec.id]) for spec in self._specs
+            ]
             return " / ".join(parts)
         else:
             return "bringing up nodes..."
@@ -386,14 +395,16 @@ class TerminalDistReporter(object):
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
             d = node.workerinfo
-            infoline = "[%s] Python %s" % (d["id"], d["version"].replace("\n", " -- "))
+            infoline = "[{}] Python {}".format(
+                d["id"], d["version"].replace("\n", " -- ")
+            )
             self.rewrite(infoline, newline=True)
         self.setstatus(node.gateway.spec, "ok")
 
     def pytest_testnodedown(self, node, error):
         if not error:
             return
-        self.write_line("[%s] node down: %s" % (node.gateway.id, error))
+        self.write_line("[{}] node down: {}".format(node.gateway.id, error))
 
 
 def get_default_max_worker_restart(config):

src/xdist/looponfail.py

@@ -136,7 +136,7 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
                 tr.line(report, red=True)
     tr.sep("#", "waiting for changes", bold=True)
     for rootdir in rootdirs:
-        tr.line("### Watching:   %s" % (rootdir,), bold=True)
+        tr.line("### Watching:   {}".format(rootdir), bold=True)
 
 
 def init_worker_session(channel, args, option_dict):

src/xdist/plugin.py

@@ -1,4 +1,5 @@
 import os
+import uuid
 
 import py
 import pytest
@@ -122,12 +123,22 @@ def pytest_addoption(parser):
         metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.",
     )
-
     group.addoption(
         "--boxed",
         action="store_true",
         help="backward compatibility alias for pytest-forked --forked",
     )
+    group.addoption(
+        "--testrunuid",
+        action="store",
+        help=(
+            "provide an identifier shared amongst all workers as the value of "
+            "the 'testrun_uid' fixture,\n\n,"
+            "if not provided, 'testrun_uid' is filled with a new unique string "
+            "on every test run."
+        ),
+    )
+
     parser.addini(
         "rsyncdirs",
         "list of (relative) paths to be rsynced for remote distributed testing.",
@@ -180,7 +191,11 @@ def pytest_configure(config):
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
     if isinstance(config.option.numprocesses, AutoInt):
-        config.option.numprocesses = 0 if usepdb else int(config.option.numprocesses)
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            config.option.numprocesses = int(config.option.numprocesses)
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -192,12 +207,10 @@ def pytest_cmdline_main(config):
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue
-    if not val("collectonly"):
-        if val("dist") != "no":
-            if usepdb:
-                raise pytest.UsageError(
-                    "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
-                )  # noqa: E501
+    if not val("collectonly") and val("dist") != "no" and usepdb:
+        raise pytest.UsageError(
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
+        )  # noqa: E501
 
 
 # -------------------------------------------------------------------------
@@ -214,3 +227,12 @@ def worker_id(request):
         return request.config.workerinput["workerid"]
     else:
         return "master"
+
+
+@pytest.fixture(scope="session")
+def testrun_uid(request):
+    """Return the unique id of the current test."""
+    if hasattr(request.config, "workerinput"):
+        return request.config.workerinput["testrunuid"]
+    else:
+        return uuid.uuid4().hex

src/xdist/remote.py

@@ -22,6 +22,7 @@ class WorkerInteractor(object):
     def __init__(self, config, channel):
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
+        self.testrunuid = config.workerinput["testrunuid"]
         self.log = py.log.Producer("worker-%s" % self.workerid)
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
@@ -112,6 +113,7 @@ class WorkerInteractor(object):
         )
         data["item_index"] = self.item_index
         data["worker_id"] = self.workerid
+        data["testrun_uid"] = self.testrunuid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
@@ -137,8 +139,20 @@ class WorkerInteractor(object):
                 fslocation=str(fslocation),
             )
 
+    # the pytest_warning_recorded hook was introduced in pytest 6.0
+    if hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+
+        def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+            self.sendevent(
+                "warning_recorded",
+                warning_message_data=serialize_warning_message(warning_message),
+                when=when,
+                nodeid=nodeid,
+                location=location,
+            )
+
     # the pytest_warning_captured hook was introduced in pytest 3.8
-    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
+    elif hasattr(_pytest.hookspec, "pytest_warning_captured"):
 
         def pytest_warning_captured(self, warning_message, when, item):
             self.sendevent(
@@ -238,6 +252,7 @@ if __name__ == "__channelexec__":
             importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
         )
 
+    os.environ["PYTEST_XDIST_TESTRUNUID"] = workerinput["testrunuid"]
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
     os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
 

src/xdist/scheduler/loadscope.py

@@ -358,12 +358,12 @@ class LoadScopeScheduling(object):
         extra_nodes = len(self.nodes) - len(self.workqueue)
 
         if extra_nodes > 0:
-            self.log("Shuting down {0} nodes".format(extra_nodes))
+            self.log("Shuting down {} nodes".format(extra_nodes))
 
             for _ in range(extra_nodes):
                 unused_node, assigned = self.assigned_work.popitem(last=True)
 
-                self.log("Shuting down unused node {0}".format(unused_node))
+                self.log("Shuting down unused node {}".format(unused_node))
                 unused_node.shutdown()
 
         # Assign initial workload

src/xdist/workermanage.py

@@ -3,6 +3,7 @@ import fnmatch
 import os
 import re
 import sys
+import uuid
 
 import py
 import pytest
@@ -35,6 +36,9 @@ class NodeManager(object):
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
         self.trace = self.config.trace.get("nodemanager")
+        self.testrunuid = self.config.getoption("testrunuid")
+        if self.testrunuid is None:
+            self.testrunuid = uuid.uuid4().hex
         self.group = execnet.Group()
         if specs is None:
             specs = self._getxspecs()
@@ -59,10 +63,7 @@ class NodeManager(object):
     def setup_nodes(self, putevent):
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        nodes = []
-        for spec in self.specs:
-            nodes.append(self.setup_node(spec, putevent))
-        return nodes
+        return [self.setup_node(spec, putevent) for spec in self.specs]
 
     def setup_node(self, spec, putevent):
         gw = self.group.makegateway(spec)
@@ -89,8 +90,16 @@ class NodeManager(object):
         import pytest
         import _pytest
 
-        pytestpath = pytest.__file__.rstrip("co")
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
+        def get_dir(p):
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
         config = self.config
         candidates = [py._pydir, pytestpath, pytestdir]
         candidates += config.option.rsyncdir
@@ -101,7 +110,7 @@ class NodeManager(object):
         for root in candidates:
             root = py.path.local(root).realpath()
             if not root.check():
-                raise pytest.UsageError("rsyncdir doesn't exist: %r" % (root,))
+                raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root))
             if root not in roots:
                 roots.append(root)
         return roots
@@ -154,11 +163,10 @@ class HostRSync(execnet.RSync):
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
-        self._ignores = []
         ignores = kwargs.pop("ignores", None) or []
-        for x in ignores:
-            x = getattr(x, "strpath", x)
-            self._ignores.append(re.compile(fnmatch.translate(x)))
+        self._ignores = [
+            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
+        ]
         super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
@@ -179,7 +187,7 @@ class HostRSync(execnet.RSync):
         if self._verbose:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
-            print("%s:%s <= %s" % (gateway.spec, remotepath, path))
+            print("{}:{} <= {}".format(gateway.spec, remotepath, path))
 
 
 def make_reltoroot(roots, args):
@@ -190,6 +198,7 @@ def make_reltoroot(roots, args):
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
         if not fspath.exists():
+            result.append(arg)
             continue
         for root in roots:
             x = fspath.relto(root)
@@ -197,7 +206,7 @@ def make_reltoroot(roots, args):
                 parts[0] = root.basename + "/" + x
                 break
         else:
-            raise ValueError("arg %s not relative to an rsync root" % (arg,))
+            raise ValueError("arg {} not relative to an rsync root".format(arg))
         result.append(splitcode.join(parts))
     return result
 
@@ -221,6 +230,7 @@ class WorkerController(object):
             "workercount": len(nodemanager.specs),
             "slaveid": gateway.id,
             "slavecount": len(nodemanager.specs),
+            "testrunuid": nodemanager.testrunuid,
             "mainargv": sys.argv,
         }
         # TODO: deprecated name, backward compatibility only. Remove it in future
@@ -232,7 +242,7 @@ class WorkerController(object):
             py.log.setconsumer(self.log._keywords, None)
 
     def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__, self.gateway.id)
+        return "<{} {}>".format(self.__class__.__name__, self.gateway.id)
 
     @property
     def shutting_down(self):
@@ -292,11 +302,11 @@ class WorkerController(object):
 
     def sendcommand(self, name, **kwargs):
         """ send a named parametrized command to the other side. """
-        self.log("sending command %s(**%s)" % (name, kwargs))
+        self.log("sending command {}(**{})".format(name, kwargs))
         self.channel.send((name, kwargs))
 
     def notify_inproc(self, eventname, **kwargs):
-        self.log("queuing %s(**%s)" % (eventname, kwargs))
+        self.log("queuing {}(**{})".format(eventname, kwargs))
         self.putevent((eventname, kwargs))
 
     def process_from_remote(self, eventcall):  # noqa too complex
@@ -318,7 +328,7 @@ class WorkerController(object):
                 return
             eventname, kwargs = eventcall
             if eventname in ("collectionstart",):
-                self.log("ignoring %s(%s)" % (eventname, kwargs))
+                self.log("ignoring {}({})".format(eventname, kwargs))
             elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
@@ -357,8 +367,19 @@ class WorkerController(object):
                     when=kwargs["when"],
                     item=kwargs["item"],
                 )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
             else:
-                raise ValueError("unknown event: %s" % (eventname,))
+                raise ValueError("unknown event: {}".format(eventname))
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise

testing/acceptance_test.py

@@ -115,6 +115,9 @@ class TestDistribution:
         assert result.ret == 2
         result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"])
 
+    @pytest.mark.xfail(
+        reason="#527: Python 3.8 failure in pytest where testdir.tmpdir returns an unexpected value"
+    )
     def test_basetemp_in_subprocesses(self, testdir):
         p1 = testdir.makepyfile(
             """
@@ -186,16 +189,46 @@ class TestDistribution:
         )
         assert result.ret == 1
 
-    def test_distribution_rsyncdirs_example(self, testdir):
+    def test_distribution_rsyncdirs_example(self, testdir, monkeypatch):
+        # use a custom plugin that has a custom command-line option to ensure
+        # this is propagated to workers (see #491)
+        testdir.makepyfile(
+            **{
+                "myplugin/src/foobarplugin.py": """
+            from __future__ import print_function
+
+            import os
+            import sys
+            import pytest
+
+            def pytest_addoption(parser):
+                parser.addoption("--foobar", action="store", dest="foobar_opt")
+
+            @pytest.mark.tryfirst
+            def pytest_load_initial_conftests(early_config):
+                opt = early_config.known_args_namespace.foobar_opt
+                print("--foobar=%s active! [%s]" % (opt, os.getpid()), file=sys.stderr)
+            """
+            }
+        )
+        assert (testdir.tmpdir / "myplugin/src/foobarplugin.py").check(file=1)
+        monkeypatch.setenv(
+            "PYTHONPATH", str(testdir.tmpdir / "myplugin/src"), prepend=os.pathsep
+        )
+
         source = testdir.mkdir("source")
         dest = testdir.mkdir("dest")
         subdir = source.mkdir("example_pkg")
         subdir.ensure("__init__.py")
         p = subdir.join("test_one.py")
         p.write("def test_5():\n  assert not __file__.startswith(%r)" % str(p))
-        result = testdir.runpytest(
+        result = testdir.runpytest_subprocess(
             "-v",
             "-d",
+            "-s",
+            "-pfoobarplugin",
+            "--foobar=123",
+            "--dist=load",
             "--rsyncdir=%(subdir)s" % locals(),
             "--tx=popen//chdir=%(dest)s" % locals(),
             p,
@@ -209,6 +242,7 @@ class TestDistribution:
                 "*1 passed*",
             ]
         )
+        result.stderr.fnmatch_lines(["--foobar=123 active! *"])
         assert dest.join(subdir.basename).check(dir=1)
 
     def test_backward_compatibility_worker_terminology(self, testdir):
@@ -297,7 +331,7 @@ class TestDistribution:
                 time.sleep(10)
         """
         )
-        child = testdir.spawn_pytest("-n1 -v")
+        child = testdir.spawn_pytest("-n1 -v", expect_timeout=30.0)
         child.expect(".*test_sleep.*")
         child.kill(2)  # keyboard interrupt
         child.expect(".*KeyboardInterrupt.*")
@@ -559,7 +593,11 @@ def test_fixture_teardown_failure(testdir):
     assert result.ret
 
 
-def test_config_initialization(testdir, pytestconfig):
+@pytest.mark.skipif(
+    sys.version_info[:2] == (2, 7),
+    reason="Only available in pytest 5.0+ (Python 3 only)",
+)
+def test_config_initialization(testdir, monkeypatch, pytestconfig):
     """Ensure workers and master are initialized consistently. Integration test for #445"""
     if not hasattr(pytestconfig, "invocation_params"):
         pytest.skip(
@@ -568,7 +606,8 @@ def test_config_initialization(testdir, pytestconfig):
     testdir.makepyfile(
         **{
             "dir_a/test_foo.py": """
-                def test_1(): pass
+                def test_1(request):
+                    assert request.config.option.verbose == 2
         """
         }
     )
@@ -579,8 +618,10 @@ def test_config_initialization(testdir, pytestconfig):
         testpaths=dir_a
     """,
     )
+    monkeypatch.setenv("PYTEST_ADDOPTS", "-v")
     result = testdir.runpytest("-n2", "-c", "myconfig.ini", "-v")
-    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*"])
+    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*", "*= 1 passed in *"])
+    assert result.ret == 0
 
 
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
@@ -784,7 +825,33 @@ class TestWarnings:
             )
         )
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warning*"])
+
+    def test_warning_captured_deprecated_in_pytest_6(self, testdir):
+        """
+        Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
+        """
+        import _pytest.hookspec
+
+        if not hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+            pytest.skip("test requires pytest 6.0+")
+
+        testdir.makeconftest(
+            """
+            def pytest_warning_captured():
+                assert False, "this hook should not be called in this version"
+        """
+        )
+        testdir.makepyfile(
+            """
+            import warnings
+            def test():
+                warnings.warn("custom warning")
+        """
+        )
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines(["* 1 passed in *"])
+        result.stdout.no_fnmatch_line("*this hook should not be called in this version")
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_custom_subclass(self, testdir, n):
@@ -808,7 +875,7 @@ class TestWarnings:
         )
         testdir.syspathinsert()
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warning*"])
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_unserializable_arguments(self, testdir, n):
@@ -825,7 +892,7 @@ class TestWarnings:
         )
         testdir.syspathinsert()
         result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warnings*"])
+        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warning*"])
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_unserializable_warning_details(self, testdir, n):
@@ -857,7 +924,7 @@ class TestWarnings:
         testdir.syspathinsert()
         result = testdir.runpytest(n)
         result.stdout.fnmatch_lines(
-            ["*ResourceWarning*unclosed*", "*1 passed, 1 warnings*"]
+            ["*ResourceWarning*unclosed*", "*1 passed, 1 warning*"]
         )
 
 
@@ -1035,6 +1102,29 @@ def test_worker_id_fixture(testdir, n):
         assert worker_ids == {"gw0", "gw1"}
 
 
+@pytest.mark.parametrize("n", [0, 2])
+def test_testrun_uid_fixture(testdir, n):
+    import glob
+
+    f = testdir.makepyfile(
+        """
+        import pytest
+        @pytest.mark.parametrize("run_num", range(2))
+        def test_testrun_uid1(testrun_uid, run_num):
+            with open("testrun_uid%s.txt" % run_num, "w") as f:
+                f.write(testrun_uid)
+    """
+    )
+    result = testdir.runpytest(f, "-n%d" % n)
+    result.stdout.fnmatch_lines("* 2 passed in *")
+    testrun_uids = set()
+    for fname in glob.glob(str(testdir.tmpdir.join("*.txt"))):
+        with open(fname) as f:
+            testrun_uids.add(f.read().strip())
+    assert len(testrun_uids) == 1
+    assert len(testrun_uids.pop()) == 32
+
+
 @pytest.mark.parametrize("tb", ["auto", "long", "short", "no", "line", "native"])
 def test_error_report_styles(testdir, tb):
     testdir.makepyfile(

testing/conftest.py

@@ -22,12 +22,16 @@ def _divert_atexit(request, monkeypatch):
 
     finalizers = []
 
-    def finish():
-        while finalizers:
-            finalizers.pop()()
+    def fake_register(func, *args, **kwargs):
+        finalizers.append((func, args, kwargs))
 
-    monkeypatch.setattr(atexit, "register", finalizers.append)
-    request.addfinalizer(finish)
+    monkeypatch.setattr(atexit, "register", fake_register)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
 
 
 def pytest_addoption(parser):

testing/test_looponfail.py

@@ -284,7 +284,7 @@ class TestFunctional:
         )
         # p = testdir.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+        child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
@@ -311,7 +311,7 @@ class TestFunctional:
                 pass
         """
         )
-        child = testdir.spawn_pytest("-f %s" % p)
+        child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")

testing/test_newhooks.py

@@ -15,7 +15,7 @@ class TestHooks:
 
     def test_runtest_logreport(self, testdir):
         """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain "node", "nodeid" and "worker_id" attributes. (#8)
+        with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
         """
         testdir.makeconftest(
             """
@@ -23,20 +23,24 @@ class TestHooks:
                 if hasattr(report, 'node'):
                     if report.when == "call":
                         workerid = report.node.workerinput['workerid']
+                        testrunuid = report.node.workerinput['testrunuid']
                         if workerid != report.worker_id:
                             print("HOOK: Worker id mismatch: %s %s"
                                    % (workerid, report.worker_id))
+                        elif testrunuid != report.testrun_uid:
+                            print("HOOK: Testrun uid mismatch: %s %s"
+                                   % (testrunuid, report.testrun_uid))
                         else:
-                            print("HOOK: %s %s"
-                                   % (report.nodeid, report.worker_id))
+                            print("HOOK: %s %s %s"
+                                   % (report.nodeid, report.worker_id, report.testrun_uid))
         """
         )
         res = testdir.runpytest("-n1", "-s")
         res.stdout.fnmatch_lines(
             [
-                "*HOOK: test_runtest_logreport.py::test_a gw0*",
-                "*HOOK: test_runtest_logreport.py::test_b gw0*",
-                "*HOOK: test_runtest_logreport.py::test_c gw0*",
+                "*HOOK: test_runtest_logreport.py::test_a gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_b gw0 *",
+                "*HOOK: test_runtest_logreport.py::test_c gw0 *",
                 "*3 passed*",
             ]
         )

testing/test_plugin.py

@@ -57,6 +57,7 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     check_options(config)
     assert config.getoption("usepdb")
     assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
 
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
@@ -95,6 +96,18 @@ def test_dsession_with_collect_only(testdir):
     assert not config.pluginmanager.hasplugin("dsession")
 
 
+def test_testrunuid_provided(testdir):
+    config = testdir.parseconfigure("--testrunuid", "test123", "--tx=popen")
+    nm = NodeManager(config)
+    assert nm.testrunuid == "test123"
+
+
+def test_testrunuid_generated(testdir):
+    config = testdir.parseconfigure("--tx=popen")
+    nm = NodeManager(config)
+    assert len(nm.testrunuid) == 32
+
+
 class TestDistOptions:
     def test_getxspecs(self, testdir):
         config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")

testing/test_remote.py

@@ -2,6 +2,7 @@ import py
 import pprint
 import pytest
 import sys
+import uuid
 
 from xdist.workermanage import WorkerController
 import execnet
@@ -25,7 +26,7 @@ class EventCall:
         self.name, self.kwargs = eventcall
 
     def __str__(self):
-        return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
+        return "<EventCall {}(**{})>".format(self.name, self.kwargs)
 
 
 class WorkerSetup:
@@ -44,6 +45,7 @@ class WorkerSetup:
         putevent = self.use_callback and self.events.put or None
 
         class DummyMananger:
+            testrunuid = uuid.uuid4().hex
             specs = [0, 1]
 
         self.slp = WorkerController(DummyMananger, self.gateway, config, putevent)
@@ -59,7 +61,7 @@ class WorkerSetup:
             ev = EventCall(data)
             if name is None or ev.name == name:
                 return ev
-            print("skipping %s" % (ev,))
+            print("skipping {}".format(ev))
 
     def sendcommand(self, name, **kwargs):
         self.slp.sendcommand(name, **kwargs)
@@ -220,6 +222,7 @@ def test_remote_env_vars(testdir):
         """
         import os
         def test():
+            assert len(os.environ['PYTEST_XDIST_TESTRUNUID']) == 32
             assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     """

testing/test_slavemanage.py

@@ -168,7 +168,7 @@ class TestNodeManager:
         # assert nodemanager.config.topdir == source == config.topdir
         nodemanager.makegateways()
         nodemanager.rsync_roots()
-        p, = nodemanager.gwmanager.multi_exec(
+        (p,) = nodemanager.gwmanager.multi_exec(
             "import os ; channel.send(os.getcwd())"
         ).receive_each()
         p = py.path.local(p)
@@ -269,5 +269,5 @@ class TestNodeManager:
         reprec = testdir.inline_run(
             "-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir
         )
-        rep, = reprec.getreports("pytest_runtest_logreport")
+        (rep,) = reprec.getreports("pytest_runtest_logreport")
         assert rep.passed

tox.ini

@@ -1,8 +1,8 @@
 [tox]
 envlist=
   linting
-  py{27,34,35,36,37}-pytestlatest
-  py37-pytest{master,features}
+  py{27,35,36,37,38}-pytestlatest
+  py38-pytestmaster
 
 [testenv]
 passenv = USER USERNAME
@@ -10,7 +10,6 @@ extras = testing
 deps =
   pytestlatest: pytest
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
-  pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
 commands=
   pytest {posargs}