Release 1.34.0

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>

CHANGELOG.rst

@@ -1,3 +1,37 @@
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
 pytest-xdist 1.32.0 (2020-05-03)
 ================================
 

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

README.rst

@@ -67,33 +67,35 @@ a checkout of the `pytest-xdist repository`_ ::
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, type::
+To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
 
-    pytest -n NUM
+    pytest -n NUMCPUS
 
-Especially for longer running tests or tests requiring
+Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
-a lot of I/O this can lead to considerable speed ups. This option can
+can lead to considerable speed ups, especially if your test suite takes a
-also be set to ``auto`` for automatic detection of the number of CPUs.
+noticeable amount of time.
 
-If a test crashes the interpreter, pytest-xdist will automatically restart
+If a test crashes a worker, pytest-xdist will automatically restart that worker
-that worker and report the failure as usual. You can use the
+and report the test’s failure. You can use the ``--max-worker-restart`` option
-``--max-worker-restart`` option to limit the number of workers that can
+to limit the number of worker restarts that are allowed, or disable restarting
-be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+altogether using ``--max-worker-restart 0``.
 
-By default, the ``-n`` option will send pending tests to any worker that is available, without
+By default, using ``--numprocesses`` will send pending tests to any worker that
-any guaranteed order, but you can control this with these options:
+is available, without any guaranteed order. You can change the test
+distribution algorithm this with the ``--dist`` option. It takes these values:
 
-* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
+* ``--dist no``: The default algorithm, distributing one test at a time.
-  by **class** for *test methods*, then each group will be sent to an available worker,
-  guaranteeing that all tests in a group run in the same process. This can be useful if you have
-  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
-  with grouping by class takes priority over grouping by module.
-  This feature was added in version ``1.19``.
 
-* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
-  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
+  and by **class** for *test methods*. Groups are distributed to available
-  in version ``1.21``.
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
 
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
 
 Making session-scoped fixtures execute only once
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/xdist/dsession.py

@@ -277,10 +277,17 @@ class DSession(object):
         self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
     def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_logwarning hook."""
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 

src/xdist/plugin.py

@@ -191,7 +191,11 @@ def pytest_configure(config):
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
     if isinstance(config.option.numprocesses, AutoInt):
-        config.option.numprocesses = 0 if usepdb else int(config.option.numprocesses)
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            config.option.numprocesses = int(config.option.numprocesses)
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -203,12 +207,10 @@ def pytest_cmdline_main(config):
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue
-    if not val("collectonly"):
+    if not val("collectonly") and val("dist") != "no" and usepdb:
-        if val("dist") != "no":
+        raise pytest.UsageError(
-            if usepdb:
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
-                raise pytest.UsageError(
+        )  # noqa: E501
-                    "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
-                )  # noqa: E501
 
 
 # -------------------------------------------------------------------------

src/xdist/remote.py

@@ -139,8 +139,20 @@ class WorkerInteractor(object):
                 fslocation=str(fslocation),
             )
 
+    # the pytest_warning_recorded hook was introduced in pytest 6.0
+    if hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+
+        def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+            self.sendevent(
+                "warning_recorded",
+                warning_message_data=serialize_warning_message(warning_message),
+                when=when,
+                nodeid=nodeid,
+                location=location,
+            )
+
     # the pytest_warning_captured hook was introduced in pytest 3.8
-    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
+    elif hasattr(_pytest.hookspec, "pytest_warning_captured"):
 
         def pytest_warning_captured(self, warning_message, when, item):
             self.sendevent(

src/xdist/workermanage.py

@@ -63,10 +63,7 @@ class NodeManager(object):
     def setup_nodes(self, putevent):
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        nodes = []
+        return [self.setup_node(spec, putevent) for spec in self.specs]
-        for spec in self.specs:
-            nodes.append(self.setup_node(spec, putevent))
-        return nodes
 
     def setup_node(self, spec, putevent):
         gw = self.group.makegateway(spec)
@@ -93,8 +90,16 @@ class NodeManager(object):
         import pytest
         import _pytest
 
-        pytestpath = pytest.__file__.rstrip("co")
+        def get_dir(p):
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
         config = self.config
         candidates = [py._pydir, pytestpath, pytestdir]
         candidates += config.option.rsyncdir
@@ -158,11 +163,10 @@ class HostRSync(execnet.RSync):
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
-        self._ignores = []
         ignores = kwargs.pop("ignores", None) or []
-        for x in ignores:
+        self._ignores = [
-            x = getattr(x, "strpath", x)
+            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
-            self._ignores.append(re.compile(fnmatch.translate(x)))
+        ]
         super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
@@ -363,6 +367,17 @@ class WorkerController(object):
                     when=kwargs["when"],
                     item=kwargs["item"],
                 )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
             else:
                 raise ValueError("unknown event: {}".format(eventname))
         except KeyboardInterrupt:

testing/acceptance_test.py

@@ -189,7 +189,6 @@ class TestDistribution:
         )
         assert result.ret == 1
 
-    @pytest.mark.xfail(reason="#527: Ignore Python 3.8 failure for the time being")
     def test_distribution_rsyncdirs_example(self, testdir, monkeypatch):
         # use a custom plugin that has a custom command-line option to ensure
         # this is propagated to workers (see #491)
@@ -332,7 +331,7 @@ class TestDistribution:
                 time.sleep(10)
         """
         )
-        child = testdir.spawn_pytest("-n1 -v")
+        child = testdir.spawn_pytest("-n1 -v", expect_timeout=30.0)
         child.expect(".*test_sleep.*")
         child.kill(2)  # keyboard interrupt
         child.expect(".*KeyboardInterrupt.*")
@@ -594,7 +593,11 @@ def test_fixture_teardown_failure(testdir):
     assert result.ret
 
 
-def test_config_initialization(testdir, pytestconfig):
+@pytest.mark.skipif(
+    sys.version_info[:2] == (2, 7),
+    reason="Only available in pytest 5.0+ (Python 3 only)",
+)
+def test_config_initialization(testdir, monkeypatch, pytestconfig):
     """Ensure workers and master are initialized consistently. Integration test for #445"""
     if not hasattr(pytestconfig, "invocation_params"):
         pytest.skip(
@@ -603,7 +606,8 @@ def test_config_initialization(testdir, pytestconfig):
     testdir.makepyfile(
         **{
             "dir_a/test_foo.py": """
-                def test_1(): pass
+                def test_1(request):
+                    assert request.config.option.verbose == 2
         """
         }
     )
@@ -614,8 +618,10 @@ def test_config_initialization(testdir, pytestconfig):
         testpaths=dir_a
     """,
     )
+    monkeypatch.setenv("PYTEST_ADDOPTS", "-v")
     result = testdir.runpytest("-n2", "-c", "myconfig.ini", "-v")
-    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*"])
+    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*", "*= 1 passed in *"])
+    assert result.ret == 0
 
 
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
@@ -821,6 +827,32 @@ class TestWarnings:
         result = testdir.runpytest(n)
         result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warning*"])
 
+    def test_warning_captured_deprecated_in_pytest_6(self, testdir):
+        """
+        Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
+        """
+        import _pytest.hookspec
+
+        if not hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+            pytest.skip("test requires pytest 6.0+")
+
+        testdir.makeconftest(
+            """
+            def pytest_warning_captured():
+                assert False, "this hook should not be called in this version"
+        """
+        )
+        testdir.makepyfile(
+            """
+            import warnings
+            def test():
+                warnings.warn("custom warning")
+        """
+        )
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines(["* 1 passed in *"])
+        result.stdout.no_fnmatch_line("*this hook should not be called in this version")
+
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_custom_subclass(self, testdir, n):
         """Check that warning subclasses that don't honor the args attribute don't break

testing/conftest.py

@@ -22,12 +22,16 @@ def _divert_atexit(request, monkeypatch):
 
     finalizers = []
 
-    def finish():
+    def fake_register(func, *args, **kwargs):
-        while finalizers:
+        finalizers.append((func, args, kwargs))
-            finalizers.pop()()
 
-    monkeypatch.setattr(atexit, "register", finalizers.append)
+    monkeypatch.setattr(atexit, "register", fake_register)
-    request.addfinalizer(finish)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
 
 
 def pytest_addoption(parser):

testing/test_looponfail.py

@@ -284,7 +284,7 @@ class TestFunctional:
         )
         # p = testdir.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+        child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
@@ -311,7 +311,7 @@ class TestFunctional:
                 pass
         """
         )
-        child = testdir.spawn_pytest("-f %s" % p)
+        child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")

testing/test_plugin.py

@@ -57,6 +57,7 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     check_options(config)
     assert config.getoption("usepdb")
     assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
 
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")