Release 1.21.1

Rename filescope to loadfile

.travis.yml

@@ -21,23 +21,26 @@ env:
 - TOXENV=py-pytest31
 - TOXENV=py-pytest32
 - TOXENV=py-pytest33
-- TOXENV=py-pytest36
 
 install: pip install tox setuptools_scm
 script: tox
 
 stages:
-- linting
+- baseline
 - test
 - name: deploy
   if: repo = pytest-dev/pytest-xdist AND tag IS present
 
 jobs:
   include:
-    - stage: linting
+    - stage: baseline
       python: '3.6'
-      script:
-      - tox -e linting
+      env: TOXENV=linting
+    - python: '3.6'
+      env: TOXENV=py36-pytestlatest
+    - python: '2.7'
+      env: TOXENV=py27-pytestlatest
+
     - stage: test
     # python x env above are already included into this stage
     - python: "2.7"

CHANGELOG.rst

@@ -1,3 +1,54 @@
+pytest-xdist 1.24.1 (2018-11-09)
+================================
+
+Bug Fixes
+---------
+
+- `#349 <https://github.com/pytest-dev/pytest-xdist/issues/349>`_: Correctly handle warnings created with arguments that can't be serialized during the transfer from workers to master node.
+
+
+pytest-xdist 1.24.0 (2018-10-18)
+================================
+
+Features
+--------
+
+- `#337 <https://github.com/pytest-dev/pytest-xdist/issues/337>`_: New ``--maxprocesses`` command-line option that limits the maximum number of workers when using ``--numprocesses=auto``.
+
+
+Bug Fixes
+---------
+
+- `#351 <https://github.com/pytest-dev/pytest-xdist/issues/351>`_: Fix scheduling deadlock in case of inter-test locking.
+
+
+pytest-xdist 1.23.2 (2018-09-28)
+================================
+
+Bug Fixes
+---------
+
+- `#344 <https://github.com/pytest-dev/pytest-xdist/issues/344>`_: Fix issue where Warnings could cause pytest to fail if they do not set the args attribute correctly.
+
+
+pytest-xdist 1.23.1 (2018-09-25)
+================================
+
+Bug Fixes
+---------
+
+- `#341 <https://github.com/pytest-dev/pytest-xdist/issues/341>`_: Fix warnings transfer between workers and master node with pytest >= 3.8.
+
+
+pytest-xdist 1.23.0 (2018-08-23)
+================================
+
+Features
+--------
+
+- `#330 <https://github.com/pytest-dev/pytest-xdist/issues/330>`_: Improve collection performance by reducing the number of events sent to ``master`` node.
+
+
 pytest-xdist 1.22.5 (2018-07-27)
 ================================
 

appveyor.yml

@@ -5,7 +5,7 @@ environment:
   - TOXENV: "py34-pytest33"
   - TOXENV: "py35-pytest33"
   - TOXENV: "py36-pytest33"
-  - TOXENV: "py36-pytest36"
+  - TOXENV: "py36-pytestlatest"
   - TOXENV: "py27-pytest33-pexpect"
   - TOXENV: "py36-pytest33-pexpect"
 

testing/acceptance_test.py

@@ -400,33 +400,6 @@ class TestTerminalReporting:
             ]
         )
 
-    @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
-    def test_logwarning(self, testdir, n, warn_type):
-        from pkg_resources import parse_version
-
-        if parse_version(pytest.__version__) < parse_version("3.1"):
-            pytest.skip("pytest warnings requires >= 3.1")
-
-        if warn_type == "builtin":
-            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
-        elif warn_type == "pytest":
-            warn_code = """request.config.warn('', 'this is a warning',
-                           fslocation=py.path.local())"""
-        else:
-            assert False
-        testdir.makepyfile(
-            """
-            import warnings, py
-            def test_func(request):
-                {warn_code}
-        """.format(
-                warn_code=warn_code
-            )
-        )
-        result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
-
     def test_logfinish_hook(self, testdir):
         """Ensure the pytest_runtest_logfinish hook is being properly handled"""
         from _pytest import hookspec
@@ -734,6 +707,83 @@ def test_sub_plugins_disabled(testdir, plugin):
     result.stdout.fnmatch_lines("*1 passed*")
 
 
+class TestWarnings:
+    @pytest.fixture(autouse=True)
+    def skip_if_unsupported_pytest_version(self):
+        """Skip tests of this class if we are running in a pytest version which does not
+        support warnings yet.
+        """
+        from pkg_resources import parse_version
+
+        if parse_version(pytest.__version__) < parse_version("3.1"):
+            pytest.skip("pytest warnings requires >= 3.1")
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
+    def test_warnings(self, testdir, n, warn_type):
+        if warn_type == "builtin":
+            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
+        elif warn_type == "pytest":
+            warn_code = """request.config.warn('', 'this is a warning',
+                           fslocation=py.path.local())"""
+        else:
+            assert False
+        testdir.makepyfile(
+            """
+            import warnings, py, pytest
+
+            @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
+            def test_func(request):
+                {warn_code}
+        """.format(
+                warn_code=warn_code
+            )
+        )
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_custom_subclass(self, testdir, n):
+        """Check that warning subclasses that don't honor the args attribute don't break
+        pytest-xdist (#344)
+        """
+        testdir.makepyfile(
+            """
+            import warnings, py, pytest
+
+            class MyWarning(UserWarning):
+
+                def __init__(self, p1, p2):
+                    self.p1 = p1
+                    self.p2 = p2
+                    self.args = ()
+
+            def test_func(request):
+                warnings.warn(MyWarning("foo", 1))
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_unserializable_arguments(self, testdir, n):
+        """Check that warnings with unserializable arguments are handled correctly (#349)."""
+        testdir.makepyfile(
+            """
+            import warnings, pytest
+
+            def test_func(tmpdir):
+                fn = (tmpdir / 'foo.txt').ensure(file=1)
+                with fn.open('r') as f:
+                    warnings.warn(UserWarning("foo", f))
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warnings*"])
+
+
 class TestNodeFailure:
     def test_load_single(self, testdir):
         f = testdir.makepyfile(
@@ -1101,6 +1151,56 @@ class TestFileScope:
         assert c1 == c2
 
 
+class TestLocking:
+    _test_content = """
+    class TestClassName%s(object):
+
+        @classmethod
+        def setup_class(cls):
+            FILE_LOCK.acquire()
+
+        @classmethod
+        def teardown_class(cls):
+            FILE_LOCK.release()
+
+        def test_a(self):
+            pass
+
+        def test_b(self):
+            pass
+
+        def test_c(self):
+            pass
+
+    """
+
+    test_file1 = """
+    import filelock
+
+    FILE_LOCK = filelock.FileLock("test.lock")
+
+    """ + (
+        (_test_content * 4) % ("A", "B", "C", "D")
+    )
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_single_file(self, testdir, scope):
+        testdir.makepyfile(test_a=self.test_file1)
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(12 if scope != "each" else 12 * 2))
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_multi_file(self, testdir, scope):
+        testdir.makepyfile(
+            test_a=self.test_file1,
+            test_b=self.test_file1,
+            test_c=self.test_file1,
+            test_d=self.test_file1,
+        )
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(48 if scope != "each" else 48 * 2))
+
+
 def parse_tests_and_workers_from_output(lines):
     result = []
     for line in lines:

testing/test_plugin.py

@@ -25,6 +25,10 @@ def test_dist_options(testdir):
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
+    config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
     config = testdir.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"

testing/test_remote.py

@@ -321,8 +321,6 @@ class TestWorkerInteractor:
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = worker.popevent()
-        assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs["data"])
         assert rep.failed
         ev = worker.popevent("collectionfinish")

tox.ini

@@ -2,7 +2,7 @@
 # if you change the envlist, please update .travis.yml file as well
 envlist=
   linting
-  py{27,34,35,36}-pytest{30,31,32,33,36}
+  py{27,34,35,36}-pytest{30,31,32,33,latest}
   py{27,36}-pytest36-pexpect
   py{27,36}-pytest{master,features}
 
@@ -18,15 +18,14 @@ deps =
   pytest31: pytest~=3.1.0
   pytest32: pytest~=3.2.0
   pytest33: pytest~=3.3.0
-  pytest36: pytest~=3.6.0
+  pytestlatest: pytest
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
   pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect
+  filelock
 platform=
   pexpect: linux|darwin
 commands=
-  # always clean to avoid code unmarshal mismatch on old python/pytest
-  py.cleanup -aq
   pytest {posargs}
 
 [testenv:linting]

xdist/dsession.py

@@ -257,15 +257,24 @@ class DSession(object):
         self.sched.mark_test_complete(node, item_index, duration)
 
     def worker_collectreport(self, node, rep):
-        """Emitted when a node calls the pytest_collectreport hook."""
-        if rep.failed:
-            self._failed_worker_collectreport(node, rep)
+        """Emitted when a node calls the pytest_collectreport hook.
+
+        Because we only need the report when there's a failure, as optimization
+        we only expect to receive failed reports from workers (#330).
+        """
+        assert rep.failed
+        self._failed_worker_collectreport(node, rep)
 
     def worker_logwarning(self, message, code, nodeid, fslocation):
         """Emitted when a node calls the pytest_logwarning hook."""
         kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
         self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
+    def worker_warning_captured(self, warning_message, when, item):
+        """Emitted when a node calls the pytest_logwarning hook."""
+        kwargs = dict(warning_message=warning_message, when=when, item=item)
+        self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 

xdist/plugin.py

@@ -47,6 +47,14 @@ def pytest_addoption(parser):
         "you can use 'auto' here for auto detection CPUs number on "
         "host system",
     )
+    group.addoption(
+        "--maxprocesses",
+        dest="maxprocesses",
+        metavar="maxprocesses",
+        action="store",
+        type=int,
+        help="limit the maximum number of workers to process the tests when using --numprocesses=auto",
+    )
     group.addoption(
         "--max-worker-restart",
         "--max-slave-restart",
@@ -172,7 +180,10 @@ def pytest_cmdline_main(config):
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
-        config.option.tx = ["popen"] * config.option.numprocesses
+        numprocesses = config.option.numprocesses
+        if config.option.maxprocesses:
+            numprocesses = min(numprocesses, config.option.maxprocesses)
+        config.option.tx = ["popen"] * numprocesses
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue

xdist/remote.py

@@ -12,6 +12,7 @@ import time
 
 import _pytest.hookspec
 import pytest
+from execnet.gateway_base import dumps, DumpError
 
 
 class WorkerInteractor(object):
@@ -109,8 +110,10 @@ class WorkerInteractor(object):
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
-        data = serialize_report(report)
-        self.sendevent("collectreport", data=data)
+        # master only needs reports that failed, as optimization send only them instead (#330)
+        if report.failed:
+            data = serialize_report(report)
+            self.sendevent("collectreport", data=data)
 
     def pytest_logwarning(self, message, code, nodeid, fslocation):
         self.sendevent(
@@ -121,6 +124,18 @@ class WorkerInteractor(object):
             fslocation=str(fslocation),
         )
 
+    # the pytest_warning_captured hook was introduced in pytest 3.8
+    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
+
+        def pytest_warning_captured(self, warning_message, when, item):
+            self.sendevent(
+                "warning_captured",
+                warning_message_data=serialize_warning_message(warning_message),
+                when=when,
+                # item cannot be serialized and will always be None when used with xdist
+                item=None,
+            )
+
 
 def serialize_report(rep):
     def disassembled_report(rep):
@@ -163,6 +178,47 @@ def serialize_report(rep):
     return d
 
 
+def serialize_warning_message(warning_message):
+    if isinstance(warning_message.message, Warning):
+        message_module = type(warning_message.message).__module__
+        message_class_name = type(warning_message.message).__name__
+        message_str = str(warning_message.message)
+        # check now if we can serialize the warning arguments (#349)
+        # if not, we will just use the exception message on the master node
+        try:
+            dumps(warning_message.message.args)
+        except DumpError:
+            message_args = None
+        else:
+            message_args = warning_message.message.args
+    else:
+        message_str = warning_message.message
+        message_module = None
+        message_class_name = None
+        message_args = None
+    if warning_message.category:
+        category_module = warning_message.category.__module__
+        category_class_name = warning_message.category.__name__
+    else:
+        category_module = None
+        category_class_name = None
+
+    result = {
+        "message_str": message_str,
+        "message_module": message_module,
+        "message_class_name": message_class_name,
+        "message_args": message_args,
+        "category_module": category_module,
+        "category_class_name": category_class_name,
+    }
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warning_message._WARNING_DETAILS:
+        if attr_name in ("message", "category"):
+            continue
+        result[attr_name] = getattr(warning_message, attr_name)
+    return result
+
+
 def getinfodict():
     import platform
 
@@ -186,6 +242,7 @@ def remote_initconfig(option_dict, args):
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
+    config.option.maxprocesses = None
     config.args = args
     return config
 

xdist/scheduler/__init__.py

@@ -1,4 +1,4 @@
 from xdist.scheduler.each import EachScheduling  # noqa
 from xdist.scheduler.load import LoadScheduling  # noqa
+from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
-from xdist.scheduler.filescope import LoadFileScheduling  # noqa

xdist/scheduler/each.py

@@ -126,6 +126,7 @@ class EachScheduling(object):
             if not pending:
                 pending[:] = range(len(self.node2collection[node]))
                 node.send_runtest_all()
+                node.shutdown()
             else:
                 node.send_runtest_some(pending)
             self._started.append(node)

xdist/scheduler/load.py

@@ -178,6 +178,9 @@ class LoadScheduling(object):
                     return
                 num_send = items_per_node_max - len(node_pending)
                 self._send_tests(node, num_send)
+        else:
+            node.shutdown()
+
         self.log("num items waiting for node:", len(self.pending))
 
     def remove_node(self, node):

xdist/scheduler/loadfile.py

@@ -1,4 +1,4 @@
-from . import LoadScopeScheduling
+from .loadscope import LoadScopeScheduling
 from py.log import Producer
 
 

xdist/scheduler/loadscope.py

@@ -306,6 +306,7 @@ class LoadScopeScheduling(object):
 
         # Check that more work is available
         if not self.workqueue:
+            node.shutdown()
             return
 
         self.log("Number of units waiting for node:", len(self.workqueue))

xdist/workermanage.py

@@ -327,6 +327,16 @@ class WorkerController(object):
                     nodeid=kwargs["nodeid"],
                     fslocation=kwargs["nodeid"],
                 )
+            elif eventname == "warning_captured":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    item=kwargs["item"],
+                )
             else:
                 raise ValueError("unknown event: %s" % (eventname,))
         except KeyboardInterrupt:
@@ -409,6 +419,48 @@ def unserialize_report(name, reportdict):
         return runner.CollectReport(**assembled_report(reportdict))
 
 
+def unserialize_warning_message(data):
+    import warnings
+    import importlib
+
+    if data["message_module"]:
+        mod = importlib.import_module(data["message_module"])
+        cls = getattr(mod, data["message_class_name"])
+        message = None
+        if data["message_args"] is not None:
+            try:
+                message = cls(*data["message_args"])
+            except TypeError:
+                pass
+        if message is None:
+            # could not recreate the original warning instance;
+            # create a generic Warning instance with the original
+            # message at least
+            message_text = "{mod}.{cls}: {msg}".format(
+                mod=data["message_module"],
+                cls=data["message_class_name"],
+                msg=data["message_str"],
+            )
+            message = Warning(message_text)
+    else:
+        message = data["message_str"]
+
+    if data["category_module"]:
+        mod = importlib.import_module(data["category_module"])
+        category = getattr(mod, data["category_class_name"])
+    else:
+        category = None
+
+    kwargs = {"message": message, "category": category}
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:
+        if attr_name in ("message", "category"):
+            continue
+        kwargs[attr_name] = data[attr_name]
+
+    return warnings.WarningMessage(**kwargs)
+
+
 def report_unserialization_failure(type_name, report_name, reportdict):
     from pprint import pprint