Release 1.21.1

Rename filescope to loadfile

.travis.yml

@@ -21,24 +21,26 @@ env:
 - TOXENV=py-pytest31
 - TOXENV=py-pytest32
 - TOXENV=py-pytest33
-- TOXENV=py-pytest36
-- TOXENV=py-pytest38
 
 install: pip install tox setuptools_scm
 script: tox
 
 stages:
-- linting
+- baseline
 - test
 - name: deploy
   if: repo = pytest-dev/pytest-xdist AND tag IS present
 
 jobs:
   include:
-    - stage: linting
+    - stage: baseline
       python: '3.6'
-      script:
-      - tox -e linting
+      env: TOXENV=linting
+    - python: '3.6'
+      env: TOXENV=py36-pytestlatest
+    - python: '2.7'
+      env: TOXENV=py27-pytestlatest
+
     - stage: test
     # python x env above are already included into this stage
     - python: "2.7"

CHANGELOG.rst

@@ -1,3 +1,27 @@
+pytest-xdist 1.24.1 (2018-11-09)
+================================
+
+Bug Fixes
+---------
+
+- `#349 <https://github.com/pytest-dev/pytest-xdist/issues/349>`_: Correctly handle warnings created with arguments that can't be serialized during the transfer from workers to master node.
+
+
+pytest-xdist 1.24.0 (2018-10-18)
+================================
+
+Features
+--------
+
+- `#337 <https://github.com/pytest-dev/pytest-xdist/issues/337>`_: New ``--maxprocesses`` command-line option that limits the maximum number of workers when using ``--numprocesses=auto``.
+
+
+Bug Fixes
+---------
+
+- `#351 <https://github.com/pytest-dev/pytest-xdist/issues/351>`_: Fix scheduling deadlock in case of inter-test locking.
+
+
 pytest-xdist 1.23.2 (2018-09-28)
 ================================
 

appveyor.yml

@@ -5,8 +5,7 @@ environment:
   - TOXENV: "py34-pytest33"
   - TOXENV: "py35-pytest33"
   - TOXENV: "py36-pytest33"
-  - TOXENV: "py36-pytest36"
-  - TOXENV: "py36-pytest38"
+  - TOXENV: "py36-pytestlatest"
   - TOXENV: "py27-pytest33-pexpect"
   - TOXENV: "py36-pytest33-pexpect"
 

testing/acceptance_test.py

@@ -400,64 +400,6 @@ class TestTerminalReporting:
             ]
         )
 
-    @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
-    def test_warnings(self, testdir, n, warn_type):
-        from pkg_resources import parse_version
-
-        if parse_version(pytest.__version__) < parse_version("3.1"):
-            pytest.skip("pytest warnings requires >= 3.1")
-
-        if warn_type == "builtin":
-            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
-        elif warn_type == "pytest":
-            warn_code = """request.config.warn('', 'this is a warning',
-                           fslocation=py.path.local())"""
-        else:
-            assert False
-        testdir.makepyfile(
-            """
-            import warnings, py, pytest
-
-            @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
-            def test_func(request):
-                {warn_code}
-        """.format(
-                warn_code=warn_code
-            )
-        )
-        result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
-
-    @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    def test_custom_subclass(self, testdir, n):
-        """Check that warning subclasses that don't honor the args attribute don't break
-        pytest-xdist (#344)
-        """
-        from pkg_resources import parse_version
-
-        if parse_version(pytest.__version__) < parse_version("3.1"):
-            pytest.skip("pytest warnings requires >= 3.1")
-
-        testdir.makepyfile(
-            """
-            import warnings, py, pytest
-
-            class MyWarning(UserWarning):
-
-                def __init__(self, p1, p2):
-                    self.p1 = p1
-                    self.p2 = p2
-                    self.args = ()
-
-            def test_func(request):
-                warnings.warn(MyWarning("foo", 1))
-        """
-        )
-        testdir.syspathinsert()
-        result = testdir.runpytest(n)
-        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
-
     def test_logfinish_hook(self, testdir):
         """Ensure the pytest_runtest_logfinish hook is being properly handled"""
         from _pytest import hookspec
@@ -765,6 +707,83 @@ def test_sub_plugins_disabled(testdir, plugin):
     result.stdout.fnmatch_lines("*1 passed*")
 
 
+class TestWarnings:
+    @pytest.fixture(autouse=True)
+    def skip_if_unsupported_pytest_version(self):
+        """Skip tests of this class if we are running in a pytest version which does not
+        support warnings yet.
+        """
+        from pkg_resources import parse_version
+
+        if parse_version(pytest.__version__) < parse_version("3.1"):
+            pytest.skip("pytest warnings requires >= 3.1")
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
+    def test_warnings(self, testdir, n, warn_type):
+        if warn_type == "builtin":
+            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
+        elif warn_type == "pytest":
+            warn_code = """request.config.warn('', 'this is a warning',
+                           fslocation=py.path.local())"""
+        else:
+            assert False
+        testdir.makepyfile(
+            """
+            import warnings, py, pytest
+
+            @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
+            def test_func(request):
+                {warn_code}
+        """.format(
+                warn_code=warn_code
+            )
+        )
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_custom_subclass(self, testdir, n):
+        """Check that warning subclasses that don't honor the args attribute don't break
+        pytest-xdist (#344)
+        """
+        testdir.makepyfile(
+            """
+            import warnings, py, pytest
+
+            class MyWarning(UserWarning):
+
+                def __init__(self, p1, p2):
+                    self.p1 = p1
+                    self.p2 = p2
+                    self.args = ()
+
+            def test_func(request):
+                warnings.warn(MyWarning("foo", 1))
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
+
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_unserializable_arguments(self, testdir, n):
+        """Check that warnings with unserializable arguments are handled correctly (#349)."""
+        testdir.makepyfile(
+            """
+            import warnings, pytest
+
+            def test_func(tmpdir):
+                fn = (tmpdir / 'foo.txt').ensure(file=1)
+                with fn.open('r') as f:
+                    warnings.warn(UserWarning("foo", f))
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*UserWarning*foo.txt*", "*1 passed, 1 warnings*"])
+
+
 class TestNodeFailure:
     def test_load_single(self, testdir):
         f = testdir.makepyfile(
@@ -1132,6 +1151,56 @@ class TestFileScope:
         assert c1 == c2
 
 
+class TestLocking:
+    _test_content = """
+    class TestClassName%s(object):
+
+        @classmethod
+        def setup_class(cls):
+            FILE_LOCK.acquire()
+
+        @classmethod
+        def teardown_class(cls):
+            FILE_LOCK.release()
+
+        def test_a(self):
+            pass
+
+        def test_b(self):
+            pass
+
+        def test_c(self):
+            pass
+
+    """
+
+    test_file1 = """
+    import filelock
+
+    FILE_LOCK = filelock.FileLock("test.lock")
+
+    """ + (
+        (_test_content * 4) % ("A", "B", "C", "D")
+    )
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_single_file(self, testdir, scope):
+        testdir.makepyfile(test_a=self.test_file1)
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(12 if scope != "each" else 12 * 2))
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_multi_file(self, testdir, scope):
+        testdir.makepyfile(
+            test_a=self.test_file1,
+            test_b=self.test_file1,
+            test_c=self.test_file1,
+            test_d=self.test_file1,
+        )
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(48 if scope != "each" else 48 * 2))
+
+
 def parse_tests_and_workers_from_output(lines):
     result = []
     for line in lines:

testing/test_plugin.py

@@ -25,6 +25,10 @@ def test_dist_options(testdir):
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
+    config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
     config = testdir.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"

tox.ini

@@ -2,7 +2,7 @@
 # if you change the envlist, please update .travis.yml file as well
 envlist=
   linting
-  py{27,34,35,36}-pytest{30,31,32,33,36,38}
+  py{27,34,35,36}-pytest{30,31,32,33,latest}
   py{27,36}-pytest36-pexpect
   py{27,36}-pytest{master,features}
 
@@ -18,16 +18,14 @@ deps =
   pytest31: pytest~=3.1.0
   pytest32: pytest~=3.2.0
   pytest33: pytest~=3.3.0
-  pytest36: pytest~=3.6.0
-  pytest38: pytest~=3.8.0
+  pytestlatest: pytest
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
   pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect
+  filelock
 platform=
   pexpect: linux|darwin
 commands=
-  # always clean to avoid code unmarshal mismatch on old python/pytest
-  py.cleanup -aq
   pytest {posargs}
 
 [testenv:linting]

xdist/plugin.py

@@ -47,6 +47,14 @@ def pytest_addoption(parser):
         "you can use 'auto' here for auto detection CPUs number on "
         "host system",
     )
+    group.addoption(
+        "--maxprocesses",
+        dest="maxprocesses",
+        metavar="maxprocesses",
+        action="store",
+        type=int,
+        help="limit the maximum number of workers to process the tests when using --numprocesses=auto",
+    )
     group.addoption(
         "--max-worker-restart",
         "--max-slave-restart",
@@ -172,7 +180,10 @@ def pytest_cmdline_main(config):
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
-        config.option.tx = ["popen"] * config.option.numprocesses
+        numprocesses = config.option.numprocesses
+        if config.option.maxprocesses:
+            numprocesses = min(numprocesses, config.option.maxprocesses)
+        config.option.tx = ["popen"] * numprocesses
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue

xdist/remote.py

@@ -12,6 +12,7 @@ import time
 
 import _pytest.hookspec
 import pytest
+from execnet.gateway_base import dumps, DumpError
 
 
 class WorkerInteractor(object):
@@ -181,8 +182,15 @@ def serialize_warning_message(warning_message):
     if isinstance(warning_message.message, Warning):
         message_module = type(warning_message.message).__module__
         message_class_name = type(warning_message.message).__name__
-        message_args = warning_message.message.args
         message_str = str(warning_message.message)
+        # check now if we can serialize the warning arguments (#349)
+        # if not, we will just use the exception message on the master node
+        try:
+            dumps(warning_message.message.args)
+        except DumpError:
+            message_args = None
+        else:
+            message_args = warning_message.message.args
     else:
         message_str = warning_message.message
         message_module = None
@@ -234,6 +242,7 @@ def remote_initconfig(option_dict, args):
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
+    config.option.maxprocesses = None
     config.args = args
     return config
 

xdist/scheduler/__init__.py

@@ -1,4 +1,4 @@
 from xdist.scheduler.each import EachScheduling  # noqa
 from xdist.scheduler.load import LoadScheduling  # noqa
+from xdist.scheduler.loadfile import LoadFileScheduling  # noqa
 from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
-from xdist.scheduler.filescope import LoadFileScheduling  # noqa

xdist/scheduler/each.py

@@ -126,6 +126,7 @@ class EachScheduling(object):
             if not pending:
                 pending[:] = range(len(self.node2collection[node]))
                 node.send_runtest_all()
+                node.shutdown()
             else:
                 node.send_runtest_some(pending)
             self._started.append(node)

xdist/scheduler/load.py

@@ -178,6 +178,9 @@ class LoadScheduling(object):
                     return
                 num_send = items_per_node_max - len(node_pending)
                 self._send_tests(node, num_send)
+        else:
+            node.shutdown()
+
         self.log("num items waiting for node:", len(self.pending))
 
     def remove_node(self, node):

xdist/scheduler/loadfile.py

@@ -1,4 +1,4 @@
-from . import LoadScopeScheduling
+from .loadscope import LoadScopeScheduling
 from py.log import Producer
 
 

xdist/scheduler/loadscope.py

@@ -306,6 +306,7 @@ class LoadScopeScheduling(object):
 
         # Check that more work is available
         if not self.workqueue:
+            node.shutdown()
             return
 
         self.log("Number of units waiting for node:", len(self.workqueue))

xdist/workermanage.py

@@ -426,9 +426,16 @@ def unserialize_warning_message(data):
     if data["message_module"]:
         mod = importlib.import_module(data["message_module"])
         cls = getattr(mod, data["message_class_name"])
+        message = None
+        if data["message_args"] is not None:
             try:
                 message = cls(*data["message_args"])
             except TypeError:
+                pass
+        if message is None:
+            # could not recreate the original warning instance;
+            # create a generic Warning instance with the original
+            # message at least
             message_text = "{mod}.{cls}: {msg}".format(
                 mod=data["message_module"],
                 cls=data["message_class_name"],