Release 1.24.0

Add --maxprocesses option to use with --numprocesses=auto

CHANGELOG.rst

@@ -1,3 +1,18 @@
+pytest-xdist 1.24.0 (2018-10-18)
+================================
+
+Features
+--------
+
+- `#337 <https://github.com/pytest-dev/pytest-xdist/issues/337>`_: New ``--maxprocesses`` command-line option that limits the maximum number of workers when using ``--numprocesses=auto``.
+
+
+Bug Fixes
+---------
+
+- `#351 <https://github.com/pytest-dev/pytest-xdist/issues/351>`_: Fix scheduling deadlock in case of inter-test locking.
+
+
 pytest-xdist 1.23.2 (2018-09-28)
 ================================
 

testing/acceptance_test.py

@@ -1132,6 +1132,56 @@ class TestFileScope:
         assert c1 == c2
 
 
+class TestLocking:
+    _test_content = """
+    class TestClassName%s(object):
+
+        @classmethod
+        def setup_class(cls):
+            FILE_LOCK.acquire()
+
+        @classmethod
+        def teardown_class(cls):
+            FILE_LOCK.release()
+
+        def test_a(self):
+            pass
+
+        def test_b(self):
+            pass
+
+        def test_c(self):
+            pass
+
+    """
+
+    test_file1 = """
+    import filelock
+
+    FILE_LOCK = filelock.FileLock("test.lock")
+
+    """ + (
+        (_test_content * 4) % ("A", "B", "C", "D")
+    )
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_single_file(self, testdir, scope):
+        testdir.makepyfile(test_a=self.test_file1)
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(12 if scope != "each" else 12 * 2))
+
+    @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
+    def test_multi_file(self, testdir, scope):
+        testdir.makepyfile(
+            test_a=self.test_file1,
+            test_b=self.test_file1,
+            test_c=self.test_file1,
+            test_d=self.test_file1,
+        )
+        result = testdir.runpytest("-n2", "--dist=%s" % scope, "-v")
+        result.assert_outcomes(passed=(48 if scope != "each" else 48 * 2))
+
+
 def parse_tests_and_workers_from_output(lines):
     result = []
     for line in lines:

testing/test_plugin.py

@@ -25,6 +25,10 @@ def test_dist_options(testdir):
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ["popen"] * 2
+    config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ["popen"] * 2
     config = testdir.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"

tox.ini

@@ -23,6 +23,7 @@ deps =
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
   pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect
+  filelock
 platform=
   pexpect: linux|darwin
 commands=

xdist/plugin.py

@@ -47,6 +47,14 @@ def pytest_addoption(parser):
         "you can use 'auto' here for auto detection CPUs number on "
         "host system",
     )
+    group.addoption(
+        "--maxprocesses",
+        dest="maxprocesses",
+        metavar="maxprocesses",
+        action="store",
+        type=int,
+        help="limit the maximum number of workers to process the tests when using --numprocesses=auto",
+    )
     group.addoption(
         "--max-worker-restart",
         "--max-slave-restart",
@@ -172,7 +180,10 @@ def pytest_cmdline_main(config):
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
-        config.option.tx = ["popen"] * config.option.numprocesses
+        numprocesses = config.option.numprocesses
+        if config.option.maxprocesses:
+            numprocesses = min(numprocesses, config.option.maxprocesses)
+        config.option.tx = ["popen"] * numprocesses
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue

xdist/remote.py

@@ -234,6 +234,7 @@ def remote_initconfig(option_dict, args):
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
+    config.option.maxprocesses = None
     config.args = args
     return config
 

xdist/scheduler/each.py

@@ -126,6 +126,7 @@ class EachScheduling(object):
             if not pending:
                 pending[:] = range(len(self.node2collection[node]))
                 node.send_runtest_all()
+                node.shutdown()
             else:
                 node.send_runtest_some(pending)
             self._started.append(node)

xdist/scheduler/load.py

@@ -178,6 +178,9 @@ class LoadScheduling(object):
                     return
                 num_send = items_per_node_max - len(node_pending)
                 self._send_tests(node, num_send)
+        else:
+            node.shutdown()
+
         self.log("num items waiting for node:", len(self.pending))
 
     def remove_node(self, node):

xdist/scheduler/loadscope.py

@@ -306,6 +306,7 @@ class LoadScopeScheduling(object):
 
         # Check that more work is available
         if not self.workqueue:
+            node.shutdown()
             return
 
         self.log("Number of units waiting for node:", len(self.workqueue))