Add --max-slave-restart option

Also changed wording used from "failed node" to "crashed slave", to conform
with other messages ("slave sw0 crashed")

CHANGELOG

@@ -1,6 +1,9 @@
 1.13.dev1
 -------------------------
 
+- new "--max-slave-restart" option that can be used to control maximum
+  number of times pytest-xdist can restart slaves due to crashes. Thanks to
+  Anatoly Bubenkov for the report and Bruno Oliveira for the PR.
 - release as wheel
 - "-n" option now can be set to "auto" for automatic detection of number
   of cpus in the host system. Thanks Suloev Dmitry for the PR.

README.txt

@@ -62,6 +62,11 @@ Especially for longer running tests or tests requiring
 a lot of IO this can lead to considerable speed ups. This option can
 also be set to ``auto`` for automatic detection of the number of CPUs.
 
+If a test crashes the interpreter, pytest-xdist will automatically restart
+that slave and report the failure as usual. You can use the
+``--max-slave-restart`` option to limit the number of slaves that can
+be restarted, or disable restarting altogether using ``--max-slave-restart=0``.
+
 
 Running tests in a Python subprocess
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

testing/acceptance_test.py

@@ -529,7 +529,7 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n1')
         res.stdout.fnmatch_lines([
-            "*Replacing failed node*",
+            "*Replacing crashed slave*",
             "*Slave*crashed while running*",
             "*1 failed*1 passed*",
         ])
@@ -544,7 +544,7 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n2')
         res.stdout.fnmatch_lines([
-            "*Replacing failed node*",
+            "*Replacing crashed slave*",
             "*Slave*crashed while running*",
             "*1 failed*3 passed*",
         ])
@@ -557,7 +557,7 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=popen')
         res.stdout.fnmatch_lines([
-            "*Replacing failed node*",
+            "*Replacing crashed slave*",
             "*Slave*crashed while running*",
             "*1 failed*1 passed*",
         ])
@@ -570,7 +570,39 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=2*popen')
         res.stdout.fnmatch_lines([
-            "*Replacing failed node*",
+            "*Replacing crashed slave*",
             "*Slave*crashed while running*",
             "*2 failed*2 passed*",
         ])
+
+    def test_max_slave_restart(self, testdir):
+        f = testdir.makepyfile("""
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): os._exit(1)
+            def test_d(): pass
+        """)
+        res = testdir.runpytest(f, '-n4', '--max-slave-restart=1')
+        res.stdout.fnmatch_lines([
+            "*Replacing crashed slave*",
+            "*Maximum crashed slaves reached: 1*",
+            "*Slave*crashed while running*",
+            "*Slave*crashed while running*",
+            "*2 failed*2 passed*",
+        ])
+
+
+    def test_disable_restart(self, testdir):
+        f = testdir.makepyfile("""
+            import os
+            def test_a(): pass
+            def test_b(): os._exit(1)
+            def test_c(): pass
+        """)
+        res = testdir.runpytest(f, '-n4', '--max-slave-restart=0')
+        res.stdout.fnmatch_lines([
+            "*Slave restarting disabled*",
+            "*Slave*crashed while running*",
+            "*1 failed*2 passed*",
+        ])

xdist/dsession.py

@@ -458,6 +458,10 @@ class DSession:
         self._session = None
         self._failed_collection_errors = {}
         self._active_nodes = set()
+        self._failed_nodes_count = 0
+        self._max_slave_restart = self.config.getoption('max_slave_restart')
+        if self._max_slave_restart is not None:
+            self._max_slave_restart = int(self._max_slave_restart)
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -583,8 +587,20 @@ class DSession:
         else:
             if crashitem:
                 self.handle_crashitem(crashitem, node)
-        self.report_line("Replacing failed node %s" % node.gateway.id)
+
-        self._clone_node(node)
+        self._failed_nodes_count += 1
+        maximum_reached = (self._max_slave_restart is not None and
+                           self._failed_nodes_count > self._max_slave_restart)
+        if maximum_reached:
+            if self._max_slave_restart == 0:
+                msg = 'Slave restarting disabled'
+            else:
+                msg = "Maximum crashed slaves reached: %d" % \
+                      self._max_slave_restart
+            self.report_line(msg)
+        else:
+            self.report_line("Replacing crashed slave %s" % node.gateway.id)
+            self._clone_node(node)
         self._active_nodes.remove(node)
 
     def slave_collectionfinish(self, node, ids):

xdist/plugin.py

@@ -14,6 +14,9 @@ def pytest_addoption(parser):
            help="shortcut for '--dist=load --tx=NUM*popen', "
                 "you can use 'auto' here for auto detection CPUs number on "
                 "host system")
+    group._addoption('--max-slave-restart', action="store", default=None,
+                     help="maximum number of slaves that can be restarted "
+                          "when crashed (set to zero to disable this feature)")
     group.addoption('--boxed',
            action="store_true", dest="boxed", default=False,
            help="box each test run in a separate process (unix)")