From 4b1ddb9c816f22f2f27901a1a248f33dd1a3a2e7 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 11 Jul 2015 13:12:12 -0300 Subject: [PATCH] Add --max-slave-restart option Also changed wording used from "failed node" to "crashed slave", to conform with other messages ("slave sw0 crashed") --- CHANGELOG | 3 +++ README.txt | 5 +++++ testing/acceptance_test.py | 40 ++++++++++++++++++++++++++++++++++---- xdist/dsession.py | 20 +++++++++++++++++-- xdist/plugin.py | 3 +++ 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7cf76e7..76e4775 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ 1.13.dev1 ------------------------- +- new "--max-slave-restart" option that can be used to control maximum + number of times pytest-xdist can restart slaves due to crashes. Thanks to + Anatoly Bubenkov for the report and Bruno Oliveira for the PR. - release as wheel - "-n" option now can be set to "auto" for automatic detection of number of cpus in the host system. Thanks Suloev Dmitry for the PR. diff --git a/README.txt b/README.txt index 0dbbfee..8b9f549 100644 --- a/README.txt +++ b/README.txt @@ -62,6 +62,11 @@ Especially for longer running tests or tests requiring a lot of IO this can lead to considerable speed ups. This option can also be set to ``auto`` for automatic detection of the number of CPUs. +If a test crashes the interpreter, pytest-xdist will automatically restart +that slave and report the failure as usual. You can use the +``--max-slave-restart`` option to limit the number of slaves that can +be restarted, or disable restarting altogether using ``--max-slave-restart=0``. + Running tests in a Python subprocess +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 0d04929..c357641 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -529,7 +529,7 @@ class TestNodeFailure: """) res = testdir.runpytest(f, '-n1') res.stdout.fnmatch_lines([ - "*Replacing failed node*", + "*Replacing crashed slave*", "*Slave*crashed while running*", "*1 failed*1 passed*", ]) @@ -544,7 +544,7 @@ class TestNodeFailure: """) res = testdir.runpytest(f, '-n2') res.stdout.fnmatch_lines([ - "*Replacing failed node*", + "*Replacing crashed slave*", "*Slave*crashed while running*", "*1 failed*3 passed*", ]) @@ -557,7 +557,7 @@ class TestNodeFailure: """) res = testdir.runpytest(f, '--dist=each', '--tx=popen') res.stdout.fnmatch_lines([ - "*Replacing failed node*", + "*Replacing crashed slave*", "*Slave*crashed while running*", "*1 failed*1 passed*", ]) @@ -570,7 +570,39 @@ class TestNodeFailure: """) res = testdir.runpytest(f, '--dist=each', '--tx=2*popen') res.stdout.fnmatch_lines([ - "*Replacing failed node*", + "*Replacing crashed slave*", "*Slave*crashed while running*", "*2 failed*2 passed*", ]) + + def test_max_slave_restart(self, testdir): + f = testdir.makepyfile(""" + import os + def test_a(): pass + def test_b(): os._exit(1) + def test_c(): os._exit(1) + def test_d(): pass + """) + res = testdir.runpytest(f, '-n4', '--max-slave-restart=1') + res.stdout.fnmatch_lines([ + "*Replacing crashed slave*", + "*Maximum crashed slaves reached: 1*", + "*Slave*crashed while running*", + "*Slave*crashed while running*", + "*2 failed*2 passed*", + ]) + + + def test_disable_restart(self, testdir): + f = testdir.makepyfile(""" + import os + def test_a(): pass + def test_b(): os._exit(1) + def test_c(): pass + """) + res = testdir.runpytest(f, '-n4', '--max-slave-restart=0') + res.stdout.fnmatch_lines([ + "*Slave restarting disabled*", + "*Slave*crashed while running*", + "*1 failed*2 passed*", + ]) diff --git a/xdist/dsession.py b/xdist/dsession.py index ec40372..e669db6 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -458,6 +458,10 @@ class DSession: self._session = None self._failed_collection_errors = {} self._active_nodes = set() + self._failed_nodes_count = 0 + self._max_slave_restart = self.config.getoption('max_slave_restart') + if self._max_slave_restart is not None: + self._max_slave_restart = int(self._max_slave_restart) try: self.terminal = config.pluginmanager.getplugin("terminalreporter") except KeyError: @@ -583,8 +587,20 @@ class DSession: else: if crashitem: self.handle_crashitem(crashitem, node) - self.report_line("Replacing failed node %s" % node.gateway.id) - self._clone_node(node) + + self._failed_nodes_count += 1 + maximum_reached = (self._max_slave_restart is not None and + self._failed_nodes_count > self._max_slave_restart) + if maximum_reached: + if self._max_slave_restart == 0: + msg = 'Slave restarting disabled' + else: + msg = "Maximum crashed slaves reached: %d" % \ + self._max_slave_restart + self.report_line(msg) + else: + self.report_line("Replacing crashed slave %s" % node.gateway.id) + self._clone_node(node) self._active_nodes.remove(node) def slave_collectionfinish(self, node, ids): diff --git a/xdist/plugin.py b/xdist/plugin.py index 246d377..fc4bf00 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -14,6 +14,9 @@ def pytest_addoption(parser): help="shortcut for '--dist=load --tx=NUM*popen', " "you can use 'auto' here for auto detection CPUs number on " "host system") + group._addoption('--max-slave-restart', action="store", default=None, + help="maximum number of slaves that can be restarted " + "when crashed (set to zero to disable this feature)") group.addoption('--boxed', action="store_true", dest="boxed", default=False, help="box each test run in a separate process (unix)")