From 9e59d079475a14bb70c03e7c17c6fdf7c3083ccf Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Wed, 4 Oct 2017 08:32:12 -0700 Subject: [PATCH] Stop looping if all workers have died If the workers are crashing and the restart limit has been met, we need to stop listening for events and trigger an internal error. --- changelog/45.bugfix | 1 + testing/acceptance_test.py | 11 +++++++++++ xdist/dsession.py | 4 ++++ 3 files changed, 16 insertions(+) create mode 100644 changelog/45.bugfix diff --git a/changelog/45.bugfix b/changelog/45.bugfix new file mode 100644 index 0000000..e4727de --- /dev/null +++ b/changelog/45.bugfix @@ -0,0 +1 @@ +Fix hang when all worker nodes crash and restart limit is reached diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 42345d7..ea5d204 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -701,6 +701,17 @@ class TestNodeFailure: "*2 failed*2 passed*", ]) + def test_max_slave_restart_die(self, testdir): + f = testdir.makepyfile(""" + import os + os._exit(1) + """) + res = testdir.runpytest(f, '-n4', '--max-slave-restart=0') + res.stdout.fnmatch_lines([ + "*Unexpectedly no active workers*", + "*INTERNALERROR*" + ]) + def test_disable_restart(self, testdir): f = testdir.makepyfile(""" import os diff --git a/xdist/dsession.py b/xdist/dsession.py index ed459ba..42277c9 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -120,6 +120,10 @@ class DSession: def loop_once(self): """Process one callback from one of the slaves.""" while 1: + if not self._active_nodes: + # If everything has died stop looping + self.triggershutdown() + raise RuntimeError("Unexpectedly no active workers available") try: eventcall = self.queue.get(timeout=2.0) break