Do not show internal error when running out of workers (#437)

Do not show internal error when running out of workers

changelog/435.bugfix.rst

@@ -0,0 +1 @@
+No longer show an internal error when we run out of workers due to crashes.

changelog/435.feature.rst

@@ -0,0 +1,2 @@
+When the test session is interrupted due to running out of workers, the reason is shown in the test summary
+for easier viewing.

src/xdist/dsession.py

@@ -47,7 +47,8 @@ class DSession(object):
         self._active_nodes = set()
         self._failed_nodes_count = 0
         self._max_worker_restart = get_default_max_worker_restart(self.config)
-
+        # summary message to print at the end of the session
+        self._summary_report = None
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -197,15 +198,23 @@ class DSession(object):
         )
         if maximum_reached:
             if self._max_worker_restart == 0:
-                msg = "Worker restarting disabled"
+                msg = "worker {} crashed and worker restarting disabled".format(
+                    node.gateway.id
+                )
             else:
-                msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
+                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
-            self.report_line(msg)
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
         else:
-            self.report_line("Replacing crashed worker %s" % node.gateway.id)
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
+    def pytest_terminal_summary(self, terminalreporter):
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
+
     def worker_collectionfinish(self, node, ids):
         """worker has finished test collection.
 
@@ -315,7 +324,7 @@ class DSession(object):
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
+        msg = "worker %r crashed while running %r" % (worker.gateway.id, nodeid)
         rep = runner.TestReport(
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )

testing/acceptance_test.py

@@ -849,8 +849,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -868,8 +868,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n2")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*3 passed*",
             ]
         )
@@ -885,8 +885,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "--dist=each", "--tx=popen")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker gw*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -922,14 +922,35 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
+                "replacing crashed worker*",
-                "*Maximum crashed workers reached: 1*",
+                "maximum crashed workers reached: 1*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
-                "*Worker*crashed while running*",
+                "worker*crashed while running*",
                 "*2 failed*2 passed*",
             ]
         )
 
+    def test_max_worker_restart_tests_queued(self, testdir):
+        f = testdir.makepyfile(
+            """
+            import os, pytest
+            @pytest.mark.parametrize('i', range(10))
+            def test(i): os._exit(1)
+        """
+        )
+        res = testdir.runpytest(f, "-n2", "--max-worker-restart=3")
+        res.stdout.fnmatch_lines(
+            [
+                "replacing crashed worker*",
+                "maximum crashed workers reached: 3*",
+                "worker*crashed while running*",
+                "worker*crashed while running*",
+                "* xdist: maximum crashed workers reached: 3 *",
+                "* 4 failed in *",
+            ]
+        )
+        assert "INTERNALERROR" not in res.stdout.str()
+
     def test_max_worker_restart_die(self, testdir):
         f = testdir.makepyfile(
             """
@@ -939,7 +960,10 @@ class TestNodeFailure:
         )
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
-            ["*Unexpectedly no active workers*", "*INTERNALERROR*"]
+            [
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* no tests ran in *",
+            ]
         )
 
     def test_disable_restart(self, testdir):
@@ -954,9 +978,10 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
             [
-                "*Worker restarting disabled*",
+                "worker gw* crashed and worker restarting disabled",
-                "*Worker*crashed while running*",
+                "*worker*crashed while running*",
-                "*1 failed*2 passed*",
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* 1 failed, 2 passed in *",
             ]
         )