fix issue 594: xdist is not executing tests parametrized with random values

Now xdist properly reports the collection errors instead of silently failing to execute
the test suite.

testing/acceptance_test.py

@@ -485,6 +485,28 @@ def test_fixture_scope_caching_issue503(testdir):
     ])
 
 
+def test_issue_594_random_parametrize(testdir):
+    """
+    Make sure that tests that are randomly parametrized display an appropriate
+    error message, instead of silently skipping the entire test run.
+    """
+    p1 = testdir.makepyfile("""
+        import pytest
+        import random
+
+        xs = list(range(10))
+        random.shuffle(xs)
+        @pytest.mark.parametrize('x', xs)
+        def test_foo(x):
+            assert 1
+    """)
+    result = testdir.runpytest(p1, '-v', '-n4')
+    assert result.ret == 1
+    result.stdout.fnmatch_lines([
+        "Different tests were collected between gw* and gw*",
+    ])
+
+
 class TestNodeFailure:
 
     def test_load_single(self, testdir):

testing/test_dsession.py

@@ -144,24 +144,36 @@ class TestLoadScheduling:
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
 
-    def test_schedule_different_tests_collected(self):
+    def test_different_tests_collected(self, testdir):
         """
-        Test that LoadScheduling is logging different tests were
-        collected by slaves when that happens.
+        Test that LoadScheduling is reporting collection errors when
+        different test ids are collected by slaves.
         """
+        class CollectHook(object):
+            """
+            Dummy hook that stores collection reports.
+            """
+
+            def __init__(self):
+                self.reports = []
+
+            def pytest_collectreport(self, report):
+                self.reports.append(report)
+
+        collect_hook = CollectHook()
+        config = testdir.parseconfig()
+        config.pluginmanager.register(collect_hook, "collect_hook")
         node1 = MockNode()
         node2 = MockNode()
-        sched = LoadScheduling(2)
-        logged_messages = []
-        py.log.setconsumer('loadsched', logged_messages.append)
+        sched = LoadScheduling(2, config=config)
         sched.addnode(node1)
         sched.addnode(node2)
         sched.addnode_collection(node1, ["a.py::test_1"])
         sched.addnode_collection(node2, ["a.py::test_2"])
         sched.init_distribute()
-        logged_content = ''.join(x.content() for x in logged_messages)
-        assert 'Different tests were collected between' in logged_content
-        assert 'Different tests collected, aborting run' in logged_content
+        assert len(collect_hook.reports) == 1
+        rep = collect_hook.reports[0]
+        assert 'Different tests were collected between' in rep.longrepr
 
 
 class TestDistReporter: