some test fixes and configure tox-testing against pytest==2.3.5 and pytest>=2.4.2

will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
bugs.

.hgtags

@@ -8,3 +8,8 @@ a423748bf17ee778a37853225210257699cad9c1 1.4
 cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
 4815040bdad8f182a5487f57a9da385483836e75 1.6
 20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
+20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
+29c38e195526f5f0fdd651fb51f59d6efaaafbb0 1.7
+0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
+0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
+1d27987c267577899350a25ba5828d55d87083ad 1.8

CHANGELOG

@@ -1,3 +1,24 @@
+1.9
+-------------------------
+
+- changed LICENSE to MIT
+
+- fix duplicate reported test ids with --looponfailing
+  (thanks Jeremy Thurgood)
+
+- fix pytest issue41: re-run tests on all file changes, not just
+  randomly select ones like .py/.c.
+
+- fix pytest issue347: slaves running on top of Python3.2 
+  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
+  bugs.
+
+1.8
+-------------------------
+
+- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol 
+  interface to perform eager teardowns for test items.
+
 1.7
 -------------------------
 

LICENSE

@@ -1,13 +1,19 @@
-The execnet package is released under the provisions of the Gnu Public
-License (GPL), version 2 or later.  
 
-See http://www.fsf.org/licensing/licenses/ for more information.
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+     
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+ 
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 
-This package also contains some minor parts which 
-which are useable under the MIT license. 
-
-If you have questions and/or want to use parts of 
-the code under a different license than the GPL 
-please contact me. 
-
-holger krekel, January 2010, holger at merlinux eu 

MANIFEST.in

@@ -2,9 +2,6 @@ include CHANGELOG
 include LICENSE 
 include README.txt
 include setup.py
+include tox.ini
 graft testing
-exclude *.orig 
-exclude *.rej 
-recursive-exclude testing *.pyc *$py.class
-prune .svn 
 prune .hg 

README.txt

@@ -4,16 +4,19 @@ xdist: pytest distributed testing plugin
 The `pytest-xdist`_ plugin extends py.test with some unique
 test execution modes:
 
-* Looponfail: run your tests repeatedly in a subprocess.  After each run py.test
-  waits until a file in your project changes and then re-runs the previously
-  failing tests.  This is repeated until all tests pass after which again
-  a full run is performed.
-
-* multiprocess Load-balancing: if you have multiple CPUs or hosts you can use
+* test run parallelization_: if you have multiple CPUs or hosts you can use
   those for a combined test run.  This allows to speed up
-  development or to use special resources of remote machines.
+  development or to use special resources of `remote machines`_.
 
-* Multi-Platform coverage: you can specify different Python interpreters
+* ``--boxed``: (not available on Windows) run each test in a boxed_ 
+  subprocess to survive ``SEGFAULTS`` or otherwise dying processes
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run 
+  py.test waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed.
+
+* `Multi-Platform`_ coverage: you can specify different Python interpreters
   or different platforms and run tests in parallel on all of them.
 
 Before running tests remotely, ``py.test`` efficiently "rsyncs" your
@@ -41,6 +44,8 @@ a checkout of the `pytest-xdist repository`_ ::
 Usage examples
 ---------------------
 
+.. _parallelization:
+
 Speed up test runs by sending tests to multiple CPUs
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -55,20 +60,42 @@ a lot of IO this can lead to considerable speed ups.
 Running tests in a Python subprocess
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-To instantiate a python2.4 sub process and send tests to it, you may type::
+To instantiate a python2.5 sub process and send tests to it, you may type::
 
-    py.test -d --tx popen//python=python2.4
+    py.test -d --tx popen//python=python2.5
 
-This will start a subprocess which is run with the "python2.4"
+This will start a subprocess which is run with the "python2.5"
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python2.4
+    --tx 3*popen//python=python2.5
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
 
+.. _boxed:
+
+Running tests in a boxed subprocess
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+If you have tests involving C or C++ libraries you might have to deal
+with tests crashing the process.  For this case you max use the boxing
+options::
+
+    py.test --boxed
+
+which will run each test in a subprocess and will report if a test
+crashed the process.  You can also combine this option with
+running multiple processes to speed up the test run and use your CPU cores::
+
+    py.test -n3 --boxed
+
+this would run 3 testing subprocesses in parallel which each 
+create new boxed subprocesses for each test.
+
+
+.. _`remote machines`:
 
 Sending tests to remote SSH accounts
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -111,6 +138,8 @@ new socket host with something like this::
 
 
 .. _`atonce`:
+.. _`Multi-Platform`:
+
 
 Running tests on many platforms at once
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -168,3 +197,9 @@ where the configuration file was found.
 .. _`pytest-xdist repository`: http://bitbucket.org/hpk42/pytest-xdist
 .. _`pytest`: http://pytest.org
 
+Issue and Bug Tracker
+------------------------
+
+Please use the pytest issue tracker for bugs in this plugin, see https://bitbucket.org/hpk42/pytest/issues .
+
+

example/boxed.txt

@@ -0,0 +1,62 @@
+
+
+If your testing involves C or C++ libraries you might have to deal
+with crashing processes.  The xdist-plugin provides the ``--boxed`` option
+to run each test in a controled subprocess.  Here is a basic example::
+
+    # content of test_module.py
+
+    import pytest
+    import os
+    import time
+
+    # run test function 50 times with different argument
+    @pytest.mark.parametrize("arg", range(50))
+    def test_func(arg):
+        time.sleep(0.05) # each tests takes a while
+        if arg % 19 == 0: 
+            os.kill(os.getpid(), 15)
+
+If you run this with::
+
+    $ py.test --boxed
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    collecting ... collected 50 items
+    
+    test_module.py f..................f..................f...........
+    
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 3.41 seconds ====================
+
+You'll see that a couple of tests are reported as crashing, indicated
+by lower-case ``f`` and the respective failure summary. You can also use
+the xdist-provided parallelization feature to speed up your testing::
+
+    $ py.test --boxed -n3
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    gw0 I / gw1 I / gw2 I
+    gw0 [50] / gw1 [50] / gw2 [50]
+    
+    scheduling tests via LoadScheduling
+    ..f...............f..................f............
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    [gw0] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 2.03 seconds ====================

setup.py

@@ -2,22 +2,22 @@ from setuptools import setup
 
 setup(
     name="pytest-xdist",
-    version='1.7.dev1',
+    version='1.9',
     description='py.test xdist plugin for distributed testing and loop-on-failing modes',
     long_description=open('README.txt').read(),
-    license='GPLv2 or later',
+    license='MIT',
     author='holger krekel and contributors',
-    author_email='py-dev@codespeak.net,holger@merlinux.eu',
+    author_email='pytest-dev@python.org,holger@merlinux.eu',
     url='http://bitbucket.org/hpk42/pytest-xdist',
     platforms=['linux', 'osx', 'win32'],
     packages = ['xdist'],
     entry_points = {'pytest11': ['xdist = xdist.plugin'],},
     zip_safe=False,
-    install_requires = ['execnet>=1.0.8', 'pytest>=2.2.0'],
+    install_requires = ['execnet>=1.1', 'pytest>=2.3.5'],
     classifiers=[
     'Development Status :: 5 - Production/Stable',
     'Intended Audience :: Developers',
-    'License :: OSI Approved :: GNU General Public License (GPL)',
+    'License :: OSI Approved :: MIT License',
     'Operating System :: POSIX',
     'Operating System :: Microsoft :: Windows',
     'Operating System :: MacOS :: MacOS X',

testing/acceptance_test.py

@@ -24,6 +24,30 @@ class TestDistribution:
             "*1 failed*",
         ])
 
+    def test_n1_import_error(self, testdir):
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result = testdir.runpytest(p1, "-n1")
+        assert result.ret == 1
+        result.stdout.fnmatch_lines([
+            "E   ImportError: *__import_of_missing_module*",
+        ])
+
+    def test_n2_import_error(self, testdir):
+        """Check that we don't report the same import error multiple times
+        in distributed mode."""
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result1 = testdir.runpytest(p1, "-n2")
+        result2 = testdir.runpytest(p1, "-n1")
+        assert len(result1.stdout.lines) == len(result2.stdout.lines)
+
     def test_n1_skip(self, testdir):
         p1 = testdir.makepyfile("""
             def test_skip():
@@ -36,6 +60,18 @@ class TestDistribution:
             "*1 skipped*",
         ])
 
+    def test_manytests_to_one_import_error(self, testdir):
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result = testdir.runpytest(p1, '--tx=popen', '--tx=popen')
+        assert result.ret == 1
+        result.stdout.fnmatch_lines([
+            "E   ImportError: *__import_of_missing_module*",
+        ])
+
     def test_manytests_to_one_popen(self, testdir):
         p1 = testdir.makepyfile("""
                 import py
@@ -314,7 +350,7 @@ def test_teardownfails_one_function(testdir):
 @py.test.mark.xfail
 def test_terminate_on_hangingnode(testdir):
     p = testdir.makeconftest("""
-        def pytest__teardown_final(session):
+        def pytest_sessionfinishes(session):
             if session.nodeid == "my": # running on slave
                 import time
                 time.sleep(3)

testing/test_dsession.py

@@ -1,4 +1,9 @@
-from xdist.dsession import DSession, LoadScheduling, EachScheduling
+from xdist.dsession import (
+    DSession,
+    LoadScheduling,
+    EachScheduling,
+    report_collection_diff,
+)
 from _pytest import main as outcome
 import py
 import execnet
@@ -12,9 +17,16 @@ def run(item, node, excinfo=None):
     rep.node = node
     return rep
 
+class MockGateway:
+    _count = 0
+    def __init__(self):
+        self.id = str(self._count)
+        self._count += 1
+
 class MockNode:
     def __init__(self):
         self.sent = []
+        self.gateway = MockGateway()
 
     def send_runtest(self, nodeid):
         self.sent.append(nodeid)
@@ -48,11 +60,11 @@ class TestEachScheduling:
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         assert node1.sent == ['ALL']
         assert node2.sent == ['ALL']
         sched.remove_item(node1, collection[0])
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         sched.remove_item(node2, collection[0])
         assert sched.tests_finished()
 
@@ -66,7 +78,7 @@ class TestEachScheduling:
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         crashitem = sched.remove_node(node1)
         assert crashitem
         assert sched.tests_finished()
@@ -88,7 +100,7 @@ class TestLoadScheduling:
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
         x = sorted(node1.sent + node2.sent)
@@ -109,6 +121,7 @@ class TestLoadScheduling:
         sched.addnode_collection(node1, col)
         sched.addnode_collection(node2, col)
         sched.init_distribute()
+        #assert not sched.tests_finished()
         sent1 = node1.sent
         sent2 = node2.sent
         chunkitems = col[:sched.ITEM_CHUNKSIZE]
@@ -164,3 +177,35 @@ class TestDistReporter:
         linecomp.assert_contains_lines([
             "[X1,X2] rsyncing: hello",
         ])
+
+
+def test_report_collection_diff_equal():
+    """Test reporting of equal collections."""
+    from_collection = to_collection = ['aaa', 'bbb', 'ccc']
+    assert report_collection_diff(from_collection, to_collection, 1, 2)
+
+
+def test_report_collection_diff_different():
+    """Test reporting of different collections."""
+    from_collection = ['aaa', 'bbb', 'ccc', 'YYY']
+    to_collection = ['aZa', 'bbb', 'XXX', 'ccc']
+    error_message = (
+        'Different tests were collected between 1 and 2. The difference is:\n'
+        '--- 1\n'
+        '\n'
+        '+++ 2\n'
+        '\n'
+        '@@ -1,4 +1,4 @@\n'
+        '\n'
+        '-aaa\n'
+        '+aZa\n'
+        ' bbb\n'
+        '+XXX\n'
+        ' ccc\n'
+        '-YYY'
+    )
+
+    try:
+        report_collection_diff(from_collection, to_collection, 1, 2)
+    except AssertionError as e:
+        assert py.builtin._totext(e) == error_message

testing/test_looponfail.py

@@ -64,11 +64,11 @@ class TestStatRecorder:
         pycfile = hello + "c"
         pycfile.ensure()
         changed = sd.check()
-        assert not changed
+        assert changed
 
         hello.write("world")
         changed = sd.check()
-        assert not pycfile.check()
+        assert changed
 
     def test_waitonchange(self, tmpdir, monkeypatch):
         tmp = tmpdir
@@ -88,16 +88,14 @@ class TestRemoteControl:
         assert not failures
 
     def test_failures_somewhere(self, testdir):
-        item = testdir.getitem("def test_func(): assert 0\n")
+        item = testdir.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
         failures = control.runsession()
         assert failures
         control.setup()
-        item.fspath.write("def test_func(): assert 1\n")
-        pyc = item.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        item.fspath.write("def test_func():\n assert 1\n")
+        removepyc(item.fspath)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
@@ -115,9 +113,7 @@ class TestRemoteControl:
             def test_new():
                 assert 0
         """))
-        pyc = modcol.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        removepyc(modcol.fspath)
         control.loop_once()
         assert not control.failures
         control.loop_once()
@@ -210,6 +206,24 @@ class TestLooponFailing:
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
+    def test_looponfail_multiple_errors(self, testdir, monkeypatch):
+        modcol = testdir.getmodulecol("""
+            def test_one():
+                assert 0
+        """)
+        remotecontrol = RemoteControl(modcol.config)
+        orig_runsession = remotecontrol.runsession
+
+        def runsession_dups():
+            # twisted.trial test cases may report multiple errors.
+            failures, reports, collection_failed = orig_runsession()
+            print (failures)
+            return failures * 2, reports, collection_failed
+
+        monkeypatch.setattr(remotecontrol, 'runsession', runsession_dups)
+        remotecontrol.loop_once()
+        assert len(remotecontrol.failures) == 1
+
 
 class TestFunctional:
     def test_fail_to_ok(self, testdir):
@@ -252,4 +266,7 @@ def removepyc(path):
     pyc = path + "c"
     if pyc.check():
         pyc.remove()
+    c = path.dirpath("__pycache__")
+    if c.check():
+        c.remove()
 

testing/test_remote.py

@@ -88,7 +88,7 @@ class TestReportSerialization:
             assert newrep.passed == rep.passed
             assert newrep.failed == rep.failed
             assert newrep.skipped == rep.skipped
-            if newrep.skipped and 'xfail' not in newrep.keywords:
+            if newrep.skipped and not hasattr(newrep, "wasxfail"):
                 assert len(newrep.longrepr) == 3
             assert newrep.outcome == rep.outcome
             assert newrep.when == rep.when
@@ -121,6 +121,23 @@ class TestReportSerialization:
             if rep.failed:
                 assert newrep.longrepr == str(rep.longrepr)
 
+    def test_extended_report_deserialization(self, testdir):
+        reprec = testdir.inline_runsource("qwe abc")
+        reports = reprec.getreports("pytest_collectreport")
+        assert reports
+        for rep in reports:
+            rep.extra = True
+            d = serialize_report(rep)
+            check_marshallable(d)
+            newrep = unserialize_report("collectreport", d)
+            assert newrep.extra
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+            if rep.failed:
+                assert newrep.longrepr == str(rep.longrepr)
+
+
 class TestSlaveInteractor:
     def test_basic_collect_and_runtests(self, slave):
         p = slave.testdir.makepyfile("""
@@ -138,6 +155,7 @@ class TestSlaveInteractor:
         ids = ev.kwargs['ids']
         assert len(ids) == 1
         slave.sendcommand("runtests", ids=ids)
+        slave.sendcommand("shutdown")
         ev = slave.popevent("testreport") # setup
         ev = slave.popevent("testreport")
         assert ev.name == "testreport"
@@ -145,7 +163,6 @@ class TestSlaveInteractor:
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        slave.sendcommand("shutdown")
         ev = slave.popevent("slavefinished")
         assert 'slaveoutput' in ev.kwargs
 

testing/test_slavemanage.py

@@ -15,12 +15,14 @@ def pytest_funcarg__config(request):
     config = testdir.parseconfig()
     return config
 
-class pytest_funcarg__mysetup:
-    def __init__(self, request):
-        temp = request.getfuncargvalue("tmpdir")
-        self.source = temp.mkdir("source")
-        self.dest = temp.mkdir("dest")
-        request.getfuncargvalue("_pytest")
+def pytest_funcarg__mysetup(request):
+    class mysetup:
+        def __init__(self, request):
+            temp = request.getfuncargvalue("tmpdir")
+            self.source = temp.mkdir("source")
+            self.dest = temp.mkdir("dest")
+            request.getfuncargvalue("_pytest")
+    return mysetup(request)
 
 class TestNodeManagerPopen:
     def test_popen_no_default_chdir(self, config):
@@ -97,11 +99,13 @@ class TestNodeManagerPopen:
         call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
 
 class TestHRSync:
-    class pytest_funcarg__mysetup:
-        def __init__(self, request):
-            tmp = request.getfuncargvalue('tmpdir')
-            self.source = tmp.mkdir("source")
-            self.dest = tmp.mkdir("dest")
+    def pytest_funcarg__mysetup(self, request):
+        class mysetup:
+            def __init__(self, request):
+                tmp = request.getfuncargvalue('tmpdir')
+                self.source = tmp.mkdir("source")
+                self.dest = tmp.mkdir("dest")
+        return mysetup(request)
 
     def test_hrsync_filter(self, mysetup):
         source, dest = mysetup.source, mysetup.dest

tox.ini

@@ -1,23 +1,26 @@
 [tox]
-envlist=py26,py31,py27,py25,py24
-indexserver=
-    testrun = http://pypi.testrun.org
-    pypi = http://pypi.python.org/simple
+envlist=py26,py32,py33,py27,py26,py26-old,py33-old
 
 [testenv]
 changedir=testing
-deps=:testrun:pytest>=2.2.0.dev2
+deps=pytest>=2.4.2
 commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
 
-[testenv:py31]
-deps=:pypi:pytest  # XXX needed because ClueRelease/pip broken
-[testenv:py32]
-deps=:pypi:pytest  # XXX needed because ClueRelease/pip broken
-
-[testenv:py26]
+[testenv:py27]
 deps=
-    :testrun:pytest
-    :pypi:pexpect
+    pytest>=2.4.2
+    pexpect
+
+[testenv:py26-old]
+deps=
+    pytest==2.3.5
+    pexpect
+
+[testenv:py33-old]
+basepython = python3.3
+deps=
+    pytest==2.3.5
 
 [pytest]
 addopts = -rsfxX
+;; hello

xdist/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '1.7.dev1'
+__version__ = '1.9'

xdist/dsession.py

@@ -1,8 +1,14 @@
-import pytest, py
 import sys
+import difflib
+
+import pytest
+import py
 from xdist.slavemanage import NodeManager
+
+
 queue = py.builtin._tryimport('queue', 'Queue')
 
+
 class EachScheduling:
 
     def __init__(self, numnodes, log=None):
@@ -24,9 +30,6 @@ class EachScheduling:
     def tests_finished(self):
         if not self.collection_is_completed:
             return False
-        for items in self.node2pending.values():
-            if items:
-                return False
         return True
 
     def addnode_collection(self, node, collection):
@@ -79,9 +82,9 @@ class LoadScheduling:
     def tests_finished(self):
         if not self.collection_is_completed or self.pending:
             return False
-        for items in self.node2pending.values():
-            if items:
-                return False
+        #for items in self.node2pending.values():
+        #    if items:
+        #        return False
         return True
 
     def addnode_collection(self, node, collection):
@@ -107,7 +110,7 @@ class LoadScheduling:
             pending.append(item)
             self.item2nodes.setdefault(item, []).append(node)
             node.send_runtest(item)
-        #self.log("items waiting for node: %d" %(len(self.pending)))
+        self.log("items waiting for node: %d" %(len(self.pending)))
         #self.log("item2pending still executing: %s" %(self.item2nodes,))
         #self.log("node2pending: %s" %(self.node2pending,))
 
@@ -130,15 +133,21 @@ class LoadScheduling:
         assert not hasattr(self, 'item2nodes')
         self.item2nodes = {}
         # XXX allow nodes to have different collections
-        col = list(self.node2collection.values())[0]
+        first_node, col = list(self.node2collection.items())[0]
         for node, collection in self.node2collection.items():
-            assert collection == col
+            report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+
         self.pending = col
         if not col:
             return
         available = list(self.node2pending.items())
         num_available = self.numnodes
-        max_one_round = num_available * self.ITEM_CHUNKSIZE -1
+        max_one_round = num_available * self.ITEM_CHUNKSIZE - 1
         for i, item in enumerate(self.pending):
             nodeindex = i % num_available
             node, pending = available[nodeindex]
@@ -147,7 +156,35 @@ class LoadScheduling:
             pending.append(item)
             if i >= max_one_round:
                 break
-        del self.pending[:i+1]
+        del self.pending[:i + 1]
+
+
+def report_collection_diff(from_collection, to_collection, from_id, to_id):
+    """Report the collected test difference between two nodes.
+
+    :returns: True if collections are equal.
+
+    :raises: AssertionError with a detailed error message describing the
+             difference between the collections.
+
+    """
+    if from_collection == to_collection:
+        return True
+
+    diff = difflib.unified_diff(
+        from_collection,
+        to_collection,
+        fromfile=from_id,
+        tofile=to_id,
+    )
+    error_message = py.builtin._totext(
+        'Different tests were collected between {from_id} and {to_id}. '
+        'The difference is:\n'
+        '{diff}'
+    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
+    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    raise AssertionError(msg)
+
 
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
@@ -162,6 +199,7 @@ class DSession:
         self.countfailures = 0
         self.maxfail = config.getvalue("maxfail")
         self.queue = queue.Queue()
+        self._failed_collection_errors = {}
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -287,14 +325,17 @@ class DSession:
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
-    def slave_teardownreport(self, node, rep):
-        rep.node = node
-        self.config.hook.pytest__teardown_final_logerror(report=rep)
-
     def slave_collectreport(self, node, rep):
-        #self.report_line("collectreport %s: %s" %(rep.id, rep.status))
-        #rep.node = node
-        self._handlefailures(rep)
+        if rep.failed:
+            self._failed_slave_collectreport(node, rep)
+
+    def _failed_slave_collectreport(self, node, rep):
+        # Check we haven't already seen this report (from
+        # another slave).
+        if rep.longrepr not in self._failed_collection_errors:
+            self._failed_collection_errors[rep.longrepr] = True
+            self.config.hook.pytest_collectreport(report=rep)
+            self._handlefailures(rep)
 
     def _handlefailures(self, rep):
         if rep.failed:

xdist/looponfail.py

@@ -89,7 +89,11 @@ class RemoteControl(object):
         if collection_failed:
             reports = ["Collection failed, keeping previous failure set"]
         else:
-            self.failures = failures
+            uniq_failures = []
+            for failure in failures:
+                if failure not in uniq_failures:
+                    uniq_failures.append(failure)
+            self.failures = uniq_failures
 
 def repr_pytest_looponfailinfo(failreports, rootdirs):
     tr = py.io.TerminalWriter()
@@ -184,7 +188,7 @@ class StatRecorder:
         self.check() # snapshot state
 
     def fil(self, p):
-        return p.ext in ('.py', '.txt', '.c', '.h')
+        return True  # we are sensitive to all file changes since 1.9
     def rec(self, p):
         return p.check(dotfile=0)
 

xdist/remote.py

@@ -26,10 +26,6 @@ class SlaveInteractor:
         for line in str(excrepr).split("\n"):
             self.log("IERROR> " + line)
 
-    def pytest__teardown_final_logerror(self, report):
-        rep = serialize_report(report)
-        self.sendevent("teardownreport", data=rep)
-
     def pytest_sessionstart(self, session):
         self.session = session
         slaveinfo = getinfodict()
@@ -46,18 +42,26 @@ class SlaveInteractor:
 
     def pytest_runtestloop(self, session):
         self.log("entering main loop")
+        torun = []
         while 1:
             name, kwargs = self.channel.receive()
             self.log("received command %s(**%s)" % (name, kwargs))
             if name == "runtests":
                 ids = kwargs['ids']
                 for nodeid in ids:
-                    item = self._id2item[nodeid]
-                    self.config.hook.pytest_runtest_protocol(item=item)
+                    torun.append(self._id2item[nodeid])
             elif name == "runtests_all":
-                for item in session.items:
-                    self.config.hook.pytest_runtest_protocol(item=item)
-            elif name == "shutdown":
+                torun.extend(session.items)
+            self.log("items to run: %s" %(len(torun)))
+            while len(torun) >= 2:
+                item = torun.pop(0)
+                nextitem = torun[0]
+                self.config.hook.pytest_runtest_protocol(item=item,
+                    nextitem=nextitem)
+            if name == "shutdown":
+                while torun:
+                    self.config.hook.pytest_runtest_protocol(
+                        item=torun.pop(0), nextitem=None)
                 break
         return True
 
@@ -118,9 +122,13 @@ def remote_initconfig(option_dict, args):
     config.option.numprocesses = None
     config.args = args
     return config
-    
+
 
 if __name__ == '__channelexec__':
+    # python3.2 is not concurrent import safe, so let's play it safe
+    # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32
+    if sys.version_info[:2] == (3,2):
+        os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
     slaveinput,args,option_dict = channel.receive()
     importpath = os.getcwd()
     sys.path.insert(0, importpath) # XXX only for remote situations

xdist/slavemanage.py

@@ -183,7 +183,7 @@ def make_reltoroot(roots, args):
             raise ValueError("arg %s not relative to an rsync root" % (arg,))
         l.append(splitcode.join(parts))
     return l
-    
+
 class SlaveController(object):
     ENDMARK = -1
 
@@ -237,8 +237,11 @@ class SlaveController(object):
         self.sendcommand("runtests_all",)
 
     def shutdown(self):
-        if not self._down and not self.channel.isclosed():
-            self.sendcommand("shutdown")
+        if not self._down:
+            try:
+                self.sendcommand("shutdown")
+            except IOError:
+                pass
 
     def sendcommand(self, name, **kwargs):
         """ send a named parametrized command to the other side. """
@@ -298,5 +301,3 @@ def unserialize_report(name, reportdict):
         return runner.TestReport(**d)
     elif name == "collectreport":
         return runner.CollectReport(**d)
-    elif name == "teardownreport":
-        return runner.TeardownErrorReport(**d)