final

of the last test in a node's set.

.hgignore

@@ -14,8 +14,14 @@ syntax:glob
 *.class
 *.orig
 
+*.sublime-*
+.Python
+
 build/
 dist/
+include/
+lib/
+bin/
 pytest_xdist.egg-info
 issue/
 3rdparty/

.hgtags

@@ -5,3 +5,12 @@ e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
 e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
 eaf8b1cb7c312883598677231be5bbeea3b5c127 1.3
 a423748bf17ee778a37853225210257699cad9c1 1.4
+cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
+4815040bdad8f182a5487f57a9da385483836e75 1.6
+20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
+20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
+29c38e195526f5f0fdd651fb51f59d6efaaafbb0 1.7
+0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
+0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
+1d27987c267577899350a25ba5828d55d87083ad 1.8
+5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9

CHANGELOG

@@ -1,3 +1,60 @@
+1.10
+-------------------------
+
+- add glob support for rsyncignores, add command line option to pass
+  additional rsyncignores. Thanks Anatoly Bubenkov.
+
+- fix pytest issue382 - produce "pytest_runtest_logstart" event again
+  in master. Thanks Aron Curzon.
+
+- fix pytest issue419 by sending/receiving indices into the test 
+  collection instead of node ids (which are not neccessarily unique
+  for functions parametrized with duplicate values)
+
+- send multiple "to test" indices in one network message to a slave
+  and improve heuristics for sending chunks where the chunksize
+  depends on the number of remaining tests rather than fixed numbers.
+  This reduces the number of master -> node messages (but not the
+  reverse direction)
+
+
+1.9
+-------------------------
+
+- changed LICENSE to MIT
+
+- fix duplicate reported test ids with --looponfailing
+  (thanks Jeremy Thurgood)
+
+- fix pytest issue41: re-run tests on all file changes, not just
+  randomly select ones like .py/.c.
+
+- fix pytest issue347: slaves running on top of Python3.2
+  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
+  bugs.
+
+1.8
+-------------------------
+
+- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
+  interface to perform eager teardowns for test items.
+
+1.7
+-------------------------
+
+- fix incompatibilities with pytest-2.2.0 (allow multiple
+  pytest_runtest_logreport reports for a test item)
+
+1.6
+-------------------------
+
+- terser collection reporting
+
+- fix issue34 - distributed testing with -p plugin now works correctly
+
+- fix race condition in looponfail mode where a concurrent file removal
+  could cause a crash
+
 1.5
 -------------------------
 

LICENSE

@@ -1,13 +1,19 @@
-The execnet package is released under the provisions of the Gnu Public
-License (GPL), version 2 or later.  
 
-See http://www.fsf.org/licensing/licenses/ for more information.
+  Permission is hereby granted, free of charge, to any person obtaining a copy
+  of this software and associated documentation files (the "Software"), to deal
+  in the Software without restriction, including without limitation the rights
+  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  copies of the Software, and to permit persons to whom the Software is
+  furnished to do so, subject to the following conditions:
+     
+  The above copyright notice and this permission notice shall be included in all
+  copies or substantial portions of the Software.
+ 
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  SOFTWARE.
 
-This package also contains some minor parts which 
-which are useable under the MIT license. 
-
-If you have questions and/or want to use parts of 
-the code under a different license than the GPL 
-please contact me. 
-
-holger krekel, January 2010, holger at merlinux eu 

MANIFEST.in

@@ -2,9 +2,6 @@ include CHANGELOG
 include LICENSE 
 include README.txt
 include setup.py
+include tox.ini
 graft testing
-exclude *.orig 
-exclude *.rej 
-recursive-exclude testing *.pyc *$py.class
-prune .svn 
 prune .hg 

README.txt

@@ -4,16 +4,19 @@ xdist: pytest distributed testing plugin
 The `pytest-xdist`_ plugin extends py.test with some unique
 test execution modes:
 
-* Looponfail: run your tests repeatedly in a subprocess.  After each run py.test
-  waits until a file in your project changes and then re-runs the previously
-  failing tests.  This is repeated until all tests pass after which again
-  a full run is performed.
-
-* multiprocess Load-balancing: if you have multiple CPUs or hosts you can use
+* test run parallelization_: if you have multiple CPUs or hosts you can use
   those for a combined test run.  This allows to speed up
-  development or to use special resources of remote machines.
+  development or to use special resources of `remote machines`_.
 
-* Multi-Platform coverage: you can specify different Python interpreters
+* ``--boxed``: (not available on Windows) run each test in a boxed_
+  subprocess to survive ``SEGFAULTS`` or otherwise dying processes
+
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
+  py.test waits until a file in your project changes and then re-runs
+  the previously failing tests.  This is repeated until all tests pass
+  after which again a full run is performed.
+
+* `Multi-Platform`_ coverage: you can specify different Python interpreters
   or different platforms and run tests in parallel on all of them.
 
 Before running tests remotely, ``py.test`` efficiently "rsyncs" your
@@ -30,7 +33,7 @@ Install the plugin with::
     easy_install pytest-xdist
 
     # or
-    
+
     pip install pytest-xdist
 
 or use the package in develope/in-place mode with
@@ -41,6 +44,8 @@ a checkout of the `pytest-xdist repository`_ ::
 Usage examples
 ---------------------
 
+.. _parallelization:
+
 Speed up test runs by sending tests to multiple CPUs
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -55,20 +60,42 @@ a lot of IO this can lead to considerable speed ups.
 Running tests in a Python subprocess
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-To instantiate a python2.4 sub process and send tests to it, you may type::
+To instantiate a python2.5 sub process and send tests to it, you may type::
 
-    py.test -d --tx popen//python=python2.4
+    py.test -d --tx popen//python=python2.5
 
-This will start a subprocess which is run with the "python2.4"
+This will start a subprocess which is run with the "python2.5"
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python2.4
+    --tx 3*popen//python=python2.5
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
 
+.. _boxed:
+
+Running tests in a boxed subprocess
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+If you have tests involving C or C++ libraries you might have to deal
+with tests crashing the process.  For this case you may use the boxing
+options::
+
+    py.test --boxed
+
+which will run each test in a subprocess and will report if a test
+crashed the process.  You can also combine this option with
+running multiple processes to speed up the test run and use your CPU cores::
+
+    py.test -n3 --boxed
+
+this would run 3 testing subprocesses in parallel which each
+create new boxed subprocesses for each test.
+
+
+.. _`remote machines`:
 
 Sending tests to remote SSH accounts
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -95,6 +122,13 @@ py.test references tests as a fully qualified python
 module path.  **You will otherwise get strange errors**
 during setup of the remote side.
 
+You can specify multiple ``--rsyncignore`` glob-patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: .*, *.pyc, *.pyo, *~
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
 Sending tests to remote Socket Servers
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
@@ -111,6 +145,8 @@ new socket host with something like this::
 
 
 .. _`atonce`:
+.. _`Multi-Platform`:
+
 
 Running tests on many platforms at once
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -168,3 +204,9 @@ where the configuration file was found.
 .. _`pytest-xdist repository`: http://bitbucket.org/hpk42/pytest-xdist
 .. _`pytest`: http://pytest.org
 
+Issue and Bug Tracker
+------------------------
+
+Please use the pytest issue tracker for bugs in this plugin, see https://bitbucket.org/hpk42/pytest/issues .
+
+

example/boxed.txt

@@ -0,0 +1,62 @@
+
+
+If your testing involves C or C++ libraries you might have to deal
+with crashing processes.  The xdist-plugin provides the ``--boxed`` option
+to run each test in a controled subprocess.  Here is a basic example::
+
+    # content of test_module.py
+
+    import pytest
+    import os
+    import time
+
+    # run test function 50 times with different argument
+    @pytest.mark.parametrize("arg", range(50))
+    def test_func(arg):
+        time.sleep(0.05) # each tests takes a while
+        if arg % 19 == 0: 
+            os.kill(os.getpid(), 15)
+
+If you run this with::
+
+    $ py.test --boxed
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    collecting ... collected 50 items
+    
+    test_module.py f..................f..................f...........
+    
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 3.41 seconds ====================
+
+You'll see that a couple of tests are reported as crashing, indicated
+by lower-case ``f`` and the respective failure summary. You can also use
+the xdist-provided parallelization feature to speed up your testing::
+
+    $ py.test --boxed -n3
+    =========================== test session starts ============================
+    platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
+    plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
+    gw0 I / gw1 I / gw2 I
+    gw0 [50] / gw1 [50] / gw2 [50]
+    
+    scheduling tests via LoadScheduling
+    ..f...............f..................f............
+    ================================= FAILURES =================================
+    _______________________________ test_func[0] _______________________________
+    [gw0] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[19] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    ______________________________ test_func[38] _______________________________
+    [gw2] linux2 -- Python 2.7.3 /home/hpk/venv/1/bin/python
+    /home/hpk/tmp/doc-exec-420/test_module.py:6: running the test CRASHED with signal 15
+    =================== 3 failed, 47 passed in 2.03 seconds ====================

setup.py

@@ -2,22 +2,22 @@ from setuptools import setup
 
 setup(
     name="pytest-xdist",
-    version='1.5',
+    version='1.10',
     description='py.test xdist plugin for distributed testing and loop-on-failing modes',
     long_description=open('README.txt').read(),
-    license='GPLv2 or later',
+    license='MIT',
     author='holger krekel and contributors',
-    author_email='py-dev@codespeak.net,holger@merlinux.eu',
+    author_email='pytest-dev@python.org,holger@merlinux.eu',
     url='http://bitbucket.org/hpk42/pytest-xdist',
     platforms=['linux', 'osx', 'win32'],
     packages = ['xdist'],
     entry_points = {'pytest11': ['xdist = xdist.plugin'],},
     zip_safe=False,
-    install_requires = ['execnet>=1.0.8', 'pytest>1.9.9'],
+    install_requires = ['execnet>=1.1', 'pytest>=2.4.2'],
     classifiers=[
     'Development Status :: 5 - Production/Stable',
     'Intended Audience :: Developers',
-    'License :: OSI Approved :: GNU General Public License (GPL)',
+    'License :: OSI Approved :: MIT License',
     'Operating System :: POSIX',
     'Operating System :: Microsoft :: Windows',
     'Operating System :: MacOS :: MacOS X',
@@ -27,4 +27,4 @@ setup(
     'Programming Language :: Python',
     'Programming Language :: Python :: 3',
     ],
-)
+)

testing/acceptance_test.py

@@ -24,6 +24,30 @@ class TestDistribution:
             "*1 failed*",
         ])
 
+    def test_n1_import_error(self, testdir):
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result = testdir.runpytest(p1, "-n1")
+        assert result.ret == 1
+        result.stdout.fnmatch_lines([
+            "E   ImportError: *__import_of_missing_module*",
+        ])
+
+    def test_n2_import_error(self, testdir):
+        """Check that we don't report the same import error multiple times
+        in distributed mode."""
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result1 = testdir.runpytest(p1, "-n2")
+        result2 = testdir.runpytest(p1, "-n1")
+        assert len(result1.stdout.lines) == len(result2.stdout.lines)
+
     def test_n1_skip(self, testdir):
         p1 = testdir.makepyfile("""
             def test_skip():
@@ -36,6 +60,18 @@ class TestDistribution:
             "*1 skipped*",
         ])
 
+    def test_manytests_to_one_import_error(self, testdir):
+        p1 = testdir.makepyfile("""
+            import __import_of_missing_module
+            def test_import():
+                pass
+        """)
+        result = testdir.runpytest(p1, '--tx=popen', '--tx=popen')
+        assert result.ret == 1
+        result.stdout.fnmatch_lines([
+            "E   ImportError: *__import_of_missing_module*",
+        ])
+
     def test_manytests_to_one_popen(self, testdir):
         p1 = testdir.makepyfile("""
                 import py
@@ -220,6 +256,7 @@ class TestDistribution:
                 time.sleep(10)
         """)
         child = testdir.spawn_pytest("-n1")
+        py.std.time.sleep(0.1)
         child.expect(".*test session starts.*")
         child.kill(2) # keyboard interrupt
         child.expect(".*KeyboardInterrupt.*")
@@ -314,7 +351,7 @@ def test_teardownfails_one_function(testdir):
 @py.test.mark.xfail
 def test_terminate_on_hangingnode(testdir):
     p = testdir.makeconftest("""
-        def pytest__teardown_final(session):
+        def pytest_sessionfinishes(session):
             if session.nodeid == "my": # running on slave
                 import time
                 time.sleep(3)
@@ -407,5 +444,18 @@ def test_skipping(testdir):
         "*1 skipped*"
     ])
 
-
-
+def test_issue34_pluginloading_in_subprocess(testdir):
+    testdir.tmpdir.join("plugin123.py").write(py.code.Source("""
+        def pytest_namespace():
+            return {'sample_variable': 'testing'}
+    """))
+    testdir.makepyfile("""
+        import pytest
+        def test_hello():
+            assert pytest.sample_variable == "testing"
+    """)
+    result = testdir.runpytest("-n1", "-p", "plugin123")
+    assert result.ret == 0
+    result.stdout.fnmatch_lines([
+        "*1 passed*",
+    ])

testing/test_boxed.py

@@ -16,13 +16,13 @@ def test_functional_boxed(testdir):
 class TestOptionEffects:
     def test_boxed_option_default(self, testdir):
         tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
-        config = testdir.reparseconfig()
+        config = testdir.parseconfig()
         assert not config.option.boxed
         py.test.importorskip("execnet")
-        config = testdir.reparseconfig(['-d', tmpdir])
+        config = testdir.parseconfig('-d', tmpdir)
         assert not config.option.boxed
 
     def test_is_not_boxed_by_default(self, testdir):
-        config = testdir.reparseconfig([testdir.tmpdir])
+        config = testdir.parseconfig(testdir.tmpdir)
         assert not config.option.boxed
 

testing/test_dsession.py

@@ -1,6 +1,12 @@
-from xdist.dsession import DSession, LoadScheduling, EachScheduling
-from _pytest import session as outcome
+from xdist.dsession import (
+    DSession,
+    LoadScheduling,
+    EachScheduling,
+    report_collection_diff,
+)
+from _pytest import main as outcome
 import py
+import pytest
 import execnet
 
 XSpec = execnet.XSpec
@@ -12,19 +18,23 @@ def run(item, node, excinfo=None):
     rep.node = node
     return rep
 
+class MockGateway:
+    _count = 0
+    def __init__(self):
+        self.id = str(self._count)
+        self._count += 1
+
 class MockNode:
     def __init__(self):
         self.sent = []
+        self.gateway = MockGateway()
 
-    def send_runtest(self, nodeid):
-        self.sent.append(nodeid)
+    def send_runtest_some(self, indices):
+        self.sent.extend(indices)
 
     def send_runtest_all(self):
         self.sent.append("ALL")
 
-    def sendlist(self, items):
-        self.sent.extend(items)
-
     def shutdown(self):
         self._shutdown=True
 
@@ -48,12 +58,12 @@ class TestEachScheduling:
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         assert node1.sent == ['ALL']
         assert node2.sent == ['ALL']
-        sched.remove_item(node1, collection[0])
-        assert not sched.tests_finished()
-        sched.remove_item(node2, collection[0])
+        sched.remove_item(node1, 0)
+        assert sched.tests_finished()
+        sched.remove_item(node2, 0)
         assert sched.tests_finished()
 
     def test_schedule_remove_node(self):
@@ -66,7 +76,7 @@ class TestEachScheduling:
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         crashitem = sched.remove_node(node1)
         assert crashitem
         assert sched.tests_finished()
@@ -88,11 +98,11 @@ class TestLoadScheduling:
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
         sched.init_distribute()
-        assert not sched.tests_finished()
+        assert sched.tests_finished()
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
         x = sorted(node1.sent + node2.sent)
-        assert x == collection
+        assert x == [0, 1]
         sched.remove_item(node1, node1.sent[0])
         sched.remove_item(node2, node2.sent[0])
         assert sched.tests_finished()
@@ -104,22 +114,22 @@ class TestLoadScheduling:
         node2 = MockNode()
         sched.addnode(node1)
         sched.addnode(node2)
-        sched.ITEM_CHUNKSIZE = 2
-        col = ["xyz"] * (2*sched.ITEM_CHUNKSIZE +1)
+        col = ["xyz"] * (3)
         sched.addnode_collection(node1, col)
         sched.addnode_collection(node2, col)
         sched.init_distribute()
+        #assert not sched.tests_finished()
         sent1 = node1.sent
         sent2 = node2.sent
-        chunkitems = col[:sched.ITEM_CHUNKSIZE]
-        assert sent1 == chunkitems
-        assert sent2 == chunkitems
+        chunkitems = col[:1]
+        assert (sent1 == [0] and sent2 == [1]) or (
+                sent1 == [1] and sent2 == [0])
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
         assert len(sched.pending) == 1
         for node in (node1, node2):
-            for i in range(sched.ITEM_CHUNKSIZE):
-                sched.remove_item(node, "xyz")
+            for i in sched.node2pending[node]:
+                sched.remove_item(node, i)
         assert not sched.pending
 
     def test_add_remove_node(self):
@@ -164,3 +174,48 @@ class TestDistReporter:
         linecomp.assert_contains_lines([
             "[X1,X2] rsyncing: hello",
         ])
+
+
+def test_report_collection_diff_equal():
+    """Test reporting of equal collections."""
+    from_collection = to_collection = ['aaa', 'bbb', 'ccc']
+    assert report_collection_diff(from_collection, to_collection, 1, 2)
+
+
+def test_report_collection_diff_different():
+    """Test reporting of different collections."""
+    from_collection = ['aaa', 'bbb', 'ccc', 'YYY']
+    to_collection = ['aZa', 'bbb', 'XXX', 'ccc']
+    error_message = (
+        'Different tests were collected between 1 and 2. The difference is:\n'
+        '--- 1\n'
+        '\n'
+        '+++ 2\n'
+        '\n'
+        '@@ -1,4 +1,4 @@\n'
+        '\n'
+        '-aaa\n'
+        '+aZa\n'
+        ' bbb\n'
+        '+XXX\n'
+        ' ccc\n'
+        '-YYY'
+    )
+
+    try:
+        report_collection_diff(from_collection, to_collection, 1, 2)
+    except AssertionError as e:
+        assert py.builtin._totext(e) == error_message
+
+@pytest.mark.xfail(reason="duplicate test ids not supported yet")
+def test_pytest_issue419(testdir):
+    testdir.makepyfile("""
+        import pytest
+
+        @pytest.mark.parametrize('birth_year', [1988, 1988, ])
+        def test_2011_table(birth_year):
+            pass
+    """)
+    reprec = testdir.inline_run("-n1")
+    reprec.assertoutcome(passed=2)
+    assert 0

testing/test_looponfail.py

@@ -14,11 +14,15 @@ class TestStatRecorder:
         changed = sd.check()
         assert changed
 
-        tmp.ensure("new.py")
+        (hello + "c").write("hello")
+        changed = sd.check()
+        assert not changed
+
+        p = tmp.ensure("new.py")
         changed = sd.check()
         assert changed
 
-        tmp.join("new.py").remove()
+        p.remove()
         changed = sd.check()
         assert changed
 
@@ -36,6 +40,30 @@ class TestStatRecorder:
         changed = sd.check()
         assert changed
 
+    def test_dirchange(self, tmpdir):
+        tmp = tmpdir
+        hello = tmp.ensure("dir", "hello.py")
+        sd = StatRecorder([tmp])
+        assert not sd.fil(tmp.join("dir"))
+
+    def test_filechange_deletion_race(self, tmpdir, monkeypatch):
+        tmp = tmpdir
+        sd = StatRecorder([tmp])
+        changed = sd.check()
+        assert not changed
+
+        p = tmp.ensure("new.py")
+        changed = sd.check()
+        assert changed
+
+        p.remove()
+        # make check()'s visit() call return our just removed
+        # path as if we were in a race condition
+        monkeypatch.setattr(tmp, 'visit', lambda *args: [p])
+
+        changed = sd.check()
+        assert changed
+
     def test_pycremoval(self, tmpdir):
         tmp = tmpdir
         hello = tmp.ensure("hello.py")
@@ -45,11 +73,9 @@ class TestStatRecorder:
 
         pycfile = hello + "c"
         pycfile.ensure()
-        changed = sd.check()
-        assert not changed
-
         hello.write("world")
         changed = sd.check()
+        assert changed
         assert not pycfile.check()
 
     def test_waitonchange(self, tmpdir, monkeypatch):
@@ -70,16 +96,14 @@ class TestRemoteControl:
         assert not failures
 
     def test_failures_somewhere(self, testdir):
-        item = testdir.getitem("def test_func(): assert 0\n")
+        item = testdir.getitem("def test_func():\n assert 0\n")
         control = RemoteControl(item.config)
         control.setup()
         failures = control.runsession()
         assert failures
         control.setup()
-        item.fspath.write("def test_func(): assert 1\n")
-        pyc = item.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        item.fspath.write("def test_func():\n assert 1\n")
+        removepyc(item.fspath)
         topdir, failures = control.runsession()[:2]
         assert not failures
 
@@ -97,9 +121,7 @@ class TestRemoteControl:
             def test_new():
                 assert 0
         """))
-        pyc = modcol.fspath.new(ext=".pyc")
-        if pyc.check():
-            pyc.remove()
+        removepyc(modcol.fspath)
         control.loop_once()
         assert not control.failures
         control.loop_once()
@@ -192,6 +214,24 @@ class TestLooponFailing:
         remotecontrol.loop_once()
         assert len(remotecontrol.failures) == 1
 
+    def test_looponfail_multiple_errors(self, testdir, monkeypatch):
+        modcol = testdir.getmodulecol("""
+            def test_one():
+                assert 0
+        """)
+        remotecontrol = RemoteControl(modcol.config)
+        orig_runsession = remotecontrol.runsession
+
+        def runsession_dups():
+            # twisted.trial test cases may report multiple errors.
+            failures, reports, collection_failed = orig_runsession()
+            print (failures)
+            return failures * 2, reports, collection_failed
+
+        monkeypatch.setattr(remotecontrol, 'runsession', runsession_dups)
+        remotecontrol.loop_once()
+        assert len(remotecontrol.failures) == 1
+
 
 class TestFunctional:
     def test_fail_to_ok(self, testdir):
@@ -234,4 +274,7 @@ def removepyc(path):
     pyc = path + "c"
     if pyc.check():
         pyc.remove()
+    c = path.dirpath("__pycache__")
+    if c.check():
+        c.remove()
 

testing/test_plugin.py

@@ -46,6 +46,11 @@ class TestDistOptions:
         assert nm.roots
         assert testdir.tmpdir in nm.roots
 
+    def test_getrsyncignore(self, testdir):
+        config = testdir.parseconfigure('--rsyncignore=fo*')
+        nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
+        assert 'fo*' in nm.rsyncoptions['ignores']
+
     def test_getrsyncdirs_with_conftest(self, testdir):
         p = py.path.local()
         for bn in 'x y z'.split():

testing/test_remote.py

@@ -62,7 +62,7 @@ def test_remoteinitconfig(testdir):
     config1 = testdir.parseconfig()
     config2 = remote_initconfig(config1.option.__dict__, config1.args)
     assert config2.option.__dict__ == config1.option.__dict__
-    py.test.raises(KeyError, 'config2.pluginmanager.getplugin("terminal")')
+    assert config2.pluginmanager.getplugin("terminal") in (-1, None)
 
 class TestReportSerialization:
     def test_itemreport_outcomes(self, testdir):
@@ -80,7 +80,7 @@ class TestReportSerialization:
                 py.test.xfail("hello")
         """)
         reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 6
+        assert len(reports) == 17 # with setup/teardown "passed" reports
         for rep in reports:
             d = serialize_report(rep)
             check_marshallable(d)
@@ -88,7 +88,7 @@ class TestReportSerialization:
             assert newrep.passed == rep.passed
             assert newrep.failed == rep.failed
             assert newrep.skipped == rep.skipped
-            if newrep.skipped and 'xfail' not in newrep.keywords:
+            if newrep.skipped and not hasattr(newrep, "wasxfail"):
                 assert len(newrep.longrepr) == 3
             assert newrep.outcome == rep.outcome
             assert newrep.when == rep.when
@@ -121,6 +121,23 @@ class TestReportSerialization:
             if rep.failed:
                 assert newrep.longrepr == str(rep.longrepr)
 
+    def test_extended_report_deserialization(self, testdir):
+        reprec = testdir.inline_runsource("qwe abc")
+        reports = reprec.getreports("pytest_collectreport")
+        assert reports
+        for rep in reports:
+            rep.extra = True
+            d = serialize_report(rep)
+            check_marshallable(d)
+            newrep = unserialize_report("collectreport", d)
+            assert newrep.extra
+            assert newrep.passed == rep.passed
+            assert newrep.failed == rep.failed
+            assert newrep.skipped == rep.skipped
+            if rep.failed:
+                assert newrep.longrepr == str(rep.longrepr)
+
+
 class TestSlaveInteractor:
     def test_basic_collect_and_runtests(self, slave):
         p = slave.testdir.makepyfile("""
@@ -137,14 +154,18 @@ class TestSlaveInteractor:
         assert ev.kwargs['topdir'] == slave.testdir.tmpdir
         ids = ev.kwargs['ids']
         assert len(ids) == 1
-        slave.sendcommand("runtests", ids=ids)
+        slave.sendcommand("runtests", indices=list(range(len(ids))))
+        slave.sendcommand("shutdown")
+        ev = slave.popevent("logstart")
+        assert ev.kwargs["nodeid"].endswith("test_func")
+        assert len(ev.kwargs["location"]) == 3
+        ev = slave.popevent("testreport") # setup
         ev = slave.popevent("testreport")
         assert ev.name == "testreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        slave.sendcommand("shutdown")
         ev = slave.popevent("slavefinished")
         assert 'slaveoutput' in ev.kwargs
 
@@ -194,16 +215,13 @@ class TestSlaveInteractor:
         ids = ev.kwargs['ids']
         assert len(ids) == 2
         slave.sendcommand("runtests_all", )
-        ev = slave.popevent("testreport")
-        assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
-        assert rep.nodeid.endswith("::test_func")
-        ev = slave.popevent("testreport")
-        assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs['data'])
-        assert rep.nodeid.endswith("::test_func2")
-        assert rep.passed
-        slave.sendcommand("shutdown")
+        slave.sendcommand("shutdown", )
+        for func in "::test_func", "::test_func2":
+            for i in range(3):  # setup/call/teardown
+                ev = slave.popevent("testreport")
+                assert ev.name == "testreport"
+                rep = unserialize_report(ev.name, ev.kwargs['data'])
+                assert rep.nodeid.endswith(func)
         ev = slave.popevent("slavefinished")
         assert 'slaveoutput' in ev.kwargs
 

testing/test_slavemanage.py

@@ -15,12 +15,14 @@ def pytest_funcarg__config(request):
     config = testdir.parseconfig()
     return config
 
-class pytest_funcarg__mysetup:
-    def __init__(self, request):
-        temp = request.getfuncargvalue("tmpdir")
-        self.source = temp.mkdir("source")
-        self.dest = temp.mkdir("dest")
-        request.getfuncargvalue("_pytest")
+def pytest_funcarg__mysetup(request):
+    class mysetup:
+        def __init__(self, request):
+            temp = request.getfuncargvalue("tmpdir")
+            self.source = temp.mkdir("source")
+            self.dest = temp.mkdir("dest")
+            request.getfuncargvalue("_pytest")
+    return mysetup(request)
 
 class TestNodeManagerPopen:
     def test_popen_no_default_chdir(self, config):
@@ -97,11 +99,13 @@ class TestNodeManagerPopen:
         call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
 
 class TestHRSync:
-    class pytest_funcarg__mysetup:
-        def __init__(self, request):
-            tmp = request.getfuncargvalue('tmpdir')
-            self.source = tmp.mkdir("source")
-            self.dest = tmp.mkdir("dest")
+    def pytest_funcarg__mysetup(self, request):
+        class mysetup:
+            def __init__(self, request):
+                tmp = request.getfuncargvalue('tmpdir')
+                self.source = tmp.mkdir("source")
+                self.dest = tmp.mkdir("dest")
+        return mysetup(request)
 
     def test_hrsync_filter(self, mysetup):
         source, dest = mysetup.source, mysetup.dest
@@ -109,7 +113,7 @@ class TestHRSync:
         source.ensure(".svn", "entries")
         source.ensure(".somedotfile", "moreentries")
         source.ensure("somedir", "editfile~")
-        syncer = HostRSync(source)
+        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
         l = list(source.visit(rec=syncer.filter,
                                    fil=syncer.filter))
         assert len(l) == 3
@@ -135,7 +139,7 @@ class TestNodeManager:
     @py.test.mark.xfail
     def test_rsync_roots_no_roots(self, testdir, mysetup):
         mysetup.source.ensure("dir1", "file1").write("hello")
-        config = testdir.reparseconfig([source])
+        config = testdir.parseconfig(source)
         nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
         #assert nodemanager.config.topdir == source == config.topdir
         nodemanager.makegateways()
@@ -179,7 +183,7 @@ class TestNodeManager:
             [pytest]
             rsyncdirs=dir1/dir2
         """))
-        config = testdir.reparseconfig([source])
+        config = testdir.parseconfig(source)
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         nodemanager.makegateways()
         nodemanager.rsync_roots()
@@ -193,12 +197,15 @@ class TestNodeManager:
         dir5 = source.ensure("dir5", "dir6", "bogus")
         dirf = source.ensure("dir5", "file")
         dir2.ensure("hello")
+        dirfoo = source.ensure("foo", "bar")
+        dirbar = source.ensure("bar", "foo")
         source.join("tox.ini").write(py.std.textwrap.dedent("""
             [pytest]
             rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6
+            rsyncignore = dir1/dir2 dir5/dir6 foo*
         """))
-        config = testdir.reparseconfig([source])
+        config = testdir.parseconfig(source)
+        config.option.rsyncignore = ['bar']
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         nodemanager.makegateways()
         nodemanager.rsync_roots()
@@ -206,13 +213,15 @@ class TestNodeManager:
         assert not dest.join("dir1", "dir2").check()
         assert dest.join("dir5","file").check()
         assert not dest.join("dir6").check()
+        assert not dest.join('foo').check()
+        assert not dest.join('bar').check()
 
     def test_optimise_popen(self, testdir, mysetup):
         source, dest = mysetup.source, mysetup.dest
         specs = ["popen"] * 3
         source.join("conftest.py").write("rsyncdirs = ['a']")
         source.ensure('a', dir=1)
-        config = testdir.reparseconfig([source])
+        config = testdir.parseconfig(source)
         nodemanager = NodeManager(config, specs)
         nodemanager.makegateways()
         nodemanager.rsync_roots()

tox.ini

@@ -1,19 +1,27 @@
 [tox]
-envlist=py26,py31,py27,py25,py24
-indexserver=
-    default = http://pypi.testrun.org
-    pypi = http://pypi.python.org/simple
+envlist=py26,py32,py33,py27,py27-pexpect,py33-pexpect,py26,py26-old,py33-old
 
 [testenv]
-
 changedir=testing
-deps=pytest
+deps=pytest>=2.5.1
 commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
 
-[testenv:py26]
+[testenv:py27-pexpect]
+deps={[testenv]deps}
+    pexpect
+[testenv:py33-pexpect]
+deps={[testenv]deps}
+    pexpect
+
+[testenv:py26-old]
 deps=
-    pytest
-    :pypi:pexpect
+    pytest==2.4.2
+
+[testenv:py33-old]
+basepython = python3.3
+deps=
+    pytest==2.4.2
 
 [pytest]
 addopts = -rsfxX
+;; hello

xdist/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '1.5'
+__version__ = '1.10'

xdist/dsession.py

@@ -1,8 +1,14 @@
-import pytest, py
 import sys
+import difflib
+
+import pytest
+import py
 from xdist.slavemanage import NodeManager
+
+
 queue = py.builtin._tryimport('queue', 'Queue')
 
+
 class EachScheduling:
 
     def __init__(self, numnodes, log=None):
@@ -24,9 +30,6 @@ class EachScheduling:
     def tests_finished(self):
         if not self.collection_is_completed:
             return False
-        for items in self.node2pending.values():
-            if items:
-                return False
         return True
 
     def addnode_collection(self, node, collection):
@@ -37,15 +40,15 @@ class EachScheduling:
         if len(self.node2pending) >= self.numnodes:
             self.collection_is_completed = True
 
-    def remove_item(self, node, item):
-        self.node2pending[node].remove(item)
+    def remove_item(self, node, item_index, duration=0):
+        self.node2pending[node].remove(item_index)
 
     def remove_node(self, node):
         # KeyError if we didn't get an addnode() yet
         pending = self.node2pending.pop(node)
         if not pending:
             return
-        crashitem = pending.pop(0)
+        crashitem = self.node2collection[node][pending.pop(0)]
         # XXX what about the rest of pending?
         return crashitem
 
@@ -53,12 +56,9 @@ class EachScheduling:
         assert self.collection_is_completed
         for node, pending in self.node2pending.items():
             node.send_runtest_all()
-            pending[:] = self.node2collection[node]
+            pending[:] = range(len(self.node2collection[node]))
 
 class LoadScheduling:
-    LOAD_THRESHOLD_NEWITEMS = 5
-    ITEM_CHUNKSIZE = 10
-
     def __init__(self, numnodes, log=None):
         self.numnodes = numnodes
         self.node2pending = {}
@@ -79,9 +79,9 @@ class LoadScheduling:
     def tests_finished(self):
         if not self.collection_is_completed or self.pending:
             return False
-        for items in self.node2pending.values():
-            if items:
-                return False
+        #for items in self.node2pending.values():
+        #    if items:
+        #        return False
         return True
 
     def addnode_collection(self, node, collection):
@@ -91,63 +91,102 @@ class LoadScheduling:
         if len(self.node2collection) >= self.numnodes:
             self.collection_is_completed = True
 
-    def remove_item(self, node, item):
-        if item not in self.item2nodes:
-            raise AssertionError(item, self.item2nodes)
-        nodes = self.item2nodes[item]
-        if node in nodes: # the node might have gone down already
-            nodes.remove(node)
-        #if not nodes:
-        #    del self.item2nodes[item]
-        pending = self.node2pending[node]
-        pending.remove(item)
+    def remove_item(self, node, item_index, duration=0):
+        node_pending = self.node2pending[node]
+        node_pending.remove(item_index)
         # pre-load items-to-test if the node may become ready
-        if self.pending and len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
-            item = self.pending.pop(0)
-            pending.append(item)
-            self.item2nodes.setdefault(item, []).append(node)
-            node.send_runtest(item)
-        #self.log("items waiting for node: %d" %(len(self.pending)))
-        #self.log("item2pending still executing: %s" %(self.item2nodes,))
-        #self.log("node2pending: %s" %(self.node2pending,))
+
+        if self.pending:
+            if duration >= 0.1 and node_pending:
+                # seems the node is doing long-running tests
+                # so let's rather wait with sending new items
+                return
+            # how many nodes do we have remaining per node roughly?
+            num_nodes = len(self.node2pending)
+            # if our node goes below a heuristic minimum, fill it out to
+            # heuristic maximum
+            items_per_node_min = max(
+                    1, len(self.pending) // num_nodes // 4)
+            items_per_node_max = max(
+                    1, len(self.pending) // num_nodes // 2)
+            if len(node_pending) <= items_per_node_min:
+                num_send = items_per_node_max - len(node_pending) + 1
+                self._send_tests(node, num_send)
+
+        self.log("num items waiting for node:", len(self.pending))
+        #self.log("node2pending:", self.node2pending)
 
     def remove_node(self, node):
         pending = self.node2pending.pop(node)
-        # KeyError if we didn't get an addnode() yet
-        for item in pending:
-            l = self.item2nodes[item]
-            l.remove(node)
-            if not l:
-                del self.item2nodes[item]
         if not pending:
             return
-        crashitem = pending.pop(0)
+        # the node must have crashed on the item if there are pending ones
+        crashitem = self.collection[pending.pop(0)]
         self.pending.extend(pending)
         return crashitem
 
     def init_distribute(self):
         assert self.collection_is_completed
-        assert not hasattr(self, 'item2nodes')
-        self.item2nodes = {}
         # XXX allow nodes to have different collections
-        col = list(self.node2collection.values())[0]
-        for node, collection in self.node2collection.items():
-            assert collection == col
-        self.pending = col
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        for node, collection in node_collection_items[1:]:
+            report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+
+        # all collections are the same, good.
+        # we now create an index
+        self.collection = col
+        self.pending[:] = range(len(col))
         if not col:
             return
-        available = list(self.node2pending.items())
-        num_available = self.numnodes
-        max_one_round = num_available * self.ITEM_CHUNKSIZE -1
-        for i, item in enumerate(self.pending):
-            nodeindex = i % num_available
-            node, pending = available[nodeindex]
-            node.send_runtest(item)
-            self.item2nodes.setdefault(item, []).append(node)
-            pending.append(item)
-            if i >= max_one_round:
-                break
-        del self.pending[:i+1]
+        # how many items per node do we have about?
+        items_per_node = len(self.collection) // len(self.node2pending)
+        # take a fraction of tests for initial distribution
+        node_chunksize = max(items_per_node // 4, 1)
+        # and initialize each node with a chunk of tests
+        for node in self.node2pending:
+            self._send_tests(node, node_chunksize)
+
+    #f = open("/tmp/sent", "w")
+    def _send_tests(self, node, num):
+        tests_per_node = self.pending[:num]
+        #print >>self.f, "sent", node, tests_per_node
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+def report_collection_diff(from_collection, to_collection, from_id, to_id):
+    """Report the collected test difference between two nodes.
+
+    :returns: True if collections are equal.
+
+    :raises: AssertionError with a detailed error message describing the
+             difference between the collections.
+
+    """
+    if from_collection == to_collection:
+        return True
+
+    diff = difflib.unified_diff(
+        from_collection,
+        to_collection,
+        fromfile=from_id,
+        tofile=to_id,
+    )
+    error_message = py.builtin._totext(
+        'Different tests were collected between {from_id} and {to_id}. '
+        'The difference is:\n'
+        '{diff}'
+    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
+    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    raise AssertionError(msg)
+
 
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
@@ -162,6 +201,7 @@ class DSession:
         self.countfailures = 0
         self.maxfail = config.getvalue("maxfail")
         self.queue = queue.Queue()
+        self._failed_collection_errors = {}
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -218,7 +258,7 @@ class DSession:
         assert callname, kwargs
         method = "slave_" + callname
         call = getattr(self, method)
-        self.log("calling method: %s(**%s)" % (method, kwargs))
+        self.log("calling method", method, kwargs)
         call(**kwargs)
         if self.sched.tests_finished():
             self.triggershutdown()
@@ -267,10 +307,11 @@ class DSession:
 
         if self.sched.collection_is_completed:
             if self.terminal:
+                self.trdist.ensure_show_status()
                 self.terminal.write_line("")
                 self.terminal.write_line("scheduling tests via %s" %(
                     self.sched.__class__.__name__))
-                
+
             self.sched.init_distribute()
 
     def slave_logstart(self, node, nodeid, location):
@@ -278,21 +319,25 @@ class DSession:
             nodeid=nodeid, location=location)
 
     def slave_testreport(self, node, rep):
-        if rep.when in ("setup", "call"):
-            self.sched.remove_item(node, rep.nodeid)
+        if not (rep.passed and rep.when != "call"):
+            if rep.when in ("setup", "call"):
+                self.sched.remove_item(node, rep.item_index, rep.duration)
         #self.report_line("testreport %s: %s" %(rep.id, rep.status))
         rep.node = node
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
-    def slave_teardownreport(self, node, rep):
-        rep.node = node
-        self.config.hook.pytest__teardown_final_logerror(report=rep)
-
     def slave_collectreport(self, node, rep):
-        #self.report_line("collectreport %s: %s" %(rep.id, rep.status))
-        #rep.node = node
-        self._handlefailures(rep)
+        if rep.failed:
+            self._failed_slave_collectreport(node, rep)
+
+    def _failed_slave_collectreport(self, node, rep):
+        # Check we haven't already seen this report (from
+        # another slave).
+        if rep.longrepr not in self._failed_collection_errors:
+            self._failed_collection_errors[rep.longrepr] = True
+            self.config.hook.pytest_collectreport(report=rep)
+            self._handlefailures(rep)
 
     def _handlefailures(self, rep):
         if rep.failed:
@@ -327,13 +372,19 @@ class TerminalDistReporter:
     def write_line(self, msg):
         self.tr.write_line(msg)
 
+    def ensure_show_status(self):
+        if not self.tr.hasmarkup:
+            self.write_line(self.getstatus())
+
     def setstatus(self, spec, status, show=True):
         self._status[spec.id] = status
-        if show:
-            parts = ["%s %s" %(spec.id, self._status[spec.id])
-                       for spec in self._specs]
-            line = " / ".join(parts)
-            self.rewrite(line)
+        if show and self.tr.hasmarkup:
+            self.rewrite(self.getstatus())
+
+    def getstatus(self):
+        parts = ["%s %s" %(spec.id, self._status[spec.id])
+                   for spec in self._specs]
+        return " / ".join(parts)
 
     def rewrite(self, line, newline=False):
         pline = line + " " * max(self._lastlen-len(line), 0)
@@ -347,8 +398,9 @@ class TerminalDistReporter:
     def pytest_xdist_setupnodes(self, specs):
         self._specs = specs
         for spec in specs:
-            self.setstatus(spec, "initializing", show=False)
-        self.setstatus(spec, "initializing", show=True)
+            self.setstatus(spec, "I", show=False)
+        self.setstatus(spec, "I", show=True)
+        self.ensure_show_status()
 
     def pytest_xdist_newgateway(self, gateway):
         if self.config.option.verbose > 0:
@@ -357,7 +409,7 @@ class TerminalDistReporter:
             self.rewrite("[%s] %s Python %s cwd: %s" % (
                 gateway.id, rinfo.platform, version, rinfo.cwd),
                 newline=True)
-        self.setstatus(gateway.spec, "collecting")
+        self.setstatus(gateway.spec, "C")
 
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
@@ -366,7 +418,7 @@ class TerminalDistReporter:
                 d['id'],
                 d['version'].replace('\n', ' -- '),)
             self.rewrite(infoline, newline=True)
-        self.setstatus(node.gateway.spec, "ready")
+        self.setstatus(node.gateway.spec, "ok")
 
     def pytest_testnodedown(self, node, error):
         if not error:

xdist/looponfail.py

@@ -89,7 +89,11 @@ class RemoteControl(object):
         if collection_failed:
             reports = ["Collection failed, keeping previous failure set"]
         else:
-            self.failures = failures
+            uniq_failures = []
+            for failure in failures:
+                if failure not in uniq_failures:
+                    uniq_failures.append(failure)
+            self.failures = uniq_failures
 
 def repr_pytest_looponfailinfo(failreports, rootdirs):
     tr = py.io.TerminalWriter()
@@ -120,9 +124,7 @@ def init_slave_session(channel, args, option_dict):
 
     #fullwidth, hasmarkup = channel.receive()
     from _pytest.config import Config
-    config = Config()
-    config.option.__dict__.update(option_dict)
-    config._preparse(list(args))
+    config = Config.fromdictargs(option_dict, list(args))
     config.args = args
     from xdist.looponfail import SlaveFailSession
     SlaveFailSession(config, channel).main()
@@ -186,7 +188,7 @@ class StatRecorder:
         self.check() # snapshot state
 
     def fil(self, p):
-        return p.ext in ('.py', '.txt', '.c', '.h')
+        return p.check(file=1, dotfile=0) and p.ext != ".pyc"
     def rec(self, p):
         return p.check(dotfile=0)
 
@@ -203,14 +205,11 @@ class StatRecorder:
         newstat = {}
         for rootdir in self.rootdirlist:
             for path in rootdir.visit(self.fil, self.rec):
-                oldstat = statcache.get(path, None)
-                if oldstat is not None:
-                    del statcache[path]
+                oldstat = statcache.pop(path, None)
                 try:
                     newstat[path] = curstat = path.stat()
                 except py.error.ENOENT:
                     if oldstat:
-                        del statcache[path]
                         changed = True
                 else:
                     if oldstat:

xdist/plugin.py

@@ -1,5 +1,5 @@
-import sys
-import py, pytest
+import py
+import pytest
 
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
@@ -28,12 +28,14 @@ def pytest_addoption(parser):
     group._addoption('-d',
            action="store_true", dest="distload", default=False,
            help="load-balance tests.  shortcut for '--dist=load'")
-    group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
+    group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
            help="add directory for rsyncing to remote tx nodes.")
+    group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
+           help="add expression for ignores when rsyncing to remote tx nodes.")
 
     parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
          ' remote distributed testing.', type="pathlist")
-    parser.addini('rsyncignore', 'list of (relative) paths to be ignored '
+    parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
          'for rsyncing.', type="pathlist")
     parser.addini("looponfailroots", type="pathlist",
         help="directories to check for changes", default=[py.path.local()])
@@ -62,6 +64,8 @@ def pytest_configure(config, __multicall__):
         from xdist.dsession import DSession
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
+        tr = config.pluginmanager.getplugin("terminalreporter")
+        tr.showfspath = False
 
 def check_options(config):
     if config.option.numprocesses:

xdist/remote.py

@@ -24,11 +24,7 @@ class SlaveInteractor:
 
     def pytest_internalerror(self, excrepr):
         for line in str(excrepr).split("\n"):
-            self.log("IERROR> " + line)
-
-    def pytest__teardown_final_logerror(self, report):
-        rep = serialize_report(report)
-        self.sendevent("teardownreport", data=rep)
+            self.log("IERROR>", line)
 
     def pytest_sessionstart(self, session):
         self.session = session
@@ -46,36 +42,44 @@ class SlaveInteractor:
 
     def pytest_runtestloop(self, session):
         self.log("entering main loop")
+        torun = []
         while 1:
             name, kwargs = self.channel.receive()
-            self.log("received command %s(**%s)" % (name, kwargs))
+            self.log("received command", name, kwargs)
             if name == "runtests":
-                ids = kwargs['ids']
-                for nodeid in ids:
-                    item = self._id2item[nodeid]
-                    self.config.hook.pytest_runtest_protocol(item=item)
+                torun.extend(kwargs['indices'])
             elif name == "runtests_all":
-                for item in session.items:
-                    self.config.hook.pytest_runtest_protocol(item=item)
-            elif name == "shutdown":
+                torun.extend(range(len(session.items)))
+            self.log("items to run:", torun)
+            while torun:
+                self.run_tests(torun)
+            if name == "shutdown":
                 break
         return True
 
+    def run_tests(self, torun):
+        items = self.session.items
+        self.item_index = torun.pop(0)
+        if torun:
+            nextitem = items[torun[0]]
+        else:
+            nextitem = None
+        self.config.hook.pytest_runtest_protocol(
+            item=items[self.item_index],
+            nextitem=nextitem)
+
     def pytest_collection_finish(self, session):
-        self._id2item = {}
-        ids = []
-        for item in session.items:
-            self._id2item[item.nodeid] = item
-            ids.append(item.nodeid)
         self.sendevent("collectionfinish",
             topdir=str(session.fspath),
-            ids=ids)
+            ids=[item.nodeid for item in session.items])
 
-    #def pytest_runtest_logstart(self, nodeid, location, fspath):
-    #    self.sendevent("logstart", nodeid=nodeid, location=location)
+    def pytest_runtest_logstart(self, nodeid, location):
+        self.sendevent("logstart", nodeid=nodeid, location=location)
 
     def pytest_runtest_logreport(self, report):
         data = serialize_report(report)
+        data["item_index"] = self.item_index
+        assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
@@ -109,10 +113,8 @@ def getinfodict():
 
 def remote_initconfig(option_dict, args):
     from _pytest.config import Config
-    config = Config()
-    config.pluginmanager.unregister(name="terminal")
-    config._preparse(args, addopts=False)
-    config.option.__dict__.update(option_dict)
+    option_dict['plugins'].append("no:terminal")
+    config = Config.fromdictargs(option_dict, args)
     config.option.looponfail = False
     config.option.usepdb = False
     config.option.dist = "no"
@@ -120,9 +122,13 @@ def remote_initconfig(option_dict, args):
     config.option.numprocesses = None
     config.args = args
     return config
-    
+
 
 if __name__ == '__channelexec__':
+    # python3.2 is not concurrent import safe, so let's play it safe
+    # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32
+    if sys.version_info[:2] == (3,2):
+        os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
     slaveinput,args,option_dict = channel.receive()
     importpath = os.getcwd()
     sys.path.insert(0, importpath) # XXX only for remote situations

xdist/slavemanage.py

@@ -1,5 +1,8 @@
-import py, pytest
-import sys, os
+import fnmatch
+import os
+
+import py
+import pytest
 import execnet
 import xdist.remote
 
@@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically
 
 class NodeManager(object):
     EXIT_TIMEOUT = 10
+    DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
         self._nodesready = py.std.threading.Event()
@@ -23,20 +27,17 @@ class NodeManager(object):
             self.group.allocate_id(spec)
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
+        self.rsyncoptions = self._getrsyncoptions()
 
     def rsync_roots(self):
         """ make sure that all remote gateways
             have the same set of roots in their
             current directory.
         """
-        options = {
-            'ignores': self.config.getini("rsyncignore"),
-            'verbose': self.config.option.verbose,
-        }
         if self.roots:
             # send each rsync root
             for root in self.roots:
-                self.rsync(root, **options)
+                self.rsync(root, **self.rsyncoptions)
 
     def makegateways(self):
         assert not list(self.group)
@@ -98,6 +99,18 @@ class NodeManager(object):
                 roots.append(root)
         return roots
 
+    def _getrsyncoptions(self):
+        """Get options to be passed for rsync."""
+        ignores = list(self.DEFAULT_IGNORES)
+        ignores += self.config.option.rsyncignore
+        ignores += self.config.getini("rsyncignore")
+
+        return {
+            'ignores': ignores,
+            'verbose': self.config.option.verbose,
+        }
+
+
     def rsync(self, source, notify=None, verbose=False, ignores=None):
         """ perform rsync to all remote hosts.
         """
@@ -144,14 +157,12 @@ class HostRSync(execnet.RSync):
 
     def filter(self, path):
         path = py.path.local(path)
-        if not path.ext in ('.pyc', '.pyo'):
-            if not path.basename.endswith('~'):
-                if path.check(dotfile=0):
-                    for x in self._ignores:
-                        if path == x:
-                            break
-                    else:
-                        return True
+        for x in self._ignores:
+            x = getattr(x, 'strpath', x)
+            if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
+                return False
+        else:
+            return True
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)
@@ -183,7 +194,7 @@ def make_reltoroot(roots, args):
             raise ValueError("arg %s not relative to an rsync root" % (arg,))
         l.append(splitcode.join(parts))
     return l
-    
+
 class SlaveController(object):
     ENDMARK = -1
 
@@ -230,15 +241,18 @@ class SlaveController(object):
             self.gateway.exit()
             #del self.gateway
 
-    def send_runtest(self, nodeid):
-        self.sendcommand("runtests", ids=[nodeid])
+    def send_runtest_some(self, indices):
+        self.sendcommand("runtests", indices=indices)
 
     def send_runtest_all(self):
         self.sendcommand("runtests_all",)
 
     def shutdown(self):
-        if not self._down and not self.channel.isclosed():
-            self.sendcommand("shutdown")
+        if not self._down:
+            try:
+                self.sendcommand("shutdown")
+            except IOError:
+                pass
 
     def sendcommand(self, name, **kwargs):
         """ send a named parametrized command to the other side. """
@@ -275,10 +289,13 @@ class SlaveController(object):
                 self._down = True
                 self.slaveoutput = kwargs['slaveoutput']
                 self.notify_inproc("slavefinished", node=self)
-            #elif eventname == "logstart":
-            #    self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "logstart":
+                self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in ("testreport", "collectreport", "teardownreport"):
+                item_index = kwargs.pop("item_index", None)
                 rep = unserialize_report(eventname, kwargs['data'])
+                if item_index is not None:
+                    rep.item_index = item_index
                 self.notify_inproc(eventname, node=self, rep=rep)
             elif eventname == "collectionfinish":
                 self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
@@ -293,10 +310,7 @@ class SlaveController(object):
             self.config.pluginmanager.notify_exception(excinfo)
 
 def unserialize_report(name, reportdict):
-    d = reportdict
     if name == "testreport":
-        return runner.TestReport(**d)
+        return runner.TestReport(**reportdict)
     elif name == "collectreport":
-        return runner.CollectReport(**d)
-    elif name == "teardownreport":
-        return runner.TeardownErrorReport(**d)
+        return runner.CollectReport(**reportdict)