modify CHANGELOG

inconsistent memo states.  Rather than adding locks, the fix actually simplifies the code by
keeping the pickling/unpickling memo's in sync directly and more efficiently, also removing
a long outstanding XXX.

.hgtags

@@ -0,0 +1,2 @@
+42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0
+eca7ce17eabf296983c36812c8b8be901e7055a3 1.1

CHANGELOG

@@ -1,3 +1,25 @@
+1.2
+-------------------------
+
+- fix issue79: sessionfinish/teardown hooks are now called systematically 
+  on the slave side 
+- introduce a new data input/output mechanism to allow the master side 
+  to send and receive data from a slave. 
+- fix race condition in underlying pickling/unpickling handling 
+- use and require new register hooks facility of py.test>=1.3.0 
+- require improved execnet>=1.0.6 because of various race conditions 
+  that can arise in xdist testing modes. 
+- fix some python3 related pickling related race conditions 
+- fix PyPI description 
+
+1.1
+-------------------------
+
+- fix an indefinite hang which would wait for events although no events
+  are pending - this happened if items arrive very quickly while 
+  the "reschedule-event" tried unconditionally avoiding a busy-loop
+  and not schedule new work. 
+
 1.0
 -------------------------
 

setup.py

@@ -7,11 +7,12 @@ for the new options.
 """
 
 from setuptools import setup
+from xdist import __version__
 
 setup(
     name="pytest-xdist",
-    version="1.0",
+    version=__version__,
-    description='py.test figleaf coverage plugin',
+    description='py.test xdist plugin for distributed testing and loop-on-failing modes',
     long_description=__doc__,
     license='GPLv2 or later',
     author='holger krekel and contributors',
@@ -21,7 +22,7 @@ setup(
     packages = ['xdist'],
     entry_points = {'pytest11': ['xdist = xdist.plugin'],},
     zip_safe=False,
-    install_requires = ['execnet>=1.0.3', 'py>=1.2.0'],
+    install_requires = ['execnet>=1.0.6', 'py>=1.3.0'],
     classifiers=[
     'Development Status :: 4 - Beta',
     'Intended Audience :: Developers',
@@ -33,5 +34,6 @@ setup(
     'Topic :: Software Development :: Quality Assurance',
     'Topic :: Utilities',
     'Programming Language :: Python',
+    'Programming Language :: Python :: 3',
     ],
 )

testing/acceptance_test.py

@@ -117,3 +117,58 @@ class TestDistribution:
         s = result.stdout.str()
         assert "2.4" in s
         assert "2.5" in s
+
+    def test_data_exchange(self, testdir):
+        c1 = testdir.makeconftest("""
+            # This hook only called on master.
+            def pytest_configure_node(node):
+                node.slaveinput['a'] = 42
+                node.slaveinput['b'] = 7
+
+            # This hook only takes action on slave.
+            def pytest_configure(config):
+                if hasattr(config, 'slaveinput'):
+                    a = config.slaveinput['a']
+                    b = config.slaveinput['b']
+                    r = a + b
+                    config.slaveoutput['r'] = r
+
+            # This hook only called on master.
+            def pytest_testnodedown(node, error):
+                node.config.calc_result = node.slaveoutput['r']
+
+            # This hook only takes action on master.
+            def pytest_terminal_summary(terminalreporter):
+                if not hasattr(terminalreporter.config, 'slaveinput'):
+                    calc_result = terminalreporter.config.calc_result
+                    terminalreporter._tw.sep('-', 
+                        'calculated result is %s' % calc_result)
+        """)
+        p1 = testdir.makepyfile("def test_func(): pass")
+        result = testdir.runpytest(p1, '-d', '--tx=popen')
+        result.stdout.fnmatch_lines([
+            "*popen*Python*",
+            "*calculated result is 49*",
+            "*1 passed*"
+        ])
+        assert result.ret == 0
+
+    def test_keyboardinterrupt_hooks_issue79(self, testdir):
+        testdir.makepyfile(__init__="", test_one="""
+            def test_hello():
+                raise KeyboardInterrupt()
+        """)
+        testdir.makeconftest("""
+            def pytest_sessionfinish(session):
+                if hasattr(session.config, 'slaveoutput'):
+                    session.config.slaveoutput['s2'] = 42
+            def pytest_testnodedown(node, error):
+                assert node.slaveoutput['s2'] == 42
+                print ("s2call-finished")
+        """)
+        args = ["-n1"]
+        result = testdir.runpytest(*args)
+        s = result.stdout.str()
+        assert result.ret 
+        assert 'SIGINT' in s
+        assert 's2call' in s

testing/test_boxed.py

@@ -8,7 +8,7 @@ def test_functional_boxed(testdir):
             os.kill(os.getpid(), 15)
     """)
     result = testdir.runpytest(p1, "--boxed")
-    assert result.stdout.fnmatch_lines([
+    result.stdout.fnmatch_lines([
         "*CRASHED*",
         "*1 failed*"
     ])

testing/test_dsession.py

@@ -1,5 +1,5 @@
 from xdist.dsession import DSession
-from py._test import outcome
+from py._test import session as outcome
 import py
 import execnet
 
@@ -110,6 +110,7 @@ class TestDSession:
         assert name == "pytest_rescheduleitems"
         assert kwargs['items'] == [item]
 
+    
     def test_keyboardinterrupt(self, testdir):
         item = testdir.getitem("def test_func(): pass")
         session = DSession(item.config)
@@ -134,14 +135,22 @@ class TestDSession:
         loopstate = session._initloopstate([])
         session.queueevent("pytest_rescheduleitems", items=[item])
         session.loop_once(loopstate)
-        # check that RescheduleEvents are not immediately
+        # we need to do work because nothing is pending / we would not wake up
-        # rescheduled if there are no nodes
+        assert loopstate.dowork == True
+
+        session.node2pending[node].append(item)
+        session.queueevent("pytest_rescheduleitems", items=[item])
+        session.loop_once(loopstate)
+        # now we want to not directly trigger work again to avoid busy-wait
         assert loopstate.dowork == False 
+        
         session.queueevent(None)
         session.loop_once(loopstate)
         session.queueevent(None)
         session.loop_once(loopstate)
-        assert node.sent == [item]
+        assert node.sent == [item, item]
+        session.queueevent("pytest_runtest_logreport", report=run(item, node))
+        session.loop_once(loopstate)
         session.queueevent("pytest_runtest_logreport", report=run(item, node))
         session.loop_once(loopstate)
         assert loopstate.shuttingdown 

testing/test_gwmanage.py

@@ -3,6 +3,7 @@ import os
 from xdist.gwmanage import GatewayManager, HostRSync
 from py._test.pluginmanager import HookRelay, Registry
 from py._plugin import hookspec
+from xdist import newhooks
 import execnet
 
 def pytest_funcarg__hookrecorder(request):
@@ -11,7 +12,7 @@ def pytest_funcarg__hookrecorder(request):
     return _pytest.gethookrecorder(hook)
 
 def pytest_funcarg__hook(request):
-    return HookRelay(hookspec, Registry())
+    return HookRelay([hookspec, newhooks], Registry())
 
 class TestGatewayManagerPopen:
     def test_popen_no_default_chdir(self, hook):

testing/test_txnode.py

@@ -12,7 +12,7 @@ class EventQueue:
         self.queue = queue
         registry.register(self)
 
-    def geteventargs(self, eventname, timeout=2.0):
+    def geteventargs(self, eventname, timeout=10.0):
         events = []
         while 1:
             try:
@@ -38,7 +38,7 @@ class MySetup:
         self.id = 0
         self.request = request
 
-    def geteventargs(self, eventname, timeout=2.0):
+    def geteventargs(self, eventname, timeout=10.0):
         eq = EventQueue(self.config.pluginmanager, self.queue)
         return eq.geteventargs(eventname, timeout=timeout)
 
@@ -52,7 +52,8 @@ class MySetup:
         self.gateway = execnet.makegateway(self.xspec)
         self.id += 1
         self.gateway.id = str(self.id)
-        self.node = TXNode(self.gateway, self.config, putevent=self.queue.put)
+        self.nodemanager = None
+        self.node = TXNode(self.nodemanager, self.gateway, self.config, putevent=self.queue.put)
         assert not self.node.channel.isclosed()
         return self.node 
 
@@ -80,7 +81,7 @@ class TestMasterSlaveConnection:
         node.send(123) # invalid item 
         kwargs = mysetup.geteventargs("pytest_testnodedown")
         assert kwargs['node'] is node 
-        assert isinstance(kwargs['error'], execnet.RemoteError)
+        #assert isinstance(kwargs['error'], execnet.RemoteError)
 
     def test_crash_killed(self, testdir, mysetup):
         if not hasattr(py.std.os, 'kill'):

xdist/__init__.py

@@ -1,3 +1,3 @@
 #
-__version__ = "1.0"
+__version__ = "1.2"
 

xdist/dsession.py

@@ -1,6 +1,5 @@
 import py
-from py._test.session import Session
+from py._test import session 
-from py._test import outcome 
 from xdist.nodemanage import NodeManager
 queue = py.builtin._tryimport('queue', 'Queue')
 
@@ -55,12 +54,14 @@ class LoopState(object):
 
     def pytest_rescheduleitems(self, items):
         self.colitems[:] = items + self.colitems
-        self.dowork = False # avoid busywait
+        for pending in self.dsession.node2pending.values():
+            if pending:
+                self.dowork = False # avoid busywait, nodes still have work
 
 class ExitFirstInterrupt(KeyboardInterrupt):
     pass
 
-class DSession(Session):
+class DSession(session.Session):
     """ 
         Session drives the collection and running of tests
         and generates test events for reporters. 
@@ -130,14 +131,15 @@ class DSession(Session):
             call(**kwargs)
 
         # termination conditions
-        if ((loopstate.testsfailed and self.config.option.exitfirst) or 
+        if (not self.node2pending or 
+            (loopstate.testsfailed and self.config.option.exitfirst) or 
             (not self.item2nodes and not colitems and not self.queue.qsize())):
             if self.config.option.exitfirst:
                 raise ExitFirstInterrupt()
             self.triggershutdown()
             loopstate.shuttingdown = True
-        elif not self.node2pending:
+            if not self.node2pending:
-            loopstate.exitstatus = outcome.EXIT_NOHOSTS
+                loopstate.exitstatus = session.EXIT_NOHOSTS
                 
     def loop_once_shutdown(self, loopstate):
         # once we are in shutdown mode we dont send 
@@ -151,16 +153,16 @@ class DSession(Session):
             self.config.hook.pytest_runtest_logreport(**kwargs)
         elif eventname == "pytest_internalerror":
             self.config.hook.pytest_internalerror(**kwargs)
-            loopstate.exitstatus = outcome.EXIT_INTERNALERROR
+            loopstate.exitstatus = session.EXIT_INTERNALERROR
         elif eventname == "pytest__teardown_final_logerror":
             self.config.hook.pytest__teardown_final_logerror(**kwargs)
-            loopstate.exitstatus = outcome.EXIT_TESTSFAILED
+            loopstate.exitstatus = session.EXIT_TESTSFAILED
         if not self.node2pending:
             # finished
             if loopstate.testsfailed:
-                loopstate.exitstatus = outcome.EXIT_TESTSFAILED
+                loopstate.exitstatus = session.EXIT_TESTSFAILED
             else:
-                loopstate.exitstatus = outcome.EXIT_OK
+                loopstate.exitstatus = session.EXIT_OK
         #self.config.pluginmanager.unregister(loopstate)
 
     def _initloopstate(self, colitems):
@@ -180,16 +182,16 @@ class DSession(Session):
         except KeyboardInterrupt:
             excinfo = py.code.ExceptionInfo()
             if excinfo.errisinstance(ExitFirstInterrupt):
-                exitstatus = outcome.EXIT_TESTSFAILED
+                exitstatus = session.EXIT_TESTSFAILED
             else:
                 self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
-                exitstatus = outcome.EXIT_INTERRUPTED
+                exitstatus = session.EXIT_INTERRUPTED
         except:
             self.config.pluginmanager.notify_exception()
-            exitstatus = outcome.EXIT_INTERNALERROR
+            exitstatus = session.EXIT_INTERNALERROR
         self.config.pluginmanager.unregister(loopstate)
         if exitstatus == 0 and self._testsfailed:
-            exitstatus = outcome.EXIT_TESTSFAILED
+            exitstatus = session.EXIT_TESTSFAILED
         return exitstatus
 
     def triggershutdown(self):

xdist/mypickle.py

@@ -31,13 +31,12 @@ class MyPickler(Pickler):
     """ Pickler with a custom memoize()
         to take care of unique ID creation. 
         See the usage in ImmutablePickler
-        XXX we could probably extend Pickler 
-            and Unpickler classes to directly
-            update the other'S memos. 
     """
-    def __init__(self, file, protocol, uneven):
+    def __init__(self, immo, file, protocol, uneven):
         Pickler.__init__(self, file, protocol)
         self.uneven = uneven
+        self._unpicklememo = immo._unpicklememo
+        self.memo = immo._picklememo
         
     def memoize(self, obj):
         if self.fast:
@@ -47,6 +46,10 @@ class MyPickler(Pickler):
         key = memo_len * 2 + self.uneven
         self.write(self.put(key))
         self.memo[id(obj)] = key, obj
+        key = makekey(key)
+        if key in self._unpicklememo:
+            assert self._unpicklememo[key] is obj
+        dict.__setitem__(self._unpicklememo, key, obj)
 
     #if sys.version_info < (3,0):
     #    def save_string(self, obj, pack=struct.pack):
@@ -54,6 +57,15 @@ class MyPickler(Pickler):
     #        self.save_unicode(obj, pack=pack)
     #    Pickler.dispatch[str] = save_string 
   
+class UnpicklingDict(dict):
+    def __init__(self, picklememo):
+        super(UnpicklingDict, self).__init__()
+        self._picklememo = picklememo
+   
+    def __setitem__(self, key, obj):
+        super(UnpicklingDict, self).__setitem__(key, obj)
+        self._picklememo[id(obj)] = (fromkey(key), obj)
+    
 class ImmutablePickler:
     def __init__(self, uneven, protocol=0):
         """ ImmutablePicklers are instantiated in Pairs. 
@@ -64,7 +76,7 @@ class ImmutablePickler:
             parameter.
         """
         self._picklememo = {}
-        self._unpicklememo = {}
+        self._unpicklememo = UnpicklingDict(self._picklememo)
         self._protocol = protocol
         self.uneven = uneven and 1 or 0
 
@@ -73,18 +85,13 @@ class ImmutablePickler:
         # which be the case e.g. if you want to pickle 
         # from a forked process back to the original 
         f = py.io.BytesIO()
-        pickler = MyPickler(f, self._protocol, uneven=self.uneven)
+        pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
-        pickler.memo = self._picklememo
         pickler.memoize(obj)
-        self._updateunpicklememo()
 
     def dumps(self, obj):
         f = py.io.BytesIO()
-        pickler = MyPickler(f, self._protocol, uneven=self.uneven)
+        pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
-        pickler.memo = self._picklememo
         pickler.dump(obj)
-        if obj is not None:
-            self._updateunpicklememo()
         #print >>debug, "dumped", obj 
         #print >>debug, "picklememo", self._picklememo
         return f.getvalue()
@@ -94,21 +101,10 @@ class ImmutablePickler:
         unpickler = Unpickler(f)
         unpickler.memo = self._unpicklememo
         res = unpickler.load()
-        self._updatepicklememo()
         #print >>debug, "loaded", res
         #print >>debug, "unpicklememo", self._unpicklememo
         return res
 
-    def _updatepicklememo(self):
-        for x, obj in self._unpicklememo.items():
-            self._picklememo[id(obj)] = (fromkey(x), obj)
-
-    def _updateunpicklememo(self):
-        for key,obj in self._picklememo.values():
-            key = makekey(key) 
-            if key in self._unpicklememo:
-                assert self._unpicklememo[key] is obj
-            self._unpicklememo[key] = obj
 
 NO_ENDMARKER_WANTED = object()
 

xdist/newhooks.py

@@ -0,0 +1,25 @@
+
+def pytest_gwmanage_newgateway(gateway, platinfo):
+    """ called on new raw gateway creation. """ 
+
+def pytest_gwmanage_rsyncstart(source, gateways):
+    """ called before rsyncing a directory to remote gateways takes place. """
+
+def pytest_gwmanage_rsyncfinish(source, gateways):
+    """ called after rsyncing a directory to remote gateways takes place. """
+
+def pytest_configure_node(node):
+    """ configure node information before it gets instantiated. """
+
+def pytest_testnodeready(node):
+    """ Test Node is ready to operate. """
+
+def pytest_testnodedown(node, error):
+    """ Test Node is down. """
+
+def pytest_rescheduleitems(items):
+    """ reschedule Items from a node that went down. """
+
+def pytest_looponfailinfo(failreports, rootdirs):
+    """ info for repeating failing tests. """
+

xdist/nodemanage.py

@@ -58,7 +58,7 @@ class NodeManager(object):
         self.rsync_roots()
         self.trace("setting up nodes")
         for gateway in self.gwmanager.group:
-            node = TXNode(gateway, self.config, putevent)
+            node = TXNode(self, gateway, self.config, putevent)
             gateway.node = node  # to keep node alive 
             self.trace("started node %r" % node)
 

xdist/plugin.py

@@ -173,6 +173,16 @@ def pytest_addoption(parser):
     group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", 
            help="add directory for rsyncing to remote tx nodes.")
 
+# -------------------------------------------------------------------------
+# distributed testing hooks
+# -------------------------------------------------------------------------
+def pytest_addhooks(pluginmanager):
+    from xdist import newhooks
+    pluginmanager.addhooks(newhooks)
+
+# -------------------------------------------------------------------------
+# distributed testing initialization
+# -------------------------------------------------------------------------
 def pytest_configure(config):
     if config.option.numprocesses:
         config.option.dist = "load"

xdist/txnode.py

@@ -3,7 +3,7 @@
 """
 import py
 from xdist.mypickle import PickleChannel
-from py._test import outcome
+from py._test.session import Session
 
 class TXNode(object):
     """ Represents a Test Execution environment in the controlling process. 
@@ -13,11 +13,13 @@ class TXNode(object):
     """
     ENDMARK = -1
 
-    def __init__(self, gateway, config, putevent):
+    def __init__(self, nodemanager, gateway, config, putevent):
+        self.nodemanager = nodemanager
         self.config = config 
         self.putevent = putevent 
         self.gateway = gateway
-        self.channel = install_slave(gateway, config)
+        self.slaveinput = {}
+        self.channel = install_slave(self)
         self.channel.setcallback(self.callback, endmarker=self.ENDMARK)
         self._down = False
 
@@ -43,7 +45,7 @@ class TXNode(object):
                 err = self.channel._getremoteerror()
                 if not self._down:
                     if not err or isinstance(err, EOFError):
-                        err = "Not properly terminated"
+                        err = "Not properly terminated" # lost connection? 
                     self.notify("pytest_testnodedown", node=self, error=err)
                     self._down = True
                 return
@@ -52,7 +54,9 @@ class TXNode(object):
                 self.notify("pytest_testnodeready", node=self)
             elif eventname == "slavefinished":
                 self._down = True
-                self.notify("pytest_testnodedown", error=None, node=self)
+                self.slaveoutput = kwargs['slaveoutput']
+                error = kwargs['error']
+                self.notify("pytest_testnodedown", error=error, node=self)
             elif eventname in ("pytest_runtest_logreport", 
                                "pytest__teardown_final_logerror"):
                 kwargs['report'].node = self
@@ -80,32 +84,44 @@ class TXNode(object):
         else:
             self.channel.send(None)
 
-# setting up slave code 
+# configuring and setting up slave node 
-def install_slave(gateway, config):
+def install_slave(node):
-    channel = gateway.remote_exec(source="""
+    channel = node.gateway.remote_exec(source="""
         import os, sys 
         sys.path.insert(0, os.getcwd()) 
         from xdist.mypickle import PickleChannel
-        from xdist.txnode import SlaveNode
+        from xdist.txnode import SlaveSession
         channel.send("basicimport")
         channel = PickleChannel(channel)
-        slavenode = SlaveNode(channel)
+        import py
-        slavenode.run()
+        config, slaveinput, basetemp, nodeid = channel.receive()
+        config.slaveinput = slaveinput
+        config.slaveoutput = {}
+        if basetemp:
+            config.basetemp = py.path.local(basetemp)
+        config.nodeid = nodeid
+        config.pluginmanager.do_configure(config)
+        session = SlaveSession(config, channel, nodeid)
+        session.dist_main()
     """)
     channel.receive()
     channel = PickleChannel(channel)
     basetemp = None
-    if gateway.spec.popen:
+    config = node.config 
+    config.hook.pytest_configure_node(node=node)
+    if node.gateway.spec.popen:
         popenbase = config.ensuretemp("popen")
         basetemp = py.path.local.make_numbered_dir(prefix="slave-", 
             keep=0, rootdir=popenbase)
         basetemp = str(basetemp)
-    channel.send((config, basetemp, gateway.id))
+    channel.send((config, node.slaveinput, basetemp, node.gateway.id))
     return channel
 
-class SlaveNode(object):
+class SlaveSession(Session):
-    def __init__(self, channel):
+    def __init__(self, config, channel, nodeid):
         self.channel = channel
+        self.nodeid = nodeid
+        super(SlaveSession, self).__init__(config=config)
 
     def __repr__(self):
         return "<%s channel=%s>" %(self.__class__.__name__, self.channel)
@@ -119,19 +135,24 @@ class SlaveNode(object):
     def pytest__teardown_final_logerror(self, report):
         self.sendevent("pytest__teardown_final_logerror", report=report)
 
-    def run(self):
+    def pytest_keyboard_interrupt(self, excinfo):
-        channel = self.channel
+        self._slaveerror = "SIGINT"
-        self.config, basetemp, self.nodeid = channel.receive()
+
-        if basetemp:
+    def pytest_internalerror(self, excrepr):
-            self.config.basetemp = py.path.local(basetemp)
+        self._slaveerror = "internal-error"
-        self.config.pluginmanager.do_configure(self.config)
+        self.sendevent("pytest_internalerror", excrepr=excrepr)
-        self.config.pluginmanager.register(self)
+
+    def dist_main(self):
         self.runner = self.config.pluginmanager.getplugin("pytest_runner")
         self.sendevent("slaveready")
-        try:
+        self.main(None)
-            self.config.hook.pytest_sessionstart(session=self)
+        error = getattr(self, '_slaveerror', None)
+        self.sendevent("slavefinished", error=error, 
+                       slaveoutput=self.config.slaveoutput)
+        
+    def _mainloop(self, colitems):
         while 1:
-                task = channel.receive()
+            task = self.channel.receive()
             if task is None: 
                 break
             if isinstance(task, list):
@@ -139,17 +160,6 @@ class SlaveNode(object):
                     self.run_single(item=item)
             else:
                 self.run_single(item=task)
-            self.config.hook.pytest_sessionfinish(
-                session=self, 
-                exitstatus=outcome.EXIT_OK)
-        except KeyboardInterrupt:
-            raise
-        except:
-            er = py.code.ExceptionInfo().getrepr(funcargs=True, showlocals=True)
-            self.sendevent("pytest_internalerror", excrepr=er)
-            raise
-        else:
-            self.sendevent("slavefinished")
 
     def run_single(self, item):
         call = self.runner.CallInfo(item._reraiseunpicklingproblem, when='setup')