* introduce progress-reporting

* rename somewhat internal pytest_gwmanage_* events to pytest_xdist_*
* make gateway setup a separate step

CHANGELOG

@@ -4,9 +4,10 @@
 - adapt to pytest-2.0 changes, rsyncdirs and rsyncignore can now
   only be specified in [pytest] sections of ini files, see "py.test -h"
   for details.
-- major internal refactoring to match the py-1.4 event refactoring
+- major internal refactoring to match the pytest-2.0 event refactoring
   - perform test collection always at slave side instead of at the master
   - make python2/python3 bridging work, remove usage of pickling
+- improve initial reporting by using line-rewriting
 - remove all trailing whitespace from source
 
 1.4

testing/test_dsession.py

@@ -156,11 +156,11 @@ class TestDistReporter:
         #    platform = "xyz"
         #    cwd = "qwe"
 
-        #dsession.pytest_gwmanage_newgateway(gw1, rinfo)
+        #dsession.pytest_xdist_newgateway(gw1, rinfo)
         #linecomp.assert_contains_lines([
         #    "*X1*popen*xyz*2.5*"
         #])
-        dsession.pytest_gwmanage_rsyncstart(source="hello", gateways=[gw1, gw2])
+        dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
         linecomp.assert_contains_lines([
             "[X1,X2] rsyncing: hello",
         ])

testing/test_slavemanage.py

@@ -15,11 +15,6 @@ def pytest_funcarg__config(request):
     config = testdir.parseconfig()
     return config
 
-    from xdist import newhooks
-    from _pytest.core import HookRelay, PluginManager
-    from _pytest import hookspec
-    return HookRelay([hookspec, newhooks], PluginManager())
-
 class pytest_funcarg__mysetup:
     def __init__(self, request):
         temp = request.getfuncargvalue("tmpdir")
@@ -42,10 +37,13 @@ class TestNodeManagerPopen:
     def test_popen_makegateway_events(self, config, hookrecorder, _pytest):
         hm = NodeManager(config, ["popen"] * 2)
         hm.makegateways()
-        call = hookrecorder.popcall("pytest_gwmanage_newgateway")
+        call = hookrecorder.popcall("pytest_xdist_setupnodes")
+        assert len(call.specs) == 2
+
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
         assert call.gateway.spec == execnet.XSpec("popen")
         assert call.gateway.id == "gw0"
-        call = hookrecorder.popcall("pytest_gwmanage_newgateway")
+        call = hookrecorder.popcall("pytest_xdist_newgateway")
         assert call.gateway.id == "gw1"
         assert len(hm.group) == 2
         hm.teardown_nodes()
@@ -92,11 +90,11 @@ class TestNodeManagerPopen:
         hm.makegateways()
         source.ensure("dir1", "dir2", "hello")
         hm.rsync(source)
-        call = hookrecorder.popcall("pytest_gwmanage_rsyncstart")
+        call = hookrecorder.popcall("pytest_xdist_rsyncstart")
         assert call.source == source
         assert len(call.gateways) == 1
         assert call.gateways[0] in hm.group
-        call = hookrecorder.popcall("pytest_gwmanage_rsyncfinish")
+        call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
 
 class TestHRSync:
     class pytest_funcarg__mysetup:
@@ -140,6 +138,7 @@ class TestNodeManager:
         config = testdir.reparseconfig([source])
         nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
         #assert nodemanager.config.topdir == source == config.topdir
+        nodemanager.makegateways()
         nodemanager.rsync_roots()
         p, = nodemanager.gwmanager.multi_exec(
             "import os ; channel.send(os.getcwd())").receive_each()
@@ -161,6 +160,7 @@ class TestNodeManager:
                 "--rsyncdir", rsyncroot,
                 source,
             ))
+            nodemanager.makegateways()
             nodemanager.rsync_roots()
             if rsyncroot == source:
                 dest = dest.join("source")
@@ -181,6 +181,7 @@ class TestNodeManager:
         """))
         config = testdir.reparseconfig([source])
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.makegateways()
         nodemanager.rsync_roots()
         assert dest.join("dir2").check()
         assert not dest.join("dir1").check()
@@ -199,6 +200,7 @@ class TestNodeManager:
         """))
         config = testdir.reparseconfig([source])
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
+        nodemanager.makegateways()
         nodemanager.rsync_roots()
         assert dest.join("dir1").check()
         assert not dest.join("dir1", "dir2").check()
@@ -212,6 +214,7 @@ class TestNodeManager:
         source.ensure('a', dir=1)
         config = testdir.reparseconfig([source])
         nodemanager = NodeManager(config, specs)
+        nodemanager.makegateways()
         nodemanager.rsync_roots()
         for gwspec in nodemanager.specs:
             assert gwspec._samefilesystem()

xdist/dsession.py

@@ -166,6 +166,9 @@ class DSession:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
             self.terminal = None
+        else:
+            self.trdist = TerminalDistReporter(config)
+            config.pluginmanager.register(self.trdist, "terminaldistreporter")
 
     def report_line(self, line):
         if self.terminal and self.config.option.verbose >= 0:
@@ -173,9 +176,6 @@ class DSession:
 
     @pytest.mark.trylast
     def pytest_sessionstart(self, session):
-        if self.config.option.verbose > 0:
-            self.report_line("instantiating gateways (use -v for details): %s" %
-                ",".join(self.config.option.tx))
         self.nodemanager = NodeManager(self.config)
         self.nodemanager.setup_nodes(putevent=self.queue.put)
 
@@ -262,10 +262,15 @@ class DSession:
 
     def slave_collectionfinish(self, node, ids):
         self.sched.addnode_collection(node, ids)
-        self.report_line("[%s] collected %d test items" %(
+        if self.terminal:
-            node.gateway.id, len(ids)))
+            self.trdist.setstatus(node.gateway.spec, "[%d]" %(len(ids)))
 
         if self.sched.collection_is_completed:
+            if self.terminal:
+                self.terminal.write_line("")
+                self.terminal.write_line("scheduling tests via %s" %(
+                    self.sched.__class__.__name__))
+                
             self.sched.init_distribute()
 
     def slave_logstart(self, node, nodeid, location):
@@ -316,16 +321,43 @@ class TerminalDistReporter:
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")
+        self._status = {}
+        self._lastlen = 0
 
     def write_line(self, msg):
         self.tr.write_line(msg)
 
-    def pytest_gwmanage_newgateway(self, gateway):
+    def setstatus(self, spec, status, show=True):
+        self._status[spec.id] = status
+        if show:
+            parts = ["%s %s" %(spec.id, self._status[spec.id])
+                       for spec in self._specs]
+            line = " / ".join(parts)
+            self.rewrite(line)
+
+    def rewrite(self, line, newline=False):
+        pline = line + " " * max(self._lastlen-len(line), 0)
+        if newline:
+            self._lastlen = 0
+            pline += "\n"
+        else:
+            self._lastlen = len(line)
+        self.tr.rewrite(pline, bold=True)
+
+    def pytest_xdist_setupnodes(self, specs):
+        self._specs = specs
+        for spec in specs:
+            self.setstatus(spec, "initializing", show=False)
+        self.setstatus(spec, "initializing", show=True)
+
+    def pytest_xdist_newgateway(self, gateway):
+        if self.config.option.verbose > 0:
             rinfo = gateway._rinfo()
-        if self.config.option.verbose >= 0:
             version = "%s.%s.%s" % rinfo.version_info[:3]
-            self.write_line("[%s] %s Python %s cwd: %s" % (
+            self.rewrite("[%s] %s Python %s cwd: %s" % (
-                gateway.id, rinfo.platform, version, rinfo.cwd))
+                gateway.id, rinfo.platform, version, rinfo.cwd),
+                newline=True)
+        self.setstatus(gateway.spec, "collecting")
 
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
@@ -333,18 +365,19 @@ class TerminalDistReporter:
             infoline = "[%s] Python %s" %(
                 d['id'],
                 d['version'].replace('\n', ' -- '),)
-            self.write_line(infoline)
+            self.rewrite(infoline, newline=True)
+        self.setstatus(node.gateway.spec, "ready")
 
     def pytest_testnodedown(self, node, error):
         if not error:
             return
         self.write_line("[%s] node down: %s" %(node.gateway.id, error))
 
-    #def pytest_gwmanage_rsyncstart(self, source, gateways):
+    #def pytest_xdist_rsyncstart(self, source, gateways):
     #    targets = ",".join([gw.id for gw in gateways])
     #    msg = "[%s] rsyncing: %s" %(targets, source)
     #    self.write_line(msg)
-    #def pytest_gwmanage_rsyncfinish(self, source, gateways):
+    #def pytest_xdist_rsyncfinish(self, source, gateways):
     #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
     #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
 

xdist/newhooks.py

@@ -1,11 +1,14 @@
 
-def pytest_gwmanage_newgateway(gateway):
+def pytest_xdist_setupnodes(config, specs):
+    """ called before any remote node is set up. """
+
+def pytest_xdist_newgateway(gateway):
     """ called on new raw gateway creation. """
 
-def pytest_gwmanage_rsyncstart(source, gateways):
+def pytest_xdist_rsyncstart(source, gateways):
     """ called before rsyncing a directory to remote gateways takes place. """
 
-def pytest_gwmanage_rsyncfinish(source, gateways):
+def pytest_xdist_rsyncfinish(source, gateways):
     """ called after rsyncing a directory to remote gateways takes place. """
 
 def pytest_configure_node(node):

xdist/plugin.py

@@ -59,13 +59,10 @@ def pytest_cmdline_main(config):
 def pytest_configure(config, __multicall__):
     __multicall__.execute()
     if config.getvalue("dist") != "no":
-        from xdist.dsession import DSession, TerminalDistReporter
+        from xdist.dsession import DSession
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
 
-        trdist = TerminalDistReporter(config)
-        config.pluginmanager.register(trdist, "terminaldistreporter")
-
 def check_options(config):
     if config.option.numprocesses:
         config.option.dist = "load"

xdist/slavemanage.py

@@ -20,20 +20,17 @@ class NodeManager(object):
                 spec = execnet.XSpec(spec)
             if not spec.chdir and not spec.popen:
                 spec.chdir = defaultchdir
+            self.group.allocate_id(spec)
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
 
-    def config_getignores(self):
-        return self.config.getini("rsyncignore")
-
     def rsync_roots(self):
         """ make sure that all remote gateways
             have the same set of roots in their
             current directory.
         """
-        self.makegateways()
         options = {
-            'ignores': self.config_getignores(),
+            'ignores': self.config.getini("rsyncignore"),
             'verbose': self.config.option.verbose,
         }
         if self.roots:
@@ -42,13 +39,15 @@ class NodeManager(object):
                 self.rsync(root, **options)
 
     def makegateways(self):
-        self.trace("making gateways")
         assert not list(self.group)
+        self.config.hook.pytest_xdist_setupnodes(config=self.config,
+            specs=self.specs)
         for spec in self.specs:
             gw = self.group.makegateway(spec)
-            self.config.hook.pytest_gwmanage_newgateway(gateway=gw)
+            self.config.hook.pytest_xdist_newgateway(gateway=gw)
 
     def setup_nodes(self, putevent):
+        self.makegateways()
         self.rsync_roots()
         self.trace("setting up nodes")
         for gateway in self.group:
@@ -122,12 +121,12 @@ class NodeManager(object):
                 seen.add(spec)
                 gateways.append(gateway)
         if seen:
-            self.config.hook.pytest_gwmanage_rsyncstart(
+            self.config.hook.pytest_xdist_rsyncstart(
                 source=source,
                 gateways=gateways,
             )
             rsync.send()
-            self.config.hook.pytest_gwmanage_rsyncfinish(
+            self.config.hook.pytest_xdist_rsyncfinish(
                 source=source,
                 gateways=gateways,
             )