merge nodemanager and gwmanager

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pytest-xdist",
-    version='1.5a9',
+    version='1.5a10',
     description='py.test xdist plugin for distributed testing and loop-on-failing modes',
     long_description=open('README.txt').read(),
     license='GPLv2 or later',

testing/test_slavemanage.py

@@ -1,15 +1,20 @@
 import py
 import os
 import execnet
-from xdist.slavemanage import GatewayManager, HostRSync
+from xdist.slavemanage import HostRSync, NodeManager
-from xdist.slavemanage import NodeManager
+
+pytest_plugins = "pytester",
 
 def pytest_funcarg__hookrecorder(request):
     _pytest = request.getfuncargvalue('_pytest')
-    hook = request.getfuncargvalue('hook')
+    config = request.getfuncargvalue('config')
-    return _pytest.gethookrecorder(hook)
+    return _pytest.gethookrecorder(config.hook)
+
+def pytest_funcarg__config(request):
+    testdir = request.getfuncargvalue("testdir")
+    config = testdir.parseconfig()
+    return config
 
-def pytest_funcarg__hook(request):
     from xdist import newhooks
     from _pytest.core import HookRelay, PluginManager
     from _pytest import hookspec
@@ -22,20 +27,20 @@ class pytest_funcarg__mysetup:
         self.dest = temp.mkdir("dest")
         request.getfuncargvalue("_pytest")
 
-class TestGatewayManagerPopen:
+class TestNodeManagerPopen:
-    def test_popen_no_default_chdir(self, hook):
+    def test_popen_no_default_chdir(self, config):
-        gm = GatewayManager(["popen"], hook)
+        gm = NodeManager(config, ["popen"])
         assert gm.specs[0].chdir is None
 
-    def test_default_chdir(self, hook):
+    def test_default_chdir(self, config):
         l = ["ssh=noco", "socket=xyz"]
-        for spec in GatewayManager(l, hook).specs:
+        for spec in NodeManager(config, l).specs:
             assert spec.chdir == "pyexecnetcache"
-        for spec in GatewayManager(l, hook, defaultchdir="abc").specs:
+        for spec in NodeManager(config, l, defaultchdir="abc").specs:
             assert spec.chdir == "abc"
 
-    def test_popen_makegateway_events(self, hook, hookrecorder, _pytest):
+    def test_popen_makegateway_events(self, config, hookrecorder, _pytest):
-        hm = GatewayManager(["popen"] * 2, hook)
+        hm = NodeManager(config, ["popen"] * 2)
         hm.makegateways()
         call = hookrecorder.popcall("pytest_gwmanage_newgateway")
         assert call.gateway.spec == execnet.XSpec("popen")
@@ -43,12 +48,12 @@ class TestGatewayManagerPopen:
         call = hookrecorder.popcall("pytest_gwmanage_newgateway")
         assert call.gateway.id == "gw1"
         assert len(hm.group) == 2
-        hm.exit()
+        hm.teardown_nodes()
         assert not len(hm.group)
 
-    def test_popens_rsync(self, hook, mysetup):
+    def test_popens_rsync(self, config, mysetup):
         source = mysetup.source
-        hm = GatewayManager(["popen"] * 2, hook)
+        hm = NodeManager(config, ["popen"] * 2)
         hm.makegateways()
         assert len(hm.group) == 2
         for gw in hm.group:
@@ -62,28 +67,28 @@ class TestGatewayManagerPopen:
         l = []
         hm.rsync(source, notify=lambda *args: l.append(args))
         assert not l
-        hm.exit()
+        hm.teardown_nodes()
         assert not len(hm.group)
         assert "sys.path.insert" in gw.remote_exec.args[0]
 
-    def test_rsync_popen_with_path(self, hook, mysetup):
+    def test_rsync_popen_with_path(self, config, mysetup):
         source, dest = mysetup.source, mysetup.dest
-        hm = GatewayManager(["popen//chdir=%s" %dest] * 1, hook)
+        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 1)
         hm.makegateways()
         source.ensure("dir1", "dir2", "hello")
         l = []
         hm.rsync(source, notify=lambda *args: l.append(args))
         assert len(l) == 1
         assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
-        hm.exit()
+        hm.teardown_nodes()
         dest = dest.join(source.basename)
         assert dest.join("dir1").check()
         assert dest.join("dir1", "dir2").check()
         assert dest.join("dir1", "dir2", 'hello').check()
 
-    def test_rsync_same_popen_twice(self, hook, mysetup, hookrecorder):
+    def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder):
         source, dest = mysetup.source, mysetup.dest
-        hm = GatewayManager(["popen//chdir=%s" %dest] * 2, hook)
+        hm = NodeManager(config, ["popen//chdir=%s" %dest] * 2)
         hm.makegateways()
         source.ensure("dir1", "dir2", "hello")
         hm.rsync(source)
@@ -156,14 +161,13 @@ class TestNodeManager:
                 "--rsyncdir", rsyncroot,
                 source,
             ))
-            #assert nodemanager.config.topdir == source
             nodemanager.rsync_roots()
             if rsyncroot == source:
                 dest = dest.join("source")
             assert dest.join("dir1").check()
             assert dest.join("dir1", "dir2").check()
             assert dest.join("dir1", "dir2", 'hello').check()
-            nodemanager.gwmanager.exit()
+            nodemanager.teardown_nodes()
 
     def test_init_rsync_roots(self, testdir, mysetup):
         source, dest = mysetup.source, mysetup.dest
@@ -209,25 +213,10 @@ class TestNodeManager:
         config = testdir.reparseconfig([source])
         nodemanager = NodeManager(config, specs)
         nodemanager.rsync_roots()
-        for gwspec in nodemanager.gwmanager.specs:
+        for gwspec in nodemanager.specs:
             assert gwspec._samefilesystem()
             assert not gwspec.chdir
 
-    def test_setup_DEBUG(self, mysetup, testdir):
-        source = mysetup.source
-        specs = ["popen"] * 2
-        source.join("conftest.py").write("rsyncdirs = ['a']")
-        source.ensure('a', dir=1)
-        config = testdir.parseconfigure(source, '--debug')
-        assert config.option.debug
-        nodemanager = NodeManager(config, specs)
-        reprec = testdir.getreportrecorder(config).hookrecorder
-        nodemanager.setup_nodes(putevent=[].append)
-        for spec in nodemanager.gwmanager.specs:
-            l = reprec.getcalls("pytest_trace")
-            assert l
-        nodemanager.teardown_nodes()
-
     def test_ssh_setup_nodes(self, specssh, testdir):
         testdir.makepyfile(__init__="", test_x="""
             def test_one():

xdist/__init__.py

@@ -1,2 +1,2 @@
 #
-__version__ = '1.5a9'
+__version__ = '1.5a10'

xdist/dsession.py

@@ -190,7 +190,7 @@ class DSession:
         return True
 
     def pytest_runtestloop(self):
-        numnodes = len(self.nodemanager.gwmanager.specs)
+        numnodes = len(self.nodemanager.specs)
         dist = self.config.getvalue("dist")
         if dist == "load":
             self.sched = LoadScheduling(numnodes, log=self.log)

xdist/slavemanage.py

@@ -6,15 +6,22 @@ import xdist.remote
 from _pytest import runner # XXX load dynamically
 
 class NodeManager(object):
-    def __init__(self, config, specs=None):
+    EXIT_TIMEOUT = 10
+    def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
-        if specs is None:
-            specs = self._getxspecs()
-        self.gwmanager = GatewayManager(specs, config.hook)
-        self.specs = self.gwmanager.specs
-        self.roots = self._getrsyncdirs()
         self._nodesready = py.std.threading.Event()
         self.trace = self.config.trace.get("nodemanager")
+        self.group = execnet.Group()
+        if specs is None:
+            specs = self._getxspecs()
+        self.specs = []
+        for spec in specs:
+            if not isinstance(spec, execnet.XSpec):
+                spec = execnet.XSpec(spec)
+            if not spec.chdir and not spec.popen:
+                spec.chdir = defaultchdir
+            self.specs.append(spec)
+        self.roots = self._getrsyncdirs()
 
     def config_getignores(self):
         return self.config.getini("rsyncignore")
@@ -32,37 +39,30 @@ class NodeManager(object):
         if self.roots:
             # send each rsync root
             for root in self.roots:
-                self.gwmanager.rsync(root, **options)
+                self.rsync(root, **options)
 
     def makegateways(self):
-        # we change to the topdir sot that
-        # PopenGateways will have their cwd
-        # such that unpickling configs will
-        # pick it up as the right topdir
-        # (for other gateways this chdir is irrelevant)
         self.trace("making gateways")
-        #old = self.config.topdir.chdir()
+        assert not list(self.group)
-        #try:
+        for spec in self.specs:
-        self.gwmanager.makegateways()
+            gw = self.group.makegateway(spec)
-        #finally:
+            self.config.hook.pytest_gwmanage_newgateway(gateway=gw)
-        #    old.chdir()
 
     def setup_nodes(self, putevent):
         self.rsync_roots()
         self.trace("setting up nodes")
-        for gateway in self.gwmanager.group:
+        for gateway in self.group:
             node = SlaveController(self, gateway, self.config, putevent)
             gateway.node = node  # to keep node alive
             node.setup()
             self.trace("started node %r" % node)
 
     def teardown_nodes(self):
-        self.gwmanager.exit()
+        self.group.terminate(self.EXIT_TIMEOUT)
 
     def _getxspecs(self):
-        config = self.config
         xspeclist = []
-        for xspec in config.getvalue("tx"):
+        for xspec in self.config.getvalue("tx"):
             i = xspec.find("*")
             try:
                 num = int(xspec[:i])
@@ -99,28 +99,6 @@ class NodeManager(object):
                 roots.append(root)
         return roots
 
-class GatewayManager:
-    """
-        instantiating, managing and rsyncing to test hosts
-    """
-    EXIT_TIMEOUT = 10
-    def __init__(self, specs, hook, defaultchdir="pyexecnetcache"):
-        self.specs = []
-        self.hook = hook
-        self.group = execnet.Group()
-        for spec in specs:
-            if not isinstance(spec, execnet.XSpec):
-                spec = execnet.XSpec(spec)
-            if not spec.chdir and not spec.popen:
-                spec.chdir = defaultchdir
-            self.specs.append(spec)
-
-    def makegateways(self):
-        assert not list(self.group)
-        for spec in self.specs:
-            gw = self.group.makegateway(spec)
-            self.hook.pytest_gwmanage_newgateway(gateway=gw)
-
     def rsync(self, source, notify=None, verbose=False, ignores=None):
         """ perform rsync to all remote hosts.
         """
@@ -144,19 +122,16 @@ class GatewayManager:
                 seen.add(spec)
                 gateways.append(gateway)
         if seen:
-            self.hook.pytest_gwmanage_rsyncstart(
+            self.config.hook.pytest_gwmanage_rsyncstart(
                 source=source,
                 gateways=gateways,
             )
             rsync.send()
-            self.hook.pytest_gwmanage_rsyncfinish(
+            self.config.hook.pytest_gwmanage_rsyncfinish(
                 source=source,
                 gateways=gateways,
             )
 
-    def exit(self):
-        self.group.terminate(self.EXIT_TIMEOUT)
-
 class HostRSync(execnet.RSync):
     """ RSyncer that filters out common files
     """