IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge nodemanager and gwmanager

Browse Source
This commit is contained in:
holger krekel
2010-11-22 12:05:55 +01:00
parent 3fc03c8c8e
commit 9108790801
5 changed files with 55 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
setup.py
View File

@@ -2,7 +2,7 @@ from setuptools import setup
setup(
name="pytest-xdist",
version='1.5a9',
version='1.5a10',
description='py.test xdist plugin for distributed testing and loop-on-failing modes',
long_description=open('README.txt').read(),
license='GPLv2 or later',

69
testing/test_slavemanage.py
View File

@@ -1,15 +1,20 @@
import py
import os
import execnet
from xdist.slavemanage import GatewayManager, HostRSync
from xdist.slavemanage import NodeManager
from xdist.slavemanage import HostRSync, NodeManager
pytest_plugins = "pytester",
def pytest_funcarg__hookrecorder(request):
_pytest = request.getfuncargvalue('_pytest')
hook = request.getfuncargvalue('hook')
return _pytest.gethookrecorder(hook)
config = request.getfuncargvalue('config')
return _pytest.gethookrecorder(config.hook)
def pytest_funcarg__config(request):
testdir = request.getfuncargvalue("testdir")
config = testdir.parseconfig()
return config
def pytest_funcarg__hook(request):
from xdist import newhooks
from _pytest.core import HookRelay, PluginManager
from _pytest import hookspec
@@ -22,20 +27,20 @@ class pytest_funcarg__mysetup:
self.dest = temp.mkdir("dest")
request.getfuncargvalue("_pytest")
class TestGatewayManagerPopen:
def test_popen_no_default_chdir(self, hook):
gm = GatewayManager(["popen"], hook)
class TestNodeManagerPopen:
def test_popen_no_default_chdir(self, config):
gm = NodeManager(config, ["popen"])
assert gm.specs[0].chdir is None
def test_default_chdir(self, hook):
def test_default_chdir(self, config):
l = ["ssh=noco", "socket=xyz"]
for spec in GatewayManager(l, hook).specs:
for spec in NodeManager(config, l).specs:
assert spec.chdir == "pyexecnetcache"
for spec in GatewayManager(l, hook, defaultchdir="abc").specs:
for spec in NodeManager(config, l, defaultchdir="abc").specs:
assert spec.chdir == "abc"
def test_popen_makegateway_events(self, hook, hookrecorder, _pytest):
hm = GatewayManager(["popen"] * 2, hook)
def test_popen_makegateway_events(self, config, hookrecorder, _pytest):
hm = NodeManager(config, ["popen"] * 2)
hm.makegateways()
call = hookrecorder.popcall("pytest_gwmanage_newgateway")
assert call.gateway.spec == execnet.XSpec("popen")
@@ -43,12 +48,12 @@ class TestGatewayManagerPopen:
call = hookrecorder.popcall("pytest_gwmanage_newgateway")
assert call.gateway.id == "gw1"
assert len(hm.group) == 2
hm.exit()
hm.teardown_nodes()
assert not len(hm.group)
def test_popens_rsync(self, hook, mysetup):
def test_popens_rsync(self, config, mysetup):
source = mysetup.source
hm = GatewayManager(["popen"] * 2, hook)
hm = NodeManager(config, ["popen"] * 2)
hm.makegateways()
assert len(hm.group) == 2
for gw in hm.group:
@@ -62,28 +67,28 @@ class TestGatewayManagerPopen:
l = []
hm.rsync(source, notify=lambda *args: l.append(args))
assert not l
hm.exit()
hm.teardown_nodes()
assert not len(hm.group)
assert "sys.path.insert" in gw.remote_exec.args[0]
def test_rsync_popen_with_path(self, hook, mysetup):
def test_rsync_popen_with_path(self, config, mysetup):
source, dest = mysetup.source, mysetup.dest
hm = GatewayManager(["popen//chdir=%s" %dest] * 1, hook)
hm = NodeManager(config, ["popen//chdir=%s" %dest] * 1)
hm.makegateways()
source.ensure("dir1", "dir2", "hello")
l = []
hm.rsync(source, notify=lambda *args: l.append(args))
assert len(l) == 1
assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
hm.exit()
hm.teardown_nodes()
dest = dest.join(source.basename)
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
def test_rsync_same_popen_twice(self, hook, mysetup, hookrecorder):
def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder):
source, dest = mysetup.source, mysetup.dest
hm = GatewayManager(["popen//chdir=%s" %dest] * 2, hook)
hm = NodeManager(config, ["popen//chdir=%s" %dest] * 2)
hm.makegateways()
source.ensure("dir1", "dir2", "hello")
hm.rsync(source)
@@ -156,14 +161,13 @@ class TestNodeManager:
"--rsyncdir", rsyncroot,
source,
))
#assert nodemanager.config.topdir == source
nodemanager.rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
nodemanager.gwmanager.exit()
nodemanager.teardown_nodes()
def test_init_rsync_roots(self, testdir, mysetup):
source, dest = mysetup.source, mysetup.dest
@@ -209,25 +213,10 @@ class TestNodeManager:
config = testdir.reparseconfig([source])
nodemanager = NodeManager(config, specs)
nodemanager.rsync_roots()
for gwspec in nodemanager.gwmanager.specs:
for gwspec in nodemanager.specs:
assert gwspec._samefilesystem()
assert not gwspec.chdir
def test_setup_DEBUG(self, mysetup, testdir):
source = mysetup.source
specs = ["popen"] * 2
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = testdir.parseconfigure(source, '--debug')
assert config.option.debug
nodemanager = NodeManager(config, specs)
reprec = testdir.getreportrecorder(config).hookrecorder
nodemanager.setup_nodes(putevent=[].append)
for spec in nodemanager.gwmanager.specs:
l = reprec.getcalls("pytest_trace")
assert l
nodemanager.teardown_nodes()
def test_ssh_setup_nodes(self, specssh, testdir):
testdir.makepyfile(__init__="", test_x="""
def test_one():

2
xdist/__init__.py
View File

@@ -1,2 +1,2 @@
#
__version__ = '1.5a9'
__version__ = '1.5a10'

2
xdist/dsession.py
View File

@@ -190,7 +190,7 @@ class DSession:
return True
def pytest_runtestloop(self):
numnodes = len(self.nodemanager.gwmanager.specs)
numnodes = len(self.nodemanager.specs)
dist = self.config.getvalue("dist")
if dist == "load":
self.sched = LoadScheduling(numnodes, log=self.log)

71
xdist/slavemanage.py
View File

@@ -6,15 +6,22 @@ import xdist.remote
from _pytest import runner # XXX load dynamically
class NodeManager(object):
def __init__(self, config, specs=None):
EXIT_TIMEOUT = 10
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
if specs is None:
specs = self._getxspecs()
self.gwmanager = GatewayManager(specs, config.hook)
self.specs = self.gwmanager.specs
self.roots = self._getrsyncdirs()
self._nodesready = py.std.threading.Event()
self.trace = self.config.trace.get("nodemanager")
self.group = execnet.Group()
if specs is None:
specs = self._getxspecs()
self.specs = []
for spec in specs:
if not isinstance(spec, execnet.XSpec):
spec = execnet.XSpec(spec)
if not spec.chdir and not spec.popen:
spec.chdir = defaultchdir
self.specs.append(spec)
self.roots = self._getrsyncdirs()
def config_getignores(self):
return self.config.getini("rsyncignore")
@@ -32,37 +39,30 @@ class NodeManager(object):
if self.roots:
# send each rsync root
for root in self.roots:
self.gwmanager.rsync(root, **options)
self.rsync(root, **options)
def makegateways(self):
# we change to the topdir sot that
# PopenGateways will have their cwd
# such that unpickling configs will
# pick it up as the right topdir
# (for other gateways this chdir is irrelevant)
self.trace("making gateways")
#old = self.config.topdir.chdir()
#try:
self.gwmanager.makegateways()
#finally:
# old.chdir()
assert not list(self.group)
for spec in self.specs:
gw = self.group.makegateway(spec)
self.config.hook.pytest_gwmanage_newgateway(gateway=gw)
def setup_nodes(self, putevent):
self.rsync_roots()
self.trace("setting up nodes")
for gateway in self.gwmanager.group:
for gateway in self.group:
node = SlaveController(self, gateway, self.config, putevent)
gateway.node = node # to keep node alive
node.setup()
self.trace("started node %r" % node)
def teardown_nodes(self):
self.gwmanager.exit()
self.group.terminate(self.EXIT_TIMEOUT)
def _getxspecs(self):
config = self.config
xspeclist = []
for xspec in config.getvalue("tx"):
for xspec in self.config.getvalue("tx"):
i = xspec.find("*")
try:
num = int(xspec[:i])
@@ -99,28 +99,6 @@ class NodeManager(object):
roots.append(root)
return roots
class GatewayManager:
"""
instantiating, managing and rsyncing to test hosts
"""
EXIT_TIMEOUT = 10
def __init__(self, specs, hook, defaultchdir="pyexecnetcache"):
self.specs = []
self.hook = hook
self.group = execnet.Group()
for spec in specs:
if not isinstance(spec, execnet.XSpec):
spec = execnet.XSpec(spec)
if not spec.chdir and not spec.popen:
spec.chdir = defaultchdir
self.specs.append(spec)
def makegateways(self):
assert not list(self.group)
for spec in self.specs:
gw = self.group.makegateway(spec)
self.hook.pytest_gwmanage_newgateway(gateway=gw)
def rsync(self, source, notify=None, verbose=False, ignores=None):
""" perform rsync to all remote hosts.
"""
@@ -144,19 +122,16 @@ class GatewayManager:
seen.add(spec)
gateways.append(gateway)
if seen:
self.hook.pytest_gwmanage_rsyncstart(
self.config.hook.pytest_gwmanage_rsyncstart(
source=source,
gateways=gateways,
)
rsync.send()
self.hook.pytest_gwmanage_rsyncfinish(
self.config.hook.pytest_gwmanage_rsyncfinish(
source=source,
gateways=gateways,
)
def exit(self):
self.group.terminate(self.EXIT_TIMEOUT)
class HostRSync(execnet.RSync):
""" RSyncer that filters out common files
"""