adapt to new refactored py baseline.
remove pickling in distributed testing refactor internal mechanism enough to work for -n1 * * * fix handling of crashing items and slave down events
This commit is contained in:
holger krekel
@@ -1,81 +1,135 @@
|
||||
import py
|
||||
from py._test import session
|
||||
import sys
|
||||
from xdist.nodemanage import NodeManager
|
||||
from py._test import session
|
||||
import kwlog
|
||||
queue = py.builtin._tryimport('queue', 'Queue')
|
||||
|
||||
debug_file = None # open('/tmp/loop.log', 'w')
|
||||
def debug(*args):
|
||||
if debug_file is not None:
|
||||
s = " ".join(map(str, args))
|
||||
debug_file.write(s+"\n")
|
||||
debug_file.flush()
|
||||
def dsession_main(config):
|
||||
config.pluginmanager.do_configure(config)
|
||||
session = DSession(config)
|
||||
trdist = TerminalDistReporter(config)
|
||||
config.pluginmanager.register(trdist, "terminaldistreporter")
|
||||
exitcode = session.main()
|
||||
config.pluginmanager.do_unconfigure(config)
|
||||
return exitcode
|
||||
|
||||
class LoopState(object):
|
||||
def __init__(self, dsession, colitems):
|
||||
self.dsession = dsession
|
||||
self.colitems = colitems
|
||||
self.exitstatus = None
|
||||
# loopstate.dowork is False after reschedule events
|
||||
# because otherwise we might very busily loop
|
||||
# waiting for a host to become ready.
|
||||
self.dowork = True
|
||||
self.shuttingdown = False
|
||||
self.testsfailed = 0
|
||||
|
||||
def __repr__(self):
|
||||
return "<LoopState exitstatus=%r shuttingdown=%r len(colitems)=%d>" % (
|
||||
self.exitstatus, self.shuttingdown, len(self.colitems))
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if report.item in self.dsession.item2nodes:
|
||||
if report.when != "teardown": # otherwise we already managed it
|
||||
self.dsession.removeitem(report.item, report.node)
|
||||
if report.failed:
|
||||
self.testsfailed += 1
|
||||
|
||||
def pytest_collectreport(self, report):
|
||||
if report.passed:
|
||||
self.colitems.extend(report.result)
|
||||
|
||||
def pytest_testnodeready(self, node):
|
||||
self.dsession.addnode(node)
|
||||
|
||||
def pytest_testnodedown(self, node, error=None):
|
||||
pending = self.dsession.removenode(node)
|
||||
if pending:
|
||||
if error:
|
||||
crashitem = pending[0]
|
||||
debug("determined crashitem", crashitem)
|
||||
self.dsession.handle_crashitem(crashitem, node)
|
||||
# XXX recovery handling for "each"?
|
||||
# currently pending items are not retried
|
||||
if self.dsession.config.option.dist == "load":
|
||||
self.colitems.extend(pending[1:])
|
||||
|
||||
def pytest_rescheduleitems(self, items):
|
||||
self.colitems[:] = items + self.colitems
|
||||
for pending in self.dsession.node2pending.values():
|
||||
if pending:
|
||||
self.dowork = False # avoid busywait, nodes still have work
|
||||
|
||||
class DSession(session.Session):
|
||||
"""
|
||||
Session drives the collection and running of tests
|
||||
and generates test events for reporters.
|
||||
"""
|
||||
class LoadScheduling:
|
||||
LOAD_THRESHOLD_NEWITEMS = 5
|
||||
ITEM_CHUNKSIZE = 10
|
||||
|
||||
def __init__(self, config):
|
||||
self.queue = queue.Queue()
|
||||
def __init__(self, numnodes, log=None):
|
||||
self.numnodes = numnodes
|
||||
self.node2pending = {}
|
||||
self.node2collection = {}
|
||||
self.pending = []
|
||||
if log is None:
|
||||
self.log = kwlog.Producer("loadsched")
|
||||
else:
|
||||
self.log = log.loadsched
|
||||
|
||||
def hasnodes(self):
|
||||
return bool(self.node2pending)
|
||||
|
||||
def addnode(self, node):
|
||||
self.node2pending[node] = []
|
||||
|
||||
def collection_is_completed(self):
|
||||
return len(self.node2collection) == self.numnodes
|
||||
|
||||
def tests_finished(self):
|
||||
if not self.collection_is_completed() or self.pending:
|
||||
return False
|
||||
for items in self.node2pending.values():
|
||||
if items:
|
||||
return False
|
||||
return True
|
||||
|
||||
def addnode_collection(self, node, collection):
|
||||
assert node in self.node2pending
|
||||
self.node2collection[node] = list(collection)
|
||||
|
||||
def remove_item(self, node, item):
|
||||
if item not in self.item2nodes:
|
||||
raise AssertionError(item, self.item2nodes)
|
||||
nodes = self.item2nodes[item]
|
||||
if node in nodes: # the node might have gone down already
|
||||
nodes.remove(node)
|
||||
#if not nodes:
|
||||
# del self.item2nodes[item]
|
||||
pending = self.node2pending[node]
|
||||
pending.remove(item)
|
||||
# pre-load items-to-test if the node may become ready
|
||||
if self.pending and len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
|
||||
item = self.pending.pop(0)
|
||||
pending.append(item)
|
||||
self.item2nodes.setdefault(item, []).append(node)
|
||||
node.send_runtest(item)
|
||||
|
||||
def remove_node(self, node):
|
||||
pending = self.node2pending.pop(node)
|
||||
# KeyError if we didn't get an addnode() yet
|
||||
for item in pending:
|
||||
l = self.item2nodes[item]
|
||||
l.remove(node)
|
||||
if not l:
|
||||
del self.item2nodes[item]
|
||||
if not pending:
|
||||
return
|
||||
crashitem = pending.pop(0)
|
||||
self.pending.extend(pending)
|
||||
return crashitem
|
||||
|
||||
def init_distribute(self):
|
||||
assert self.collection_is_completed()
|
||||
assert not hasattr(self, 'item2nodes')
|
||||
self.item2nodes = {}
|
||||
super(DSession, self).__init__(config=config)
|
||||
# XXX allow nodes to have different collections
|
||||
col = list(self.node2collection.values())[0]
|
||||
for node, collection in self.node2collection.items():
|
||||
assert collection == col
|
||||
self.pending = col
|
||||
|
||||
def triggertesting(self):
|
||||
if not self.pending:
|
||||
return
|
||||
available = []
|
||||
for node, pending in self.node2pending.items():
|
||||
if len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
|
||||
available.append((node, pending))
|
||||
num_available = len(available)
|
||||
if num_available:
|
||||
max_one_round = num_available * self.ITEM_CHUNKSIZE -1
|
||||
for i, item in enumerate(self.pending):
|
||||
nodeindex = i % num_available
|
||||
node, pending = available[nodeindex]
|
||||
node.send_runtest(item)
|
||||
self.item2nodes.setdefault(item, []).append(node)
|
||||
#item.ihook.pytest_itemstart(item=item, node=node)
|
||||
pending.append(item)
|
||||
if i >= max_one_round:
|
||||
break
|
||||
del self.pending[:i+1]
|
||||
if self.pending:
|
||||
self.log.debug("triggertesting remaining:", len(self.pending))
|
||||
|
||||
class Interrupted(KeyboardInterrupt):
|
||||
""" signals an immediate interruption. """
|
||||
|
||||
class DSession:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.log = kwlog.Producer("dsession")
|
||||
#kwlog.setconsumer(self.log, kwlog.Path("/tmp/x.log"))
|
||||
kwlog.setconsumer(self.log, None)
|
||||
self.shuttingdown = False
|
||||
self.countfailures = 0
|
||||
self.maxfail = config.getvalue("maxfail")
|
||||
self.queue = queue.Queue()
|
||||
try:
|
||||
self.terminal = config.pluginmanager.getplugin("terminalreporter")
|
||||
except KeyError:
|
||||
self.terminal = None
|
||||
self._nodesready = py.std.threading.Event()
|
||||
|
||||
def report_line(self, line):
|
||||
if self.terminal:
|
||||
@@ -90,98 +144,75 @@ class DSession(session.Session):
|
||||
# targets = ", ".join(["[%s]" % gw.id for gw in gateways])
|
||||
# self.write_line("rsyncfinish: %s -> %s" %(source, targets))
|
||||
|
||||
def main(self, colitems):
|
||||
self.sessionstarts()
|
||||
def main(self):
|
||||
self.config.hook.pytest_sessionstart(session=self)
|
||||
self.setup()
|
||||
allitems = self.collect_all_items(colitems)
|
||||
exitstatus = self.loop(allitems)
|
||||
exitstatus = self.loop()
|
||||
self.teardown()
|
||||
self.sessionfinishes(exitstatus=exitstatus)
|
||||
self.config.hook.pytest_sessionfinish(session=self,
|
||||
exitstatus=exitstatus,)
|
||||
return exitstatus
|
||||
|
||||
def collect_all_items(self, colitems):
|
||||
verbose = self.config.getvalue("verbose")
|
||||
if verbose:
|
||||
self.report_line("[master] starting full item collection ...")
|
||||
allitems = list(self.collect(colitems))
|
||||
if verbose:
|
||||
self.report_line("[master] collected %d items" %(len(allitems)))
|
||||
return allitems
|
||||
def slave_slaveready(self, node):
|
||||
self.sched.addnode(node)
|
||||
if self.shuttingdown:
|
||||
node.sendcommand("shutdown")
|
||||
|
||||
def loop_once(self, loopstate):
|
||||
if loopstate.shuttingdown:
|
||||
return self.loop_once_shutdown(loopstate)
|
||||
colitems = loopstate.colitems
|
||||
if self._nodesready.isSet() and loopstate.dowork and colitems:
|
||||
self.triggertesting(loopstate.colitems)
|
||||
colitems[:] = []
|
||||
# we use a timeout here so that control-C gets through
|
||||
while 1:
|
||||
try:
|
||||
eventcall = self.queue.get(timeout=2.0)
|
||||
break
|
||||
except queue.Empty:
|
||||
continue
|
||||
loopstate.dowork = True
|
||||
def slave_slavefinished(self, node):
|
||||
crashitem = self.sched.remove_node(node)
|
||||
assert not crashitem, (crashitem, node)
|
||||
if self.shuttingdown and not self.sched.hasnodes():
|
||||
self.session_finished = True
|
||||
|
||||
callname, args, kwargs = eventcall
|
||||
if callname is not None:
|
||||
call = getattr(self.config.hook, callname)
|
||||
assert not args
|
||||
call(**kwargs)
|
||||
def slave_errordown(self, node, error):
|
||||
self.report_line("node %r down on error: %s" %(node.gateway.id, error,))
|
||||
crashitem = self.sched.remove_node(node)
|
||||
if crashitem:
|
||||
self.handle_crashitem(crashitem, node)
|
||||
#self.report_line("item crashed on node: %s" % crashitem)
|
||||
if not self.sched.hasnodes():
|
||||
self.session_finished = True
|
||||
|
||||
# termination conditions
|
||||
maxfail = self.config.getvalue("maxfail")
|
||||
if (not self.node2pending or
|
||||
(loopstate.testsfailed and maxfail and
|
||||
loopstate.testsfailed >= maxfail) or
|
||||
(not self.item2nodes and not colitems and not self.queue.qsize())):
|
||||
if maxfail and loopstate.testsfailed >= maxfail:
|
||||
raise self.Interrupted("stopping after %d failures" % (
|
||||
loopstate.testsfailed))
|
||||
self.triggershutdown()
|
||||
loopstate.shuttingdown = True
|
||||
if not self.node2pending:
|
||||
loopstate.exitstatus = session.EXIT_NOHOSTS
|
||||
def slave_collectionfinish(self, node, ids):
|
||||
self.sched.addnode_collection(node, ids)
|
||||
self.report_line("[%s] collected %d test items" %(
|
||||
node.gateway.id, len(ids)))
|
||||
|
||||
def loop_once_shutdown(self, loopstate):
|
||||
# once we are in shutdown mode we dont send
|
||||
# events other than HostDown upstream
|
||||
eventname, args, kwargs = self.queue.get()
|
||||
if eventname == "pytest_testnodedown":
|
||||
self.config.hook.pytest_testnodedown(**kwargs)
|
||||
self.removenode(kwargs['node'])
|
||||
elif eventname == "pytest_runtest_logreport":
|
||||
# might be some teardown report
|
||||
self.config.hook.pytest_runtest_logreport(**kwargs)
|
||||
elif eventname == "pytest_internalerror":
|
||||
self.config.hook.pytest_internalerror(**kwargs)
|
||||
loopstate.exitstatus = session.EXIT_INTERNALERROR
|
||||
elif eventname == "pytest__teardown_final_logerror":
|
||||
self.config.hook.pytest__teardown_final_logerror(**kwargs)
|
||||
loopstate.exitstatus = session.EXIT_TESTSFAILED
|
||||
if not self.node2pending:
|
||||
# finished
|
||||
if loopstate.testsfailed:
|
||||
loopstate.exitstatus = session.EXIT_TESTSFAILED
|
||||
else:
|
||||
loopstate.exitstatus = session.EXIT_OK
|
||||
#self.config.pluginmanager.unregister(loopstate)
|
||||
if self.sched.collection_is_completed():
|
||||
self.sched.init_distribute()
|
||||
self.sched.triggertesting()
|
||||
|
||||
def _initloopstate(self, colitems):
|
||||
loopstate = LoopState(self, colitems)
|
||||
self.config.pluginmanager.register(loopstate)
|
||||
return loopstate
|
||||
def slave_logstart(self, node, nodeid, location):
|
||||
self.config.hook.pytest_runtest_logstart(
|
||||
nodeid=nodeid, location=location)
|
||||
|
||||
def loop(self, colitems):
|
||||
def slave_testreport(self, node, rep):
|
||||
self.sched.remove_item(node, rep.nodeid)
|
||||
#self.report_line("testreport %s: %s" %(rep.id, rep.status))
|
||||
self.config.hook.pytest_runtest_logreport(report=rep)
|
||||
self._handlefailures(rep)
|
||||
|
||||
def slave_collectreport(self, node, rep):
|
||||
#self.report_line("collectreport %s: %s" %(rep.id, rep.status))
|
||||
self._handlefailures(rep)
|
||||
|
||||
def _handlefailures(self, rep):
|
||||
if rep.failed:
|
||||
self.countfailures += 1
|
||||
if self.maxfail and self.countfailures >= self.maxfail:
|
||||
self.shouldstop = "stopping after %d failures" % (
|
||||
self.countfailures)
|
||||
|
||||
def loop(self):
|
||||
self.sched = LoadScheduling(len(self.config.option.tx), log=self.log)
|
||||
self.shouldstop = False
|
||||
self.session_finished = False
|
||||
exitstatus = 0
|
||||
try:
|
||||
loopstate = self._initloopstate(colitems)
|
||||
loopstate.dowork = False # first receive at least one HostUp events
|
||||
while 1:
|
||||
self.loop_once(loopstate)
|
||||
if loopstate.exitstatus is not None:
|
||||
exitstatus = loopstate.exitstatus
|
||||
break
|
||||
while not (self.session_finished or self.shouldstop):
|
||||
self.loop_once()
|
||||
if self.shouldstop:
|
||||
raise Interrupted(str(self.shouldstop))
|
||||
except KeyboardInterrupt:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
|
||||
@@ -189,103 +220,40 @@ class DSession(session.Session):
|
||||
except:
|
||||
self.config.pluginmanager.notify_exception()
|
||||
exitstatus = session.EXIT_INTERNALERROR
|
||||
self.config.pluginmanager.unregister(loopstate)
|
||||
if exitstatus == 0 and self._testsfailed:
|
||||
#self.config.pluginmanager.unregister(loopstate)
|
||||
if exitstatus == 0 and self.countfailures:
|
||||
exitstatus = session.EXIT_TESTSFAILED
|
||||
return exitstatus
|
||||
|
||||
def loop_once(self):
|
||||
while 1:
|
||||
try:
|
||||
eventcall = self.queue.get(timeout=2.0)
|
||||
break
|
||||
except queue.Empty:
|
||||
continue
|
||||
callname, kwargs = eventcall
|
||||
assert callname, kwargs
|
||||
method = "slave_" + callname
|
||||
call = getattr(self, method)
|
||||
self.log.debug("calling method: %s(**%s)" % (method, kwargs))
|
||||
call(**kwargs)
|
||||
if self.sched.tests_finished():
|
||||
self.triggershutdown()
|
||||
|
||||
def triggershutdown(self):
|
||||
for node in self.node2pending:
|
||||
self.shuttingdown = True
|
||||
for node in self.sched.node2pending:
|
||||
node.shutdown()
|
||||
|
||||
def addnode(self, node):
|
||||
assert node not in self.node2pending
|
||||
self.node2pending[node] = []
|
||||
if (not hasattr(self, 'nodemanager') or
|
||||
len(self.node2pending) == len(self.nodemanager.gwmanager.group)):
|
||||
self._nodesready.set()
|
||||
|
||||
def removenode(self, node):
|
||||
try:
|
||||
pending = self.node2pending.pop(node)
|
||||
except KeyError:
|
||||
# this happens if we didn't receive a testnodeready event yet
|
||||
return []
|
||||
for item in pending:
|
||||
l = self.item2nodes[item]
|
||||
l.remove(node)
|
||||
if not l:
|
||||
del self.item2nodes[item]
|
||||
return pending
|
||||
|
||||
def triggertesting(self, colitems):
|
||||
# for now we don't allow sending collectors
|
||||
for next in colitems:
|
||||
assert isinstance(next, py.test.collect.Item), next
|
||||
senditems = list(colitems)
|
||||
if self.config.option.dist == "each":
|
||||
self.senditems_each(senditems)
|
||||
else:
|
||||
# XXX assert self.config.option.dist == "load"
|
||||
self.senditems_load(senditems)
|
||||
|
||||
def queueevent(self, eventname, **kwargs):
|
||||
self.queue.put((eventname, (), kwargs))
|
||||
|
||||
def senditems_each(self, tosend):
|
||||
if not tosend:
|
||||
return
|
||||
for node, pending in self.node2pending.items():
|
||||
node.sendlist(tosend)
|
||||
pending.extend(tosend)
|
||||
for item in tosend:
|
||||
nodes = self.item2nodes.setdefault(item, [])
|
||||
assert node not in nodes
|
||||
nodes.append(node)
|
||||
item.ihook.pytest_itemstart(item=item, node=node)
|
||||
tosend[:] = []
|
||||
|
||||
def senditems_load(self, tosend):
|
||||
if not tosend:
|
||||
return
|
||||
available = []
|
||||
for node, pending in self.node2pending.items():
|
||||
if len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
|
||||
available.append((node, pending))
|
||||
num_available = len(available)
|
||||
max_one_round = num_available * self.ITEM_CHUNKSIZE -1
|
||||
if num_available:
|
||||
for i, item in enumerate(tosend):
|
||||
nodeindex = i % num_available
|
||||
node, pending = available[nodeindex]
|
||||
node.send(item)
|
||||
self.item2nodes.setdefault(item, []).append(node)
|
||||
item.ihook.pytest_itemstart(item=item, node=node)
|
||||
pending.append(item)
|
||||
if i >= max_one_round:
|
||||
break
|
||||
del tosend[:i+1]
|
||||
if tosend:
|
||||
# we have some left, give it to the main loop
|
||||
self.queueevent("pytest_rescheduleitems", items=tosend)
|
||||
|
||||
def removeitem(self, item, node):
|
||||
if item not in self.item2nodes:
|
||||
raise AssertionError(item, self.item2nodes)
|
||||
nodes = self.item2nodes[item]
|
||||
if node in nodes: # the node might have gone down already
|
||||
nodes.remove(node)
|
||||
if not nodes:
|
||||
del self.item2nodes[item]
|
||||
pending = self.node2pending[node]
|
||||
pending.remove(item)
|
||||
|
||||
def handle_crashitem(self, item, node):
|
||||
runner = item.config.pluginmanager.getplugin("runner")
|
||||
info = "!!! Node %r crashed during running of test %r" %(node, item)
|
||||
rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
|
||||
rep.node = node
|
||||
item.ihook.pytest_runtest_logreport(report=rep)
|
||||
def handle_crashitem(self, nodeid, slave):
|
||||
# XXX get more reporting info by recording pytest_runtest_logstart?
|
||||
runner = self.config.pluginmanager.getplugin("runner")
|
||||
fspath = nodeid.split("::")[0]
|
||||
msg = "Slave %r crashed while running %r" %(slave.gateway.id, nodeid)
|
||||
rep = runner.TestReport(nodeid, (), fspath, (fspath, None, fspath), (),
|
||||
"failed", msg, "???")
|
||||
self.config.hook.pytest_runtest_logreport(report=rep)
|
||||
|
||||
def setup(self):
|
||||
""" setup any neccessary resources ahead of the test run. """
|
||||
@@ -298,3 +266,55 @@ class DSession(session.Session):
|
||||
def teardown(self):
|
||||
""" teardown any resources after a test run. """
|
||||
self.nodemanager.teardown_nodes()
|
||||
|
||||
class TerminalDistReporter:
|
||||
def __init__(self, config):
|
||||
self.gateway2info = {}
|
||||
self.config = config
|
||||
self.tplugin = config.pluginmanager.getplugin("terminal")
|
||||
self.tr = config.pluginmanager.getplugin("terminalreporter")
|
||||
|
||||
def write_line(self, msg):
|
||||
self.tr.write_line(msg)
|
||||
|
||||
def pytest_itemstart(self, __multicall__):
|
||||
try:
|
||||
__multicall__.methods.remove(self.tr.pytest_itemstart)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if hasattr(report, 'node'):
|
||||
report.headerlines.append(self.gateway2info.get(
|
||||
report.node.gateway,
|
||||
"node %r (platinfo not found? strange)"))
|
||||
|
||||
def pytest_gwmanage_newgateway(self, gateway, platinfo):
|
||||
#self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec))
|
||||
d = {}
|
||||
d['version'] = self.tplugin.repr_pythonversion(platinfo.version_info)
|
||||
d['id'] = gateway.id
|
||||
d['spec'] = gateway.spec._spec
|
||||
d['platform'] = platinfo.platform
|
||||
if self.config.option.verbose:
|
||||
d['extra'] = "- " + platinfo.executable
|
||||
else:
|
||||
d['extra'] = ""
|
||||
d['cwd'] = platinfo.cwd
|
||||
infoline = ("[%(id)s] %(spec)s -- platform %(platform)s, "
|
||||
"Python %(version)s "
|
||||
"cwd: %(cwd)s"
|
||||
"%(extra)s" % d)
|
||||
if self.config.getvalue("verbose"):
|
||||
self.write_line(infoline)
|
||||
self.gateway2info[gateway] = infoline
|
||||
|
||||
def pytest_testnodeready(self, node):
|
||||
if self.config.getvalue("verbose"):
|
||||
self.write_line(
|
||||
"[%s] txnode ready to receive tests" %(node.gateway.id,))
|
||||
|
||||
def pytest_testnodedown(self, node, error):
|
||||
if not error:
|
||||
return
|
||||
self.write_line("[%s] node down, error: %s" %(node.gateway.id, error))
|
||||
|
||||
@@ -184,8 +184,9 @@ class SlaveFailSession:
|
||||
self.config.hook.pytest_cmdline_main(config=self.config)
|
||||
trails, failreports = [], []
|
||||
for rep in self.recorded_failures:
|
||||
trails.append(self.collection.getid(rep.getnode()))
|
||||
loc = rep._getcrashline()
|
||||
trails.append(rep.nodeid)
|
||||
loc = rep.longrepr
|
||||
loc = str(getattr(loc, 'reprcrash', loc))
|
||||
failreports.append(loc)
|
||||
topdir = str(self.topdir)
|
||||
self.channel.send((topdir, trails, failreports, self.collection_failed))
|
||||
|
||||
@@ -1,183 +0,0 @@
|
||||
"""
|
||||
|
||||
Pickling support for two processes that want to exchange
|
||||
*immutable* object instances. Immutable in the sense
|
||||
that the receiving side of an object can modify its
|
||||
copy but when it sends it back the original sending
|
||||
side will continue to see its unmodified version
|
||||
(and no actual state will go over the wire).
|
||||
|
||||
This module also implements an experimental
|
||||
execnet pickling channel using this idea.
|
||||
|
||||
"""
|
||||
|
||||
import py
|
||||
import sys, os, struct
|
||||
#debug = open("log-mypickle-%d" % os.getpid(), 'w')
|
||||
|
||||
if sys.version_info >= (3,0):
|
||||
makekey = lambda x: x
|
||||
fromkey = lambda x: x
|
||||
from pickle import _Pickler as Pickler
|
||||
from pickle import _Unpickler as Unpickler
|
||||
else:
|
||||
makekey = str
|
||||
fromkey = int
|
||||
from pickle import Pickler, Unpickler
|
||||
|
||||
|
||||
class MyPickler(Pickler):
|
||||
""" Pickler with a custom memoize()
|
||||
to take care of unique ID creation.
|
||||
See the usage in ImmutablePickler
|
||||
"""
|
||||
def __init__(self, immo, file, protocol, uneven):
|
||||
Pickler.__init__(self, file, protocol)
|
||||
self.uneven = uneven
|
||||
self._unpicklememo = immo._unpicklememo
|
||||
self.memo = immo._picklememo
|
||||
|
||||
def memoize(self, obj):
|
||||
if self.fast:
|
||||
return
|
||||
assert id(obj) not in self.memo
|
||||
memo_len = len(self.memo)
|
||||
key = memo_len * 2 + self.uneven
|
||||
self.write(self.put(key))
|
||||
self.memo[id(obj)] = key, obj
|
||||
key = makekey(key)
|
||||
if key in self._unpicklememo:
|
||||
assert self._unpicklememo[key] is obj
|
||||
dict.__setitem__(self._unpicklememo, key, obj)
|
||||
|
||||
#if sys.version_info < (3,0):
|
||||
# def save_string(self, obj, pack=struct.pack):
|
||||
# obj = unicode(obj)
|
||||
# self.save_unicode(obj, pack=pack)
|
||||
# Pickler.dispatch[str] = save_string
|
||||
|
||||
class UnpicklingDict(dict):
|
||||
def __init__(self, picklememo):
|
||||
super(UnpicklingDict, self).__init__()
|
||||
self._picklememo = picklememo
|
||||
|
||||
def __setitem__(self, key, obj):
|
||||
super(UnpicklingDict, self).__setitem__(key, obj)
|
||||
self._picklememo[id(obj)] = (fromkey(key), obj)
|
||||
|
||||
class ImmutablePickler:
|
||||
def __init__(self, uneven, protocol=0):
|
||||
""" ImmutablePicklers are instantiated in Pairs.
|
||||
The two sides need to create unique IDs
|
||||
while pickling their objects. This is
|
||||
done by using either even or uneven
|
||||
numbers, depending on the instantiation
|
||||
parameter.
|
||||
"""
|
||||
self._picklememo = {}
|
||||
self._unpicklememo = UnpicklingDict(self._picklememo)
|
||||
self._protocol = protocol
|
||||
self.uneven = uneven and 1 or 0
|
||||
|
||||
def selfmemoize(self, obj):
|
||||
# this is for feeding objects to ourselfes
|
||||
# which be the case e.g. if you want to pickle
|
||||
# from a forked process back to the original
|
||||
f = py.io.BytesIO()
|
||||
pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
|
||||
pickler.memoize(obj)
|
||||
|
||||
def dumps(self, obj):
|
||||
f = py.io.BytesIO()
|
||||
pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
|
||||
pickler.dump(obj)
|
||||
#print >>debug, "dumped", obj
|
||||
#print >>debug, "picklememo", self._picklememo
|
||||
return f.getvalue()
|
||||
|
||||
def loads(self, string):
|
||||
f = py.io.BytesIO(string)
|
||||
unpickler = Unpickler(f)
|
||||
unpickler.memo = self._unpicklememo
|
||||
res = unpickler.load()
|
||||
#print >>debug, "loaded", res
|
||||
#print >>debug, "unpicklememo", self._unpicklememo
|
||||
return res
|
||||
|
||||
|
||||
NO_ENDMARKER_WANTED = object()
|
||||
|
||||
class UnpickleError(Exception):
|
||||
""" Problems while unpickling. """
|
||||
def __init__(self, formatted):
|
||||
self.formatted = formatted
|
||||
Exception.__init__(self, formatted)
|
||||
def __str__(self):
|
||||
return self.formatted
|
||||
|
||||
class PickleChannel(object):
|
||||
""" PickleChannels wrap execnet channels
|
||||
and allow to send/receive by using
|
||||
"immutable pickling".
|
||||
"""
|
||||
_unpicklingerror = None
|
||||
def __init__(self, channel):
|
||||
self._channel = channel
|
||||
# we use the fact that each side of a
|
||||
# gateway connection counts with uneven
|
||||
# or even numbers depending on which
|
||||
# side it is (for the purpose of creating
|
||||
# unique ids - which is what we need it here for)
|
||||
uneven = channel.gateway._channelfactory.count % 2
|
||||
self._ipickle = ImmutablePickler(uneven=uneven)
|
||||
self.RemoteError = channel.RemoteError
|
||||
|
||||
def send(self, obj):
|
||||
pickled_obj = self._ipickle.dumps(obj)
|
||||
self._channel.send(pickled_obj)
|
||||
|
||||
def receive(self):
|
||||
pickled_obj = self._channel.receive()
|
||||
return self._unpickle(pickled_obj)
|
||||
|
||||
def _unpickle(self, pickled_obj):
|
||||
if isinstance(pickled_obj, self._channel.__class__):
|
||||
return pickled_obj
|
||||
return self._ipickle.loads(pickled_obj)
|
||||
|
||||
def _getremoteerror(self):
|
||||
return self._unpicklingerror or self._channel._getremoteerror()
|
||||
|
||||
def close(self):
|
||||
return self._channel.close()
|
||||
|
||||
def isclosed(self):
|
||||
return self._channel.isclosed()
|
||||
|
||||
def waitclose(self, timeout=None):
|
||||
return self._channel.waitclose(timeout=timeout)
|
||||
|
||||
def setcallback(self, callback, endmarker=NO_ENDMARKER_WANTED):
|
||||
if endmarker is NO_ENDMARKER_WANTED:
|
||||
def unpickle_callback(pickled_obj):
|
||||
obj = self._unpickle(pickled_obj)
|
||||
callback(obj)
|
||||
self._channel.setcallback(unpickle_callback)
|
||||
return
|
||||
uniqueendmarker = object()
|
||||
def unpickle_callback(pickled_obj):
|
||||
if pickled_obj is uniqueendmarker:
|
||||
return callback(endmarker)
|
||||
try:
|
||||
obj = self._unpickle(pickled_obj)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
formatted = str(excinfo.getrepr(showlocals=True,funcargs=True))
|
||||
self._unpicklingerror = UnpickleError(formatted)
|
||||
callback(endmarker)
|
||||
else:
|
||||
callback(obj)
|
||||
self._channel.setcallback(unpickle_callback, uniqueendmarker)
|
||||
@@ -16,8 +16,3 @@ def pytest_testnodeready(node):
|
||||
|
||||
def pytest_testnodedown(node, error):
|
||||
""" Test Node is down. """
|
||||
|
||||
def pytest_rescheduleitems(items):
|
||||
""" reschedule Items from a node that went down. """
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import py
|
||||
import sys, os
|
||||
import xdist
|
||||
from xdist.txnode import TXNode
|
||||
from xdist.remote import SlaveController
|
||||
|
||||
from xdist.gwmanage import GatewayManager
|
||||
import execnet
|
||||
|
||||
@@ -48,18 +49,19 @@ class NodeManager(object):
|
||||
# pick it up as the right topdir
|
||||
# (for other gateways this chdir is irrelevant)
|
||||
self.trace("making gateways")
|
||||
old = self.config.topdir.chdir()
|
||||
try:
|
||||
self.gwmanager.makegateways()
|
||||
finally:
|
||||
old.chdir()
|
||||
#old = self.config.topdir.chdir()
|
||||
#try:
|
||||
self.gwmanager.makegateways()
|
||||
#finally:
|
||||
# old.chdir()
|
||||
|
||||
def setup_nodes(self, putevent):
|
||||
self.rsync_roots()
|
||||
self.trace("setting up nodes")
|
||||
for gateway in self.gwmanager.group:
|
||||
node = TXNode(self, gateway, self.config, putevent)
|
||||
node = SlaveController(self, gateway, self.config, putevent)
|
||||
gateway.node = node # to keep node alive
|
||||
node.setup()
|
||||
self.trace("started node %r" % node)
|
||||
|
||||
def teardown_nodes(self):
|
||||
|
||||
116
xdist/plugin.py
116
xdist/plugin.py
@@ -183,7 +183,18 @@ def pytest_addhooks(pluginmanager):
|
||||
# -------------------------------------------------------------------------
|
||||
# distributed testing initialization
|
||||
# -------------------------------------------------------------------------
|
||||
def pytest_configure(config):
|
||||
|
||||
def pytest_cmdline_main(config):
|
||||
check_options(config)
|
||||
if config.getvalue("looponfail"):
|
||||
from xdist.looponfail import looponfail_main
|
||||
looponfail_main(config)
|
||||
return 2 # looponfail only can get stop with ctrl-C anyway
|
||||
elif config.getvalue("dist") != "no":
|
||||
from xdist.dsession import dsession_main
|
||||
return dsession_main(config)
|
||||
|
||||
def check_options(config):
|
||||
if config.option.numprocesses:
|
||||
config.option.dist = "load"
|
||||
config.option.tx = ['popen'] * int(config.option.numprocesses)
|
||||
@@ -199,32 +210,6 @@ def pytest_configure(config):
|
||||
if usepdb:
|
||||
raise config.Error("--pdb incompatible with distributing tests.")
|
||||
|
||||
def pytest_cmdline_main(config):
|
||||
if config.getvalue("looponfail"):
|
||||
from xdist.looponfail import looponfail_main
|
||||
looponfail_main(config)
|
||||
return 2 # looponfail only can get stop with ctrl-C anyway
|
||||
elif config.getvalue("dist"):
|
||||
pass
|
||||
return
|
||||
from py._test.session import Session, Collection
|
||||
collection = Collection(config)
|
||||
# instantiate session already because it
|
||||
# records failures and implements maxfail handling
|
||||
session = Session(config, collection)
|
||||
exitstatus = collection.do_collection()
|
||||
if not exitstatus:
|
||||
exitstatus = session.main()
|
||||
return exitstatus
|
||||
|
||||
def pytest_sessionstart(session):
|
||||
config = session.config
|
||||
if hasattr(config, '_isdistsession'):
|
||||
if not config.pluginmanager.hasplugin("terminal") or \
|
||||
not config.pluginmanager.hasplugin("terminalreporter"):
|
||||
return
|
||||
trdist = TerminalDistReporter(config)
|
||||
config.pluginmanager.register(trdist, "terminaldistreporter")
|
||||
|
||||
def pytest_runtest_protocol(item):
|
||||
if item.config.getvalue("boxed"):
|
||||
@@ -238,23 +223,20 @@ def forked_run_report(item):
|
||||
# XXX optionally allow sharing of setup/teardown
|
||||
from py._plugin.pytest_runner import runtestprotocol
|
||||
EXITSTATUS_TESTEXIT = 4
|
||||
from xdist.mypickle import ImmutablePickler
|
||||
ipickle = ImmutablePickler(uneven=0)
|
||||
ipickle.selfmemoize(item.config)
|
||||
# XXX workaround the issue that 2.6 cannot pickle
|
||||
# instances of classes defined in global conftest.py files
|
||||
ipickle.selfmemoize(item)
|
||||
import marshal
|
||||
from xdist.remote import serialize_report, unserialize_report
|
||||
def runforked():
|
||||
try:
|
||||
reports = runtestprotocol(item, log=False)
|
||||
except KeyboardInterrupt:
|
||||
py.std.os._exit(EXITSTATUS_TESTEXIT)
|
||||
return ipickle.dumps(reports)
|
||||
return marshal.dumps([serialize_report(x) for x in reports])
|
||||
|
||||
ff = py.process.ForkedFunc(runforked)
|
||||
result = ff.waitfinish()
|
||||
if result.retval is not None:
|
||||
return ipickle.loads(result.retval)
|
||||
report_dumps = marshal.loads(result.retval)
|
||||
return [unserialize_report(x) for x in report_dumps]
|
||||
else:
|
||||
if result.exitstatus == EXITSTATUS_TESTEXIT:
|
||||
py.test.exit("forked test item %s raised Exit" %(item,))
|
||||
@@ -264,62 +246,8 @@ def report_process_crash(item, result):
|
||||
path, lineno = item._getfslineno()
|
||||
info = "%s:%s: running the test CRASHED with signal %d" %(
|
||||
path, lineno, result.signal)
|
||||
from py._plugin.pytest_runner import ItemTestReport
|
||||
return ItemTestReport(item, excinfo=info, when="???")
|
||||
|
||||
class TerminalDistReporter:
|
||||
def __init__(self, config):
|
||||
self.gateway2info = {}
|
||||
self.config = config
|
||||
self.tplugin = config.pluginmanager.getplugin("terminal")
|
||||
self.tr = config.pluginmanager.getplugin("terminalreporter")
|
||||
|
||||
def write_line(self, msg):
|
||||
self.tr.write_line(msg)
|
||||
|
||||
def pytest_itemstart(self, __multicall__):
|
||||
try:
|
||||
__multicall__.methods.remove(self.tr.pytest_itemstart)
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if hasattr(report, 'node'):
|
||||
report.headerlines.append(self.gateway2info.get(
|
||||
report.node.gateway,
|
||||
"node %r (platinfo not found? strange)"))
|
||||
|
||||
def pytest_gwmanage_newgateway(self, gateway, platinfo):
|
||||
#self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec))
|
||||
d = {}
|
||||
d['version'] = self.tplugin.repr_pythonversion(platinfo.version_info)
|
||||
d['id'] = gateway.id
|
||||
d['spec'] = gateway.spec._spec
|
||||
d['platform'] = platinfo.platform
|
||||
if self.config.option.verbose:
|
||||
d['extra'] = "- " + platinfo.executable
|
||||
else:
|
||||
d['extra'] = ""
|
||||
d['cwd'] = platinfo.cwd
|
||||
infoline = ("[%(id)s] %(spec)s -- platform %(platform)s, "
|
||||
"Python %(version)s "
|
||||
"cwd: %(cwd)s"
|
||||
"%(extra)s" % d)
|
||||
if self.config.getvalue("verbose"):
|
||||
self.write_line(infoline)
|
||||
self.gateway2info[gateway] = infoline
|
||||
|
||||
def pytest_testnodeready(self, node):
|
||||
if self.config.getvalue("verbose"):
|
||||
self.write_line(
|
||||
"[%s] txnode ready to receive tests" %(node.gateway.id,))
|
||||
|
||||
def pytest_testnodedown(self, node, error):
|
||||
if not error:
|
||||
return
|
||||
self.write_line("[%s] node down, error: %s" %(node.gateway.id, error))
|
||||
|
||||
def pytest_rescheduleitems(self, items):
|
||||
if self.config.option.debug:
|
||||
self.write_sep("!", "RESCHEDULING %s " %(items,))
|
||||
|
||||
from py._plugin import pytest_runner as runner
|
||||
call = runner.CallInfo(lambda: 0/0, "???")
|
||||
call.excinfo = info
|
||||
rep = runner.pytest_runtest_makereport(item, call)
|
||||
return rep
|
||||
|
||||
213
xdist/remote.py
Normal file
213
xdist/remote.py
Normal file
@@ -0,0 +1,213 @@
|
||||
"""
|
||||
Implement --dist=* testing
|
||||
"""
|
||||
|
||||
import py
|
||||
import sys
|
||||
import execnet
|
||||
import kwlog
|
||||
from py._plugin import pytest_runner as runner # XXX load dynamically
|
||||
|
||||
class SlaveController(object):
|
||||
ENDMARK = -1
|
||||
|
||||
def __init__(self, nodemanager, gateway, config, putevent):
|
||||
#self.nodemanager = nodemanager
|
||||
self.putevent = putevent
|
||||
self.gateway = gateway
|
||||
self.config = config
|
||||
self.status = None
|
||||
self._down = False
|
||||
self.status = "gateway-init"
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s id=%s status=%s>" %(self.__class__.__name__,
|
||||
self.gateway.id, self.status)
|
||||
|
||||
def trace(self, *args):
|
||||
if self.config.option.debug:
|
||||
msg = " ".join([str(x) for x in args])
|
||||
py.builtin.print_("SlaveController:", msg)
|
||||
|
||||
def setup(self):
|
||||
self.trace("setting up slave session")
|
||||
assert self.status == "gateway-init"
|
||||
self.channel = self.gateway.remote_exec(init_slave_session,
|
||||
args=self.config.args,
|
||||
option_dict=vars(self.config.option),
|
||||
)
|
||||
self.status = "slave-init"
|
||||
if self.putevent:
|
||||
self.channel.setcallback(self.process_from_remote,
|
||||
endmarker=self.ENDMARK)
|
||||
|
||||
def ensure_teardown(self):
|
||||
if hasattr(self, 'channel'):
|
||||
if not self.channel.isclosed():
|
||||
self.trace("closing", self.channel)
|
||||
self.channel.close()
|
||||
#del self.channel
|
||||
if hasattr(self, 'gateway'):
|
||||
self.trace("exiting", self.gateway)
|
||||
self.gateway.exit()
|
||||
#del self.gateway
|
||||
|
||||
def send_runtest(self, nodeid):
|
||||
self.sendcommand("runtests", ids=[nodeid])
|
||||
|
||||
def shutdown(self):
|
||||
if not self._down and not self.channel.isclosed():
|
||||
self.sendcommand("shutdown")
|
||||
|
||||
def sendcommand(self, name, **kwargs):
|
||||
""" send a named parametrized command to the other side. """
|
||||
self.trace("sending command %s(**%s)" % (name, kwargs))
|
||||
self.channel.send((name, kwargs))
|
||||
|
||||
def notify_inproc(self, eventname, **kwargs):
|
||||
self.trace("queuing %s(**%s)" % (eventname, kwargs))
|
||||
self.putevent((eventname, kwargs))
|
||||
|
||||
def process_from_remote(self, eventcall):
|
||||
""" this gets called for each object we receive from
|
||||
the other side and if the channel closes.
|
||||
|
||||
Note that channel callbacks run in the receiver
|
||||
thread of execnet gateways - we need to
|
||||
avoid raising exceptions or doing heavy work.
|
||||
"""
|
||||
try:
|
||||
if eventcall == self.ENDMARK:
|
||||
err = self.channel._getremoteerror()
|
||||
if not self._down:
|
||||
if not err or isinstance(err, EOFError):
|
||||
err = "Not properly terminated" # lost connection?
|
||||
self.notify_inproc("errordown", node=self, error=err)
|
||||
self._down = True
|
||||
return
|
||||
eventname, kwargs = eventcall
|
||||
if eventname in ("collectionstart"):
|
||||
self.trace("ignoring %s(%s)" %(eventname, kwargs))
|
||||
elif eventname == "slaveready":
|
||||
self.notify_inproc(eventname, node=self)
|
||||
elif eventname == "slavefinished":
|
||||
self._down = True
|
||||
self.slaveoutput = kwargs['slaveoutput']
|
||||
self.notify_inproc("slavefinished", node=self)
|
||||
#elif eventname == "logstart":
|
||||
# self.notify_inproc(eventname, node=self, **kwargs)
|
||||
elif eventname in ("testreport", "collectreport"):
|
||||
rep = unserialize_report(kwargs['data'])
|
||||
self.notify_inproc(eventname, node=self, rep=rep)
|
||||
elif eventname == "collectionfinish":
|
||||
self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
|
||||
else:
|
||||
raise ValueError("unknown event: %s" %(eventname,))
|
||||
except KeyboardInterrupt:
|
||||
# should not land in receiver-thread
|
||||
raise
|
||||
except:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
py.builtin.print_("!" * 20, excinfo)
|
||||
self.config.pluginmanager.notify_exception(excinfo)
|
||||
|
||||
def init_slave_session(channel, args, option_dict):
|
||||
import py
|
||||
#outchannel = channel.gateway.newchannel()
|
||||
#sys.stdout = sys.stderr = outchannel.makefile('w')
|
||||
#channel.send(outchannel)
|
||||
#fullwidth, hasmarkup = channel.receive()
|
||||
from xdist.remote import remote_initconfig, SlaveInteractor
|
||||
config = remote_initconfig(py.test.config, option_dict, args)
|
||||
interactor = SlaveInteractor(config, channel)
|
||||
config.hook.pytest_cmdline_main(config=config)
|
||||
|
||||
def remote_initconfig(config, option_dict, args):
|
||||
config._preparse(args)
|
||||
config.option.__dict__.update(option_dict)
|
||||
config.option.looponfail = False
|
||||
config.option.usepdb = False
|
||||
config.option.dist = "no"
|
||||
config.option.distload = False
|
||||
config.option.numprocesses = None
|
||||
#kwlog.Producer("slave").DEBUG("option dict", config.option.__dict__)
|
||||
config.args = args
|
||||
return config
|
||||
|
||||
class SlaveInteractor:
|
||||
def __init__(self, config, channel):
|
||||
self.config = config
|
||||
self.log = kwlog.Producer("slave")
|
||||
kwlog.setconsumer(self.log, None)
|
||||
self.log.info("initializing SlaveInteractor")
|
||||
self.channel = channel
|
||||
config.pluginmanager.register(self)
|
||||
|
||||
def sendevent(self, name, **kwargs):
|
||||
self.log.debug("sending", name, kwargs)
|
||||
self.channel.send((name, kwargs))
|
||||
|
||||
def pytest_internalerror(self, excrepr):
|
||||
for line in str(excrepr).split("\n"):
|
||||
self.log.debug("IERROR> " + line)
|
||||
|
||||
def pytest_sessionstart(self, session):
|
||||
self.session = session
|
||||
self.collection = session.collection
|
||||
self.sendevent("slaveready")
|
||||
|
||||
def pytest_sessionfinish(self):
|
||||
self.sendevent("slavefinished", slaveoutput={})
|
||||
|
||||
def pytest_perform_collection(self, session):
|
||||
self.sendevent("collectionstart")
|
||||
|
||||
def pytest_runtest_mainloop(self, session):
|
||||
self.log.debug("entering main loop")
|
||||
while 1:
|
||||
name, kwargs = self.channel.receive()
|
||||
self.log.debug("received command %s(**%s)" % (name, kwargs))
|
||||
if name == "runtests":
|
||||
ids = kwargs['ids']
|
||||
for nodeid in ids:
|
||||
for item in self.collection.getbyid(nodeid):
|
||||
self.config.hook.pytest_runtest_protocol(item=item)
|
||||
elif name == "shutdown":
|
||||
break
|
||||
return True
|
||||
|
||||
def pytest_log_finishcollection(self, collection):
|
||||
self.log.debug("pytest_log_finishcollection")
|
||||
ids = [collection.getid(item) for item in collection.items]
|
||||
self.sendevent("collectionfinish",
|
||||
topdir=str(collection.topdir),
|
||||
ids=ids)
|
||||
|
||||
#def pytest_runtest_logstart(self, nodeid, location):
|
||||
# self.sendevent("logstart", nodeid=nodeid, location=location)
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
data = serialize_report(report)
|
||||
self.sendevent("testreport", data=data)
|
||||
|
||||
def pytest_collectreport(self, report):
|
||||
data = serialize_report(report)
|
||||
self.sendevent("collectreport", data=data)
|
||||
|
||||
def serialize_report(rep):
|
||||
d = rep.__dict__.copy()
|
||||
d['longrepr'] = rep.longrepr and str(rep.longrepr) or None
|
||||
for name in d:
|
||||
if isinstance(d[name], py.path.local):
|
||||
d[name] = str(d[name])
|
||||
elif name == "result":
|
||||
d[name] = None # for now
|
||||
return d
|
||||
|
||||
def unserialize_report(reportdict):
|
||||
d = reportdict
|
||||
if 'result' in d:
|
||||
return runner.CollectReport(**d)
|
||||
else:
|
||||
return runner.TestReport(**d)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Manage setup, running and local representation of remote nodes/processes.
|
||||
"""
|
||||
import py
|
||||
from xdist.mypickle import PickleChannel
|
||||
from py._test.session import Session
|
||||
|
||||
class TXNode(object):
|
||||
@@ -19,7 +18,7 @@ class TXNode(object):
|
||||
self.putevent = putevent
|
||||
self.gateway = gateway
|
||||
self.slaveinput = {}
|
||||
self.channel = install_slave(self)
|
||||
self.channel = self.setup()
|
||||
self.channel.setcallback(self.callback, endmarker=self.ENDMARK)
|
||||
self._down = False
|
||||
|
||||
@@ -84,44 +83,47 @@ class TXNode(object):
|
||||
else:
|
||||
self.channel.send(None)
|
||||
|
||||
# configuring and setting up slave node
|
||||
def install_slave(node):
|
||||
channel = node.gateway.remote_exec(source="""
|
||||
import os, sys
|
||||
sys.path.insert(0, os.getcwd())
|
||||
from xdist.mypickle import PickleChannel
|
||||
from xdist.txnode import SlaveSession
|
||||
channel.send("basicimport")
|
||||
channel = PickleChannel(channel)
|
||||
import py
|
||||
config, slaveinput, basetemp, nodeid = channel.receive()
|
||||
config.slaveinput = slaveinput
|
||||
config.slaveoutput = {}
|
||||
if basetemp:
|
||||
config.basetemp = py.path.local(basetemp)
|
||||
config.nodeid = nodeid
|
||||
config.pluginmanager.do_configure(config)
|
||||
session = SlaveSession(config, channel, nodeid)
|
||||
session.dist_main()
|
||||
""")
|
||||
channel.receive()
|
||||
channel = PickleChannel(channel)
|
||||
basetemp = None
|
||||
config = node.config
|
||||
config.hook.pytest_configure_node(node=node)
|
||||
if node.gateway.spec.popen:
|
||||
popenbase = config.ensuretemp("popen")
|
||||
basetemp = py.path.local.make_numbered_dir(prefix="slave-",
|
||||
keep=0, rootdir=popenbase)
|
||||
basetemp = str(basetemp)
|
||||
channel.send((config, node.slaveinput, basetemp, node.gateway.id))
|
||||
return channel
|
||||
# configuring and setting up slave node
|
||||
def setup(self):
|
||||
basetemp = None
|
||||
config = self.config
|
||||
config.hook.pytest_configure_node(node=self)
|
||||
if self.gateway.spec.popen:
|
||||
popenbase = config.ensuretemp("popen")
|
||||
basetemp = py.path.local.make_numbered_dir(prefix="slave-",
|
||||
keep=0, rootdir=popenbase)
|
||||
basetemp = str(basetemp)
|
||||
return self.gateway.remote_exec(init_slave_session,
|
||||
args=self.config.args,
|
||||
option_dict=vars(self.config.option),
|
||||
slaveinput={}, # XXX,
|
||||
basetemp=basetemp,
|
||||
nodeid=self.gateway.id,
|
||||
)
|
||||
|
||||
class SlaveSession(Session):
|
||||
def __init__(self, config, channel, nodeid):
|
||||
def init_slave_session(channel, args, option_dict,
|
||||
slaveinput, basetemp, nodeid):
|
||||
import os, sys
|
||||
#sys.path.insert(0, os.getcwd())
|
||||
from xdist.txnode import SlaveSession
|
||||
import py
|
||||
config = py.test.config
|
||||
config.option.__dict__.update(option_dict)
|
||||
config._preparse(args)
|
||||
config.args = args
|
||||
config.slaveinput = slaveinput
|
||||
config.slaveoutput = {}
|
||||
if basetemp:
|
||||
config.basetemp = py.path.local(basetemp)
|
||||
config.nodeid = nodeid
|
||||
return SlaveSession(config, channel).dist_main()
|
||||
|
||||
class SlaveSession:
|
||||
def __init__(self, config, channel):
|
||||
self.channel = channel
|
||||
self.nodeid = nodeid
|
||||
super(SlaveSession, self).__init__(config=config)
|
||||
self.config = config
|
||||
self.runner = self.config.pluginmanager.getplugin("pytest_runner")
|
||||
config.pluginmanager.register(self, "slavesession")
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s channel=%s>" %(self.__class__.__name__, self.channel)
|
||||
@@ -143,7 +145,6 @@ class SlaveSession(Session):
|
||||
self.sendevent("pytest_internalerror", excrepr=excrepr)
|
||||
|
||||
def dist_main(self):
|
||||
self.runner = self.config.pluginmanager.getplugin("pytest_runner")
|
||||
self.sendevent("slaveready")
|
||||
self.main(None)
|
||||
error = getattr(self, '_slaveerror', None)
|
||||
|
||||
Reference in New Issue
Block a user