initial copy and modifications from the original py/impl/test/dist and
looponfail code. Now works as its own "xdist" plugin.
This commit is contained in:
holger krekel
1
xdist/__init__.py
Normal file
1
xdist/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
#
|
||||
271
xdist/dsession.py
Normal file
271
xdist/dsession.py
Normal file
@@ -0,0 +1,271 @@
|
||||
import py
|
||||
from py.impl.test.session import Session
|
||||
from py.impl.test import outcome
|
||||
from xdist.nodemanage import NodeManager
|
||||
queue = py.builtin._tryimport('queue', 'Queue')
|
||||
|
||||
debug_file = None # open('/tmp/loop.log', 'w')
|
||||
def debug(*args):
|
||||
if debug_file is not None:
|
||||
s = " ".join(map(str, args))
|
||||
debug_file.write(s+"\n")
|
||||
debug_file.flush()
|
||||
|
||||
class LoopState(object):
|
||||
def __init__(self, dsession, colitems):
|
||||
self.dsession = dsession
|
||||
self.colitems = colitems
|
||||
self.exitstatus = None
|
||||
# loopstate.dowork is False after reschedule events
|
||||
# because otherwise we might very busily loop
|
||||
# waiting for a host to become ready.
|
||||
self.dowork = True
|
||||
self.shuttingdown = False
|
||||
self.testsfailed = False
|
||||
|
||||
def __repr__(self):
|
||||
return "<LoopState exitstatus=%r shuttingdown=%r len(colitems)=%d>" % (
|
||||
self.exitstatus, self.shuttingdown, len(self.colitems))
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if report.item in self.dsession.item2nodes:
|
||||
if report.when != "teardown": # otherwise we already managed it
|
||||
self.dsession.removeitem(report.item, report.node)
|
||||
if report.failed:
|
||||
self.testsfailed = True
|
||||
|
||||
def pytest_collectreport(self, report):
|
||||
if report.passed:
|
||||
self.colitems.extend(report.result)
|
||||
|
||||
def pytest_testnodeready(self, node):
|
||||
self.dsession.addnode(node)
|
||||
|
||||
def pytest_testnodedown(self, node, error=None):
|
||||
pending = self.dsession.removenode(node)
|
||||
if pending:
|
||||
if error:
|
||||
crashitem = pending[0]
|
||||
debug("determined crashitem", crashitem)
|
||||
self.dsession.handle_crashitem(crashitem, node)
|
||||
# XXX recovery handling for "each"?
|
||||
# currently pending items are not retried
|
||||
if self.dsession.config.option.dist == "load":
|
||||
self.colitems.extend(pending[1:])
|
||||
|
||||
def pytest_rescheduleitems(self, items):
|
||||
self.colitems.extend(items)
|
||||
self.dowork = False # avoid busywait
|
||||
|
||||
class DSession(Session):
|
||||
"""
|
||||
Session drives the collection and running of tests
|
||||
and generates test events for reporters.
|
||||
"""
|
||||
MAXITEMSPERHOST = 15
|
||||
|
||||
def __init__(self, config):
|
||||
self.queue = queue.Queue()
|
||||
self.node2pending = {}
|
||||
self.item2nodes = {}
|
||||
super(DSession, self).__init__(config=config)
|
||||
|
||||
def main(self, colitems):
|
||||
self.sessionstarts()
|
||||
self.setup()
|
||||
exitstatus = self.loop(colitems)
|
||||
self.teardown()
|
||||
self.sessionfinishes(exitstatus=exitstatus)
|
||||
return exitstatus
|
||||
|
||||
def loop_once(self, loopstate):
|
||||
if loopstate.shuttingdown:
|
||||
return self.loop_once_shutdown(loopstate)
|
||||
colitems = loopstate.colitems
|
||||
if loopstate.dowork and colitems:
|
||||
self.triggertesting(loopstate.colitems)
|
||||
colitems[:] = []
|
||||
# we use a timeout here so that control-C gets through
|
||||
while 1:
|
||||
try:
|
||||
eventcall = self.queue.get(timeout=2.0)
|
||||
break
|
||||
except queue.Empty:
|
||||
continue
|
||||
loopstate.dowork = True
|
||||
|
||||
callname, args, kwargs = eventcall
|
||||
if callname is not None:
|
||||
call = getattr(self.config.hook, callname)
|
||||
assert not args
|
||||
call(**kwargs)
|
||||
|
||||
# termination conditions
|
||||
if ((loopstate.testsfailed and self.config.option.exitfirst) or
|
||||
(not self.item2nodes and not colitems and not self.queue.qsize())):
|
||||
self.triggershutdown()
|
||||
loopstate.shuttingdown = True
|
||||
elif not self.node2pending:
|
||||
loopstate.exitstatus = outcome.EXIT_NOHOSTS
|
||||
|
||||
def loop_once_shutdown(self, loopstate):
|
||||
# once we are in shutdown mode we dont send
|
||||
# events other than HostDown upstream
|
||||
eventname, args, kwargs = self.queue.get()
|
||||
if eventname == "pytest_testnodedown":
|
||||
self.config.hook.pytest_testnodedown(**kwargs)
|
||||
self.removenode(kwargs['node'])
|
||||
elif eventname == "pytest_runtest_logreport":
|
||||
# might be some teardown report
|
||||
self.config.hook.pytest_runtest_logreport(**kwargs)
|
||||
elif eventname == "pytest_internalerror":
|
||||
self.config.hook.pytest_internalerror(**kwargs)
|
||||
loopstate.exitstatus = outcome.EXIT_INTERNALERROR
|
||||
elif eventname == "pytest__teardown_final_logerror":
|
||||
self.config.hook.pytest__teardown_final_logerror(**kwargs)
|
||||
loopstate.exitstatus = outcome.EXIT_TESTSFAILED
|
||||
if not self.node2pending:
|
||||
# finished
|
||||
if loopstate.testsfailed:
|
||||
loopstate.exitstatus = outcome.EXIT_TESTSFAILED
|
||||
else:
|
||||
loopstate.exitstatus = outcome.EXIT_OK
|
||||
#self.config.pluginmanager.unregister(loopstate)
|
||||
|
||||
def _initloopstate(self, colitems):
|
||||
loopstate = LoopState(self, colitems)
|
||||
self.config.pluginmanager.register(loopstate)
|
||||
return loopstate
|
||||
|
||||
def loop(self, colitems):
|
||||
try:
|
||||
loopstate = self._initloopstate(colitems)
|
||||
loopstate.dowork = False # first receive at least one HostUp events
|
||||
while 1:
|
||||
self.loop_once(loopstate)
|
||||
if loopstate.exitstatus is not None:
|
||||
exitstatus = loopstate.exitstatus
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
|
||||
exitstatus = outcome.EXIT_INTERRUPTED
|
||||
except:
|
||||
self.config.pluginmanager.notify_exception()
|
||||
exitstatus = outcome.EXIT_INTERNALERROR
|
||||
self.config.pluginmanager.unregister(loopstate)
|
||||
if exitstatus == 0 and self._testsfailed:
|
||||
exitstatus = outcome.EXIT_TESTSFAILED
|
||||
return exitstatus
|
||||
|
||||
def triggershutdown(self):
|
||||
for node in self.node2pending:
|
||||
node.shutdown()
|
||||
|
||||
def addnode(self, node):
|
||||
assert node not in self.node2pending
|
||||
self.node2pending[node] = []
|
||||
|
||||
def removenode(self, node):
|
||||
try:
|
||||
pending = self.node2pending.pop(node)
|
||||
except KeyError:
|
||||
# this happens if we didn't receive a testnodeready event yet
|
||||
return []
|
||||
for item in pending:
|
||||
l = self.item2nodes[item]
|
||||
l.remove(node)
|
||||
if not l:
|
||||
del self.item2nodes[item]
|
||||
return pending
|
||||
|
||||
def triggertesting(self, colitems):
|
||||
colitems = self.filteritems(colitems)
|
||||
senditems = []
|
||||
for next in colitems:
|
||||
if isinstance(next, py.test.collect.Item):
|
||||
senditems.append(next)
|
||||
else:
|
||||
self.config.hook.pytest_collectstart(collector=next)
|
||||
colrep = self.config.hook.pytest_make_collect_report(collector=next)
|
||||
self.queueevent("pytest_collectreport", report=colrep)
|
||||
if self.config.option.dist == "each":
|
||||
self.senditems_each(senditems)
|
||||
else:
|
||||
# XXX assert self.config.option.dist == "load"
|
||||
self.senditems_load(senditems)
|
||||
|
||||
def queueevent(self, eventname, **kwargs):
|
||||
self.queue.put((eventname, (), kwargs))
|
||||
|
||||
def senditems_each(self, tosend):
|
||||
if not tosend:
|
||||
return
|
||||
room = self.MAXITEMSPERHOST
|
||||
for node, pending in self.node2pending.items():
|
||||
room = min(self.MAXITEMSPERHOST - len(pending), room)
|
||||
sending = tosend[:room]
|
||||
if sending:
|
||||
for node, pending in self.node2pending.items():
|
||||
node.sendlist(sending)
|
||||
pending.extend(sending)
|
||||
for item in sending:
|
||||
nodes = self.item2nodes.setdefault(item, [])
|
||||
assert node not in nodes
|
||||
nodes.append(node)
|
||||
item.ihook.pytest_itemstart(item=item, node=node)
|
||||
tosend[:] = tosend[room:] # update inplace
|
||||
if tosend:
|
||||
# we have some left, give it to the main loop
|
||||
self.queueevent("pytest_rescheduleitems", items=tosend)
|
||||
|
||||
def senditems_load(self, tosend):
|
||||
if not tosend:
|
||||
return
|
||||
for node, pending in self.node2pending.items():
|
||||
room = self.MAXITEMSPERHOST - len(pending)
|
||||
if room > 0:
|
||||
sending = tosend[:room]
|
||||
node.sendlist(sending)
|
||||
for item in sending:
|
||||
#assert item not in self.item2node, (
|
||||
# "sending same item %r to multiple "
|
||||
# "not implemented" %(item,))
|
||||
self.item2nodes.setdefault(item, []).append(node)
|
||||
item.ihook.pytest_itemstart(item=item, node=node)
|
||||
pending.extend(sending)
|
||||
tosend[:] = tosend[room:] # update inplace
|
||||
if not tosend:
|
||||
break
|
||||
if tosend:
|
||||
# we have some left, give it to the main loop
|
||||
self.queueevent("pytest_rescheduleitems", items=tosend)
|
||||
|
||||
def removeitem(self, item, node):
|
||||
if item not in self.item2nodes:
|
||||
raise AssertionError(item, self.item2nodes)
|
||||
nodes = self.item2nodes[item]
|
||||
if node in nodes: # the node might have gone down already
|
||||
nodes.remove(node)
|
||||
if not nodes:
|
||||
del self.item2nodes[item]
|
||||
pending = self.node2pending[node]
|
||||
pending.remove(item)
|
||||
|
||||
def handle_crashitem(self, item, node):
|
||||
runner = item.config.pluginmanager.getplugin("runner")
|
||||
info = "!!! Node %r crashed during running of test %r" %(node, item)
|
||||
rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
|
||||
rep.node = node
|
||||
item.ihook.pytest_runtest_logreport(report=rep)
|
||||
|
||||
def setup(self):
|
||||
""" setup any neccessary resources ahead of the test run. """
|
||||
self.nodemanager = NodeManager(self.config)
|
||||
self.nodemanager.setup_nodes(putevent=self.queue.put)
|
||||
if self.config.option.dist == "each":
|
||||
self.nodemanager.wait_nodesready(5.0)
|
||||
|
||||
def teardown(self):
|
||||
""" teardown any resources after a test run. """
|
||||
self.nodemanager.teardown_nodes()
|
||||
99
xdist/gwmanage.py
Normal file
99
xdist/gwmanage.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
instantiating, managing and rsyncing to test hosts
|
||||
"""
|
||||
|
||||
import py
|
||||
import sys, os.path
|
||||
import execnet
|
||||
from execnet.gateway_base import RemoteError
|
||||
|
||||
class GatewayManager:
|
||||
RemoteError = RemoteError
|
||||
def __init__(self, specs, hook, defaultchdir="pyexecnetcache"):
|
||||
self.specs = []
|
||||
self.hook = hook
|
||||
self.group = execnet.Group()
|
||||
for spec in specs:
|
||||
if not isinstance(spec, execnet.XSpec):
|
||||
spec = execnet.XSpec(spec)
|
||||
if not spec.chdir and not spec.popen:
|
||||
spec.chdir = defaultchdir
|
||||
self.specs.append(spec)
|
||||
|
||||
def makegateways(self):
|
||||
assert not list(self.group)
|
||||
for spec in self.specs:
|
||||
gw = self.group.makegateway(spec)
|
||||
self.hook.pytest_gwmanage_newgateway(
|
||||
gateway=gw, platinfo=gw._rinfo())
|
||||
|
||||
def rsync(self, source, notify=None, verbose=False, ignores=None):
|
||||
""" perform rsync to all remote hosts.
|
||||
"""
|
||||
rsync = HostRSync(source, verbose=verbose, ignores=ignores)
|
||||
seen = py.builtin.set()
|
||||
gateways = []
|
||||
for gateway in self.group:
|
||||
spec = gateway.spec
|
||||
if spec.popen and not spec.chdir:
|
||||
# XXX this assumes that sources are python-packages
|
||||
# and that adding the basedir does not hurt
|
||||
gateway.remote_exec("""
|
||||
import sys ; sys.path.insert(0, %r)
|
||||
""" % os.path.dirname(str(source))).waitclose()
|
||||
continue
|
||||
if spec not in seen:
|
||||
def finished():
|
||||
if notify:
|
||||
notify("rsyncrootready", spec, source)
|
||||
rsync.add_target_host(gateway, finished=finished)
|
||||
seen.add(spec)
|
||||
gateways.append(gateway)
|
||||
if seen:
|
||||
self.hook.pytest_gwmanage_rsyncstart(
|
||||
source=source,
|
||||
gateways=gateways,
|
||||
)
|
||||
rsync.send()
|
||||
self.hook.pytest_gwmanage_rsyncfinish(
|
||||
source=source,
|
||||
gateways=gateways,
|
||||
)
|
||||
|
||||
def exit(self):
|
||||
self.group.terminate()
|
||||
|
||||
class HostRSync(execnet.RSync):
|
||||
""" RSyncer that filters out common files
|
||||
"""
|
||||
def __init__(self, sourcedir, *args, **kwargs):
|
||||
self._synced = {}
|
||||
ignores= None
|
||||
if 'ignores' in kwargs:
|
||||
ignores = kwargs.pop('ignores')
|
||||
self._ignores = ignores or []
|
||||
super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
|
||||
|
||||
def filter(self, path):
|
||||
path = py.path.local(path)
|
||||
if not path.ext in ('.pyc', '.pyo'):
|
||||
if not path.basename.endswith('~'):
|
||||
if path.check(dotfile=0):
|
||||
for x in self._ignores:
|
||||
if path == x:
|
||||
break
|
||||
else:
|
||||
return True
|
||||
|
||||
def add_target_host(self, gateway, finished=None):
|
||||
remotepath = os.path.basename(self._sourcedir)
|
||||
super(HostRSync, self).add_target(gateway, remotepath,
|
||||
finishedcallback=finished,
|
||||
delete=True,)
|
||||
|
||||
def _report_send_file(self, gateway, modified_rel_path):
|
||||
if self._verbose:
|
||||
path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
|
||||
remotepath = gateway.spec.chdir
|
||||
py.builtin.print_('%s:%s <= %s' %
|
||||
(gateway.spec, remotepath, path))
|
||||
187
xdist/mypickle.py
Normal file
187
xdist/mypickle.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
|
||||
Pickling support for two processes that want to exchange
|
||||
*immutable* object instances. Immutable in the sense
|
||||
that the receiving side of an object can modify its
|
||||
copy but when it sends it back the original sending
|
||||
side will continue to see its unmodified version
|
||||
(and no actual state will go over the wire).
|
||||
|
||||
This module also implements an experimental
|
||||
execnet pickling channel using this idea.
|
||||
|
||||
"""
|
||||
|
||||
import py
|
||||
import sys, os, struct
|
||||
#debug = open("log-mypickle-%d" % os.getpid(), 'w')
|
||||
|
||||
if sys.version_info >= (3,0):
|
||||
makekey = lambda x: x
|
||||
fromkey = lambda x: x
|
||||
from pickle import _Pickler as Pickler
|
||||
from pickle import _Unpickler as Unpickler
|
||||
else:
|
||||
makekey = str
|
||||
fromkey = int
|
||||
from pickle import Pickler, Unpickler
|
||||
|
||||
|
||||
class MyPickler(Pickler):
|
||||
""" Pickler with a custom memoize()
|
||||
to take care of unique ID creation.
|
||||
See the usage in ImmutablePickler
|
||||
XXX we could probably extend Pickler
|
||||
and Unpickler classes to directly
|
||||
update the other'S memos.
|
||||
"""
|
||||
def __init__(self, file, protocol, uneven):
|
||||
Pickler.__init__(self, file, protocol)
|
||||
self.uneven = uneven
|
||||
|
||||
def memoize(self, obj):
|
||||
if self.fast:
|
||||
return
|
||||
assert id(obj) not in self.memo
|
||||
memo_len = len(self.memo)
|
||||
key = memo_len * 2 + self.uneven
|
||||
self.write(self.put(key))
|
||||
self.memo[id(obj)] = key, obj
|
||||
|
||||
#if sys.version_info < (3,0):
|
||||
# def save_string(self, obj, pack=struct.pack):
|
||||
# obj = unicode(obj)
|
||||
# self.save_unicode(obj, pack=pack)
|
||||
# Pickler.dispatch[str] = save_string
|
||||
|
||||
class ImmutablePickler:
|
||||
def __init__(self, uneven, protocol=0):
|
||||
""" ImmutablePicklers are instantiated in Pairs.
|
||||
The two sides need to create unique IDs
|
||||
while pickling their objects. This is
|
||||
done by using either even or uneven
|
||||
numbers, depending on the instantiation
|
||||
parameter.
|
||||
"""
|
||||
self._picklememo = {}
|
||||
self._unpicklememo = {}
|
||||
self._protocol = protocol
|
||||
self.uneven = uneven and 1 or 0
|
||||
|
||||
def selfmemoize(self, obj):
|
||||
# this is for feeding objects to ourselfes
|
||||
# which be the case e.g. if you want to pickle
|
||||
# from a forked process back to the original
|
||||
f = py.io.BytesIO()
|
||||
pickler = MyPickler(f, self._protocol, uneven=self.uneven)
|
||||
pickler.memo = self._picklememo
|
||||
pickler.memoize(obj)
|
||||
self._updateunpicklememo()
|
||||
|
||||
def dumps(self, obj):
|
||||
f = py.io.BytesIO()
|
||||
pickler = MyPickler(f, self._protocol, uneven=self.uneven)
|
||||
pickler.memo = self._picklememo
|
||||
pickler.dump(obj)
|
||||
if obj is not None:
|
||||
self._updateunpicklememo()
|
||||
#print >>debug, "dumped", obj
|
||||
#print >>debug, "picklememo", self._picklememo
|
||||
return f.getvalue()
|
||||
|
||||
def loads(self, string):
|
||||
f = py.io.BytesIO(string)
|
||||
unpickler = Unpickler(f)
|
||||
unpickler.memo = self._unpicklememo
|
||||
res = unpickler.load()
|
||||
self._updatepicklememo()
|
||||
#print >>debug, "loaded", res
|
||||
#print >>debug, "unpicklememo", self._unpicklememo
|
||||
return res
|
||||
|
||||
def _updatepicklememo(self):
|
||||
for x, obj in self._unpicklememo.items():
|
||||
self._picklememo[id(obj)] = (fromkey(x), obj)
|
||||
|
||||
def _updateunpicklememo(self):
|
||||
for key,obj in self._picklememo.values():
|
||||
key = makekey(key)
|
||||
if key in self._unpicklememo:
|
||||
assert self._unpicklememo[key] is obj
|
||||
self._unpicklememo[key] = obj
|
||||
|
||||
NO_ENDMARKER_WANTED = object()
|
||||
|
||||
class UnpickleError(Exception):
|
||||
""" Problems while unpickling. """
|
||||
def __init__(self, formatted):
|
||||
self.formatted = formatted
|
||||
Exception.__init__(self, formatted)
|
||||
def __str__(self):
|
||||
return self.formatted
|
||||
|
||||
class PickleChannel(object):
|
||||
""" PickleChannels wrap execnet channels
|
||||
and allow to send/receive by using
|
||||
"immutable pickling".
|
||||
"""
|
||||
_unpicklingerror = None
|
||||
def __init__(self, channel):
|
||||
self._channel = channel
|
||||
# we use the fact that each side of a
|
||||
# gateway connection counts with uneven
|
||||
# or even numbers depending on which
|
||||
# side it is (for the purpose of creating
|
||||
# unique ids - which is what we need it here for)
|
||||
uneven = channel.gateway._channelfactory.count % 2
|
||||
self._ipickle = ImmutablePickler(uneven=uneven)
|
||||
self.RemoteError = channel.RemoteError
|
||||
|
||||
def send(self, obj):
|
||||
pickled_obj = self._ipickle.dumps(obj)
|
||||
self._channel.send(pickled_obj)
|
||||
|
||||
def receive(self):
|
||||
pickled_obj = self._channel.receive()
|
||||
return self._unpickle(pickled_obj)
|
||||
|
||||
def _unpickle(self, pickled_obj):
|
||||
if isinstance(pickled_obj, self._channel.__class__):
|
||||
return pickled_obj
|
||||
return self._ipickle.loads(pickled_obj)
|
||||
|
||||
def _getremoteerror(self):
|
||||
return self._unpicklingerror or self._channel._getremoteerror()
|
||||
|
||||
def close(self):
|
||||
return self._channel.close()
|
||||
|
||||
def isclosed(self):
|
||||
return self._channel.isclosed()
|
||||
|
||||
def waitclose(self, timeout=None):
|
||||
return self._channel.waitclose(timeout=timeout)
|
||||
|
||||
def setcallback(self, callback, endmarker=NO_ENDMARKER_WANTED):
|
||||
if endmarker is NO_ENDMARKER_WANTED:
|
||||
def unpickle_callback(pickled_obj):
|
||||
obj = self._unpickle(pickled_obj)
|
||||
callback(obj)
|
||||
self._channel.setcallback(unpickle_callback)
|
||||
return
|
||||
uniqueendmarker = object()
|
||||
def unpickle_callback(pickled_obj):
|
||||
if pickled_obj is uniqueendmarker:
|
||||
return callback(endmarker)
|
||||
try:
|
||||
obj = self._unpickle(pickled_obj)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
formatted = str(excinfo.getrepr(showlocals=True,funcargs=True))
|
||||
self._unpicklingerror = UnpickleError(formatted)
|
||||
callback(endmarker)
|
||||
else:
|
||||
callback(obj)
|
||||
self._channel.setcallback(unpickle_callback, uniqueendmarker)
|
||||
114
xdist/nodemanage.py
Normal file
114
xdist/nodemanage.py
Normal file
@@ -0,0 +1,114 @@
|
||||
import py
|
||||
import sys, os
|
||||
import xdist
|
||||
from xdist.txnode import TXNode
|
||||
from xdist.gwmanage import GatewayManager
|
||||
import execnet
|
||||
|
||||
class NodeManager(object):
|
||||
def __init__(self, config, specs=None):
|
||||
self.config = config
|
||||
if specs is None:
|
||||
specs = self._getxspecs()
|
||||
self.roots = self._getrsyncdirs()
|
||||
self.gwmanager = GatewayManager(specs, config.hook)
|
||||
self.nodes = []
|
||||
self._nodesready = py.std.threading.Event()
|
||||
|
||||
def trace(self, msg):
|
||||
self.config.hook.pytest_trace(category="nodemanage", msg=msg)
|
||||
|
||||
def config_getignores(self):
|
||||
return self.config.getconftest_pathlist("rsyncignore")
|
||||
|
||||
def rsync_roots(self):
|
||||
""" make sure that all remote gateways
|
||||
have the same set of roots in their
|
||||
current directory.
|
||||
"""
|
||||
self.makegateways()
|
||||
options = {
|
||||
'ignores': self.config_getignores(),
|
||||
'verbose': self.config.option.verbose,
|
||||
}
|
||||
if self.roots:
|
||||
# send each rsync root
|
||||
for root in self.roots:
|
||||
self.gwmanager.rsync(root, **options)
|
||||
else:
|
||||
XXX # do we want to care for situations without explicit rsyncdirs?
|
||||
# we transfer our topdir as the root
|
||||
self.gwmanager.rsync(self.config.topdir, **options)
|
||||
# and cd into it
|
||||
self.gwmanager.multi_chdir(self.config.topdir.basename, inplacelocal=False)
|
||||
|
||||
def makegateways(self):
|
||||
# we change to the topdir sot that
|
||||
# PopenGateways will have their cwd
|
||||
# such that unpickling configs will
|
||||
# pick it up as the right topdir
|
||||
# (for other gateways this chdir is irrelevant)
|
||||
self.trace("making gateways")
|
||||
old = self.config.topdir.chdir()
|
||||
try:
|
||||
self.gwmanager.makegateways()
|
||||
finally:
|
||||
old.chdir()
|
||||
|
||||
def setup_nodes(self, putevent):
|
||||
self.rsync_roots()
|
||||
self.trace("setting up nodes")
|
||||
for gateway in self.gwmanager.group:
|
||||
node = TXNode(gateway, self.config, putevent, slaveready=self._slaveready)
|
||||
gateway.node = node # to keep node alive
|
||||
self.trace("started node %r" % node)
|
||||
|
||||
def _slaveready(self, node):
|
||||
#assert node.gateway == node.gateway
|
||||
#assert node.gateway.node == node
|
||||
self.nodes.append(node)
|
||||
self.trace("%s slave node ready %r" % (node.gateway.id, node))
|
||||
if len(self.nodes) == len(list(self.gwmanager.group)):
|
||||
self._nodesready.set()
|
||||
|
||||
def wait_nodesready(self, timeout=None):
|
||||
self._nodesready.wait(timeout)
|
||||
if not self._nodesready.isSet():
|
||||
raise IOError("nodes did not get ready for %r secs" % timeout)
|
||||
|
||||
def teardown_nodes(self):
|
||||
# XXX do teardown nodes?
|
||||
self.gwmanager.exit()
|
||||
|
||||
def _getxspecs(self):
|
||||
config = self.config
|
||||
xspeclist = []
|
||||
for xspec in config.getvalue("tx"):
|
||||
i = xspec.find("*")
|
||||
try:
|
||||
num = int(xspec[:i])
|
||||
except ValueError:
|
||||
xspeclist.append(xspec)
|
||||
else:
|
||||
xspeclist.extend([xspec[i+1:]] * num)
|
||||
if not xspeclist:
|
||||
raise config.Error(
|
||||
"MISSING test execution (tx) nodes: please specify --tx")
|
||||
return [execnet.XSpec(x) for x in xspeclist]
|
||||
|
||||
def _getrsyncdirs(self):
|
||||
config = self.config
|
||||
candidates = [py._pydir]
|
||||
candidates += [py.path.local(xdist.__file__).dirpath()]
|
||||
candidates += config.option.rsyncdir
|
||||
conftestroots = config.getconftest_pathlist("rsyncdirs")
|
||||
if conftestroots:
|
||||
candidates.extend(conftestroots)
|
||||
roots = []
|
||||
for root in candidates:
|
||||
root = py.path.local(root).realpath()
|
||||
if not root.check():
|
||||
raise config.Error("rsyncdir doesn't exist: %r" %(root,))
|
||||
if root not in roots:
|
||||
roots.append(root)
|
||||
return roots
|
||||
235
xdist/plugin.py
Normal file
235
xdist/plugin.py
Normal file
@@ -0,0 +1,235 @@
|
||||
"""loop on failing tests, distribute test runs to CPUs and hosts.
|
||||
|
||||
The `pytest-xdist`_ plugin extends py.test with some unique
|
||||
test execution modes:
|
||||
|
||||
* Looponfail: run your tests in a subprocess. After it finishes py.test
|
||||
waits until a file in your project changes and then re-runs only the
|
||||
failing tests. This is repeated until all tests pass after which again
|
||||
a full run is performed.
|
||||
|
||||
* Load-balancing: if you have multiple CPUs or hosts you can use
|
||||
those for a combined test run. This allows to speed up
|
||||
development or to use special resources of remote machines.
|
||||
|
||||
* Multi-Platform coverage: you can specify different Python interpreters
|
||||
or different platforms and run tests in parallel on all of them.
|
||||
|
||||
Before running tests remotely, ``py.test`` efficiently synchronizes your
|
||||
program source code to the remote place. All test results
|
||||
are reported back and displayed to your local test session.
|
||||
You may specify different Python versions and interpreters.
|
||||
|
||||
|
||||
Usage examples
|
||||
---------------------
|
||||
|
||||
Speed up test runs by sending tests to multiple CPUs
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
To send tests to multiple CPUs, type::
|
||||
|
||||
py.test -n NUM
|
||||
|
||||
Especially for longer running tests or tests requiring
|
||||
a lot of IO this can lead to considerable speed ups.
|
||||
|
||||
|
||||
Running tests in a Python subprocess
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
To instantiate a python2.4 sub process and send tests to it, you may type::
|
||||
|
||||
py.test -d --tx popen//python=python2.4
|
||||
|
||||
This will start a subprocess which is run with the "python2.4"
|
||||
Python interpreter, found in your system binary lookup path.
|
||||
|
||||
If you prefix the --tx option value like this::
|
||||
|
||||
--tx 3*popen//python=python2.4
|
||||
|
||||
then three subprocesses would be created and tests
|
||||
will be load-balanced across these three processes.
|
||||
|
||||
|
||||
Sending tests to remote SSH accounts
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
Suppose you have a package ``mypkg`` which contains some
|
||||
tests that you can successfully run locally. And you
|
||||
have a ssh-reachable machine ``myhost``. Then
|
||||
you can ad-hoc distribute your tests by typing::
|
||||
|
||||
py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
|
||||
|
||||
This will synchronize your ``mypkg`` package directory
|
||||
to an remote ssh account and then locally collect tests
|
||||
and send them to remote places for execution.
|
||||
|
||||
You can specify multiple ``--rsyncdir`` directories
|
||||
to be sent to the remote side.
|
||||
|
||||
**NOTE:** For py.test to collect and send tests correctly
|
||||
you not only need to make sure all code and tests
|
||||
directories are rsynced, but that any test (sub) directory
|
||||
also has an ``__init__.py`` file because internally
|
||||
py.test references tests as a fully qualified python
|
||||
module path. **You will otherwise get strange errors**
|
||||
during setup of the remote side.
|
||||
|
||||
Sending tests to remote Socket Servers
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
Download the single-module `socketserver.py`_ Python program
|
||||
and run it like this::
|
||||
|
||||
python socketserver.py
|
||||
|
||||
It will tell you that it starts listening on the default
|
||||
port. You can now on your home machine specify this
|
||||
new socket host with something like this::
|
||||
|
||||
py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
|
||||
|
||||
|
||||
.. _`atonce`:
|
||||
|
||||
Running tests on many platforms at once
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
The basic command to run tests on multiple platforms is::
|
||||
|
||||
py.test --dist=each --tx=spec1 --tx=spec2
|
||||
|
||||
If you specify a windows host, an OSX host and a Linux
|
||||
environment this command will send each tests to all
|
||||
platforms - and report back failures from all platforms
|
||||
at once. The specifications strings use the `xspec syntax`_.
|
||||
|
||||
.. _`xspec syntax`: http://codespeak.net/execnet/trunk/basics.html#xspec
|
||||
|
||||
.. _`socketserver.py`: http://codespeak.net/svn/py/dist/py/execnet/script/socketserver.py
|
||||
|
||||
.. _`execnet`: http://codespeak.net/execnet
|
||||
|
||||
Specifying test exec environments in a conftest.py
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
Instead of specifying command line options, you can
|
||||
put options values in a ``conftest.py`` file like this::
|
||||
|
||||
pytest_option_tx = ['ssh=myhost//python=python2.5', 'popen//python=python2.5']
|
||||
pytest_option_dist = True
|
||||
|
||||
Any commandline ``--tx`` specifictions will add to the list of available execution
|
||||
environments.
|
||||
|
||||
Specifying "rsync" dirs in a conftest.py
|
||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||
|
||||
In your ``mypkg/conftest.py`` you may specify directories to synchronise
|
||||
or to exclude::
|
||||
|
||||
rsyncdirs = ['.', '../plugins']
|
||||
rsyncignore = ['_cache']
|
||||
|
||||
These directory specifications are relative to the directory
|
||||
where the ``conftest.py`` is found.
|
||||
|
||||
"""
|
||||
|
||||
import sys
|
||||
import py
|
||||
|
||||
def pytest_addoption(parser):
|
||||
group = parser.getgroup("xdist", "distributed and subprocess testing")
|
||||
group._addoption('-f', '--looponfail',
|
||||
action="store_true", dest="looponfail", default=False,
|
||||
help="run tests in subprocess, wait for modified files "
|
||||
"and re-run failing test set until all pass.")
|
||||
group._addoption('-n', dest="numprocesses", metavar="numprocesses",
|
||||
action="store", type="int",
|
||||
help="shortcut for '--dist=load --tx=NUM*popen'")
|
||||
group.addoption('--boxed',
|
||||
action="store_true", dest="boxed", default=False,
|
||||
help="box each test run in a separate process (unix)")
|
||||
group._addoption('--dist', metavar="distmode",
|
||||
action="store", choices=['load', 'each', 'no'],
|
||||
type="choice", dest="dist", default="no",
|
||||
help=("set mode for distributing tests to exec environments.\n\n"
|
||||
"each: send each test to each available environment.\n\n"
|
||||
"load: send each test to available environment.\n\n"
|
||||
"(default) no: run tests inprocess, don't distribute."))
|
||||
group._addoption('--tx', dest="tx", action="append", default=[],
|
||||
metavar="xspec",
|
||||
help=("add a test execution environment. some examples: "
|
||||
"--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
|
||||
"--tx ssh=user@codespeak.net//chdir=testcache"))
|
||||
group._addoption('-d',
|
||||
action="store_true", dest="distload", default=False,
|
||||
help="load-balance tests. shortcut for '--dist=load'")
|
||||
group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
|
||||
help="add directory for rsyncing to remote tx nodes.")
|
||||
|
||||
def pytest_configure(config):
|
||||
if config.option.numprocesses:
|
||||
config.option.dist = "load"
|
||||
config.option.tx = ['popen'] * int(config.option.numprocesses)
|
||||
if config.option.distload:
|
||||
config.option.dist = "load"
|
||||
val = config.getvalue
|
||||
if not val("collectonly"):
|
||||
usepdb = config.option.usepdb # a core option
|
||||
if val("looponfail"):
|
||||
if usepdb:
|
||||
raise config.Error("--pdb incompatible with --looponfail.")
|
||||
from xdist.remote import LooponfailingSession
|
||||
config.setsessionclass(LooponfailingSession)
|
||||
elif val("dist") != "no":
|
||||
if usepdb:
|
||||
raise config.Error("--pdb incompatible with distributing tests.")
|
||||
from xdist.dsession import DSession
|
||||
config.setsessionclass(DSession)
|
||||
|
||||
def pytest_runtest_protocol(item):
|
||||
if item.config.getvalue("boxed"):
|
||||
reports = forked_run_report(item)
|
||||
for rep in reports:
|
||||
item.ihook.pytest_runtest_logreport(report=rep)
|
||||
return True
|
||||
|
||||
def forked_run_report(item):
|
||||
# for now, we run setup/teardown in the subprocess
|
||||
# XXX optionally allow sharing of setup/teardown
|
||||
from py.plugin.pytest_runner import runtestprotocol
|
||||
EXITSTATUS_TESTEXIT = 4
|
||||
from xdist.mypickle import ImmutablePickler
|
||||
ipickle = ImmutablePickler(uneven=0)
|
||||
ipickle.selfmemoize(item.config)
|
||||
# XXX workaround the issue that 2.6 cannot pickle
|
||||
# instances of classes defined in global conftest.py files
|
||||
ipickle.selfmemoize(item)
|
||||
def runforked():
|
||||
try:
|
||||
reports = runtestprotocol(item, log=False)
|
||||
except KeyboardInterrupt:
|
||||
py.std.os._exit(EXITSTATUS_TESTEXIT)
|
||||
return ipickle.dumps(reports)
|
||||
|
||||
ff = py.process.ForkedFunc(runforked)
|
||||
result = ff.waitfinish()
|
||||
if result.retval is not None:
|
||||
return ipickle.loads(result.retval)
|
||||
else:
|
||||
if result.exitstatus == EXITSTATUS_TESTEXIT:
|
||||
py.test.exit("forked test item %s raised Exit" %(item,))
|
||||
return [report_process_crash(item, result)]
|
||||
|
||||
def report_process_crash(item, result):
|
||||
path, lineno = item._getfslineno()
|
||||
info = "%s:%s: running the test CRASHED with signal %d" %(
|
||||
path, lineno, result.signal)
|
||||
from py.plugin.pytest_runner import ItemTestReport
|
||||
return ItemTestReport(item, excinfo=info, when="???")
|
||||
|
||||
165
xdist/remote.py
Normal file
165
xdist/remote.py
Normal file
@@ -0,0 +1,165 @@
|
||||
"""
|
||||
LooponfailingSession and Helpers.
|
||||
|
||||
NOTE that one really has to avoid loading and depending on
|
||||
application modules within the controlling process
|
||||
(the one that starts repeatedly test processes)
|
||||
otherwise changes to source code can crash
|
||||
the controlling process which should never happen.
|
||||
"""
|
||||
import py
|
||||
import sys
|
||||
import execnet
|
||||
from py.impl.test.session import Session
|
||||
from xdist import util
|
||||
|
||||
class LooponfailingSession(Session):
|
||||
def __init__(self, config):
|
||||
super(LooponfailingSession, self).__init__(config=config)
|
||||
self.rootdirs = [self.config.topdir] # xxx dist_rsync_roots?
|
||||
self.statrecorder = util.StatRecorder(self.rootdirs)
|
||||
self.remotecontrol = RemoteControl(self.config)
|
||||
self.out = py.io.TerminalWriter()
|
||||
|
||||
def main(self, initialitems):
|
||||
try:
|
||||
self.loopstate = loopstate = LoopState([])
|
||||
self.remotecontrol.setup()
|
||||
while 1:
|
||||
self.loop_once(loopstate)
|
||||
if not loopstate.colitems and loopstate.wasfailing:
|
||||
continue # the last failures passed, let's rerun all
|
||||
self.statrecorder.waitonchange(checkinterval=2.0)
|
||||
except KeyboardInterrupt:
|
||||
print
|
||||
|
||||
def loop_once(self, loopstate):
|
||||
colitems = loopstate.colitems
|
||||
loopstate.wasfailing = colitems and len(colitems)
|
||||
loopstate.colitems = self.remotecontrol.runsession(colitems or ())
|
||||
self.remotecontrol.setup()
|
||||
|
||||
class LoopState:
|
||||
def __init__(self, colitems=None):
|
||||
self.colitems = colitems
|
||||
|
||||
class RemoteControl(object):
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
|
||||
def trace(self, *args):
|
||||
if self.config.option.debug:
|
||||
msg = " ".join([str(x) for x in args])
|
||||
py.builtin.print_("RemoteControl:", msg)
|
||||
|
||||
def initgateway(self):
|
||||
return execnet.makegateway("popen")
|
||||
|
||||
def setup(self, out=None):
|
||||
if out is None:
|
||||
out = py.io.TerminalWriter()
|
||||
if hasattr(self, 'gateway'):
|
||||
raise ValueError("already have gateway %r" % self.gateway)
|
||||
self.trace("setting up slave session")
|
||||
self.gateway = self.initgateway()
|
||||
self.channel = channel = self.gateway.remote_exec("""
|
||||
import os
|
||||
import py
|
||||
chdir = channel.receive()
|
||||
outchannel = channel.gateway.newchannel()
|
||||
channel.send(outchannel)
|
||||
os.chdir(chdir) # unpickling config uses cwd as topdir
|
||||
config_state = channel.receive()
|
||||
fullwidth, hasmarkup = channel.receive()
|
||||
py.test.config.__setstate__(config_state)
|
||||
|
||||
import sys
|
||||
sys.stdout = sys.stderr = outchannel.makefile('w')
|
||||
|
||||
from xdist.remote import slave_runsession
|
||||
slave_runsession(channel, py.test.config, fullwidth, hasmarkup)
|
||||
""")
|
||||
channel.send(str(self.config.topdir))
|
||||
remote_outchannel = channel.receive()
|
||||
def write(s):
|
||||
out._file.write(s)
|
||||
out._file.flush()
|
||||
remote_outchannel.setcallback(write)
|
||||
channel.send(self.config.__getstate__())
|
||||
channel.send((out.fullwidth, out.hasmarkup))
|
||||
self.trace("set up of slave session complete")
|
||||
|
||||
def ensure_teardown(self):
|
||||
if hasattr(self, 'channel'):
|
||||
if not self.channel.isclosed():
|
||||
self.trace("closing", self.channel)
|
||||
self.channel.close()
|
||||
del self.channel
|
||||
if hasattr(self, 'gateway'):
|
||||
self.trace("exiting", self.gateway)
|
||||
self.gateway.exit()
|
||||
del self.gateway
|
||||
|
||||
def runsession(self, colitems=()):
|
||||
try:
|
||||
self.trace("sending", colitems)
|
||||
trails = colitems
|
||||
self.channel.send(trails)
|
||||
try:
|
||||
return self.channel.receive()
|
||||
except self.channel.RemoteError:
|
||||
e = sys.exc_info()[1]
|
||||
self.trace("ERROR", e)
|
||||
raise
|
||||
finally:
|
||||
self.ensure_teardown()
|
||||
|
||||
def slave_runsession(channel, config, fullwidth, hasmarkup):
|
||||
""" we run this on the other side. """
|
||||
if config.option.debug:
|
||||
def DEBUG(*args):
|
||||
print(" ".join(map(str, args)))
|
||||
else:
|
||||
def DEBUG(*args): pass
|
||||
|
||||
DEBUG("SLAVE: received configuration, using topdir:", config.topdir)
|
||||
#config.option.session = None
|
||||
config.option.looponfail = False
|
||||
config.option.usepdb = False
|
||||
trails = channel.receive()
|
||||
config.pluginmanager.do_configure(config)
|
||||
DEBUG("SLAVE: initsession()")
|
||||
session = config.initsession()
|
||||
# XXX configure the reporter object's terminal writer more directly
|
||||
# XXX and write a test for this remote-terminal setting logic
|
||||
config.pytest_terminal_hasmarkup = hasmarkup
|
||||
config.pytest_terminal_fullwidth = fullwidth
|
||||
if trails:
|
||||
colitems = []
|
||||
for trail in trails:
|
||||
try:
|
||||
colitem = config._rootcol.fromtrail(trail)
|
||||
except ValueError:
|
||||
#XXX send info for "test disappeared" or so
|
||||
continue
|
||||
colitems.append(colitem)
|
||||
else:
|
||||
colitems = config.getinitialnodes()
|
||||
session.shouldclose = channel.isclosed
|
||||
|
||||
class Failures(list):
|
||||
def pytest_runtest_logreport(self, report):
|
||||
if report.failed:
|
||||
self.append(report)
|
||||
pytest_collectreport = pytest_runtest_logreport
|
||||
|
||||
failreports = Failures()
|
||||
session.pluginmanager.register(failreports)
|
||||
|
||||
DEBUG("SLAVE: starting session.main()")
|
||||
session.main(colitems)
|
||||
session.config.hook.pytest_looponfailinfo(
|
||||
failreports=list(failreports),
|
||||
rootdirs=[config.topdir])
|
||||
rootcol = session.config._rootcol
|
||||
channel.send([rootcol.totrail(rep.getnode()) for rep in failreports])
|
||||
164
xdist/txnode.py
Normal file
164
xdist/txnode.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Manage setup, running and local representation of remote nodes/processes.
|
||||
"""
|
||||
import py
|
||||
from xdist.mypickle import PickleChannel
|
||||
from py.impl.test import outcome
|
||||
|
||||
class TXNode(object):
|
||||
""" Represents a Test Execution environment in the controlling process.
|
||||
- sets up a slave node through an execnet gateway
|
||||
- manages sending of test-items and receival of results and events
|
||||
- creates events when the remote side crashes
|
||||
"""
|
||||
ENDMARK = -1
|
||||
|
||||
def __init__(self, gateway, config, putevent, slaveready=None):
|
||||
self.config = config
|
||||
self.putevent = putevent
|
||||
self.gateway = gateway
|
||||
self.channel = install_slave(gateway, config)
|
||||
self._sendslaveready = slaveready
|
||||
self.channel.setcallback(self.callback, endmarker=self.ENDMARK)
|
||||
self._down = False
|
||||
|
||||
def __repr__(self):
|
||||
id = self.gateway.id
|
||||
status = self._down and 'true' or 'false'
|
||||
return "<TXNode %r down=%s>" %(id, status)
|
||||
|
||||
def notify(self, eventname, *args, **kwargs):
|
||||
assert not args
|
||||
self.putevent((eventname, args, kwargs))
|
||||
|
||||
def callback(self, eventcall):
|
||||
""" this gets called for each object we receive from
|
||||
the other side and if the channel closes.
|
||||
|
||||
Note that channel callbacks run in the receiver
|
||||
thread of execnet gateways - we need to
|
||||
avoid raising exceptions or doing heavy work.
|
||||
"""
|
||||
try:
|
||||
if eventcall == self.ENDMARK:
|
||||
err = self.channel._getremoteerror()
|
||||
if not self._down:
|
||||
if not err or isinstance(err, EOFError):
|
||||
err = "Not properly terminated"
|
||||
self.notify("pytest_testnodedown", node=self, error=err)
|
||||
self._down = True
|
||||
return
|
||||
eventname, args, kwargs = eventcall
|
||||
if eventname == "slaveready":
|
||||
if self._sendslaveready:
|
||||
self._sendslaveready(self)
|
||||
self.notify("pytest_testnodeready", node=self)
|
||||
elif eventname == "slavefinished":
|
||||
self._down = True
|
||||
self.notify("pytest_testnodedown", error=None, node=self)
|
||||
elif eventname in ("pytest_runtest_logreport",
|
||||
"pytest__teardown_final_logerror"):
|
||||
kwargs['report'].node = self
|
||||
self.notify(eventname, **kwargs)
|
||||
else:
|
||||
self.notify(eventname, **kwargs)
|
||||
except KeyboardInterrupt:
|
||||
# should not land in receiver-thread
|
||||
raise
|
||||
except:
|
||||
excinfo = py.code.ExceptionInfo()
|
||||
py.builtin.print_("!" * 20, excinfo)
|
||||
self.config.pluginmanager.notify_exception(excinfo)
|
||||
|
||||
def send(self, item):
|
||||
assert item is not None
|
||||
self.channel.send(item)
|
||||
|
||||
def sendlist(self, itemlist):
|
||||
self.channel.send(itemlist)
|
||||
|
||||
def shutdown(self):
|
||||
self.channel.send(None)
|
||||
|
||||
# setting up slave code
|
||||
def install_slave(gateway, config):
|
||||
channel = gateway.remote_exec(source="""
|
||||
import os, sys
|
||||
sys.path.insert(0, os.getcwd())
|
||||
from xdist.mypickle import PickleChannel
|
||||
from xdist.txnode import SlaveNode
|
||||
channel.send("basicimport")
|
||||
channel = PickleChannel(channel)
|
||||
slavenode = SlaveNode(channel)
|
||||
slavenode.run()
|
||||
""")
|
||||
channel.receive()
|
||||
channel = PickleChannel(channel)
|
||||
basetemp = None
|
||||
if gateway.spec.popen:
|
||||
popenbase = config.ensuretemp("popen")
|
||||
basetemp = py.path.local.make_numbered_dir(prefix="slave-",
|
||||
keep=0, rootdir=popenbase)
|
||||
basetemp = str(basetemp)
|
||||
channel.send((config, basetemp, gateway.id))
|
||||
return channel
|
||||
|
||||
class SlaveNode(object):
|
||||
def __init__(self, channel):
|
||||
self.channel = channel
|
||||
|
||||
def __repr__(self):
|
||||
return "<%s channel=%s>" %(self.__class__.__name__, self.channel)
|
||||
|
||||
def sendevent(self, eventname, *args, **kwargs):
|
||||
self.channel.send((eventname, args, kwargs))
|
||||
|
||||
def pytest_runtest_logreport(self, report):
|
||||
self.sendevent("pytest_runtest_logreport", report=report)
|
||||
|
||||
def pytest__teardown_final_logerror(self, report):
|
||||
self.sendevent("pytest__teardown_final_logerror", report=report)
|
||||
|
||||
def run(self):
|
||||
channel = self.channel
|
||||
self.config, basetemp, self.nodeid = channel.receive()
|
||||
if basetemp:
|
||||
self.config.basetemp = py.path.local(basetemp)
|
||||
self.config.pluginmanager.do_configure(self.config)
|
||||
self.config.pluginmanager.register(self)
|
||||
self.runner = self.config.pluginmanager.getplugin("pytest_runner")
|
||||
self.sendevent("slaveready")
|
||||
try:
|
||||
self.config.hook.pytest_sessionstart(session=self)
|
||||
while 1:
|
||||
task = channel.receive()
|
||||
if task is None:
|
||||
break
|
||||
if isinstance(task, list):
|
||||
for item in task:
|
||||
self.run_single(item=item)
|
||||
else:
|
||||
self.run_single(item=task)
|
||||
self.config.hook.pytest_sessionfinish(
|
||||
session=self,
|
||||
exitstatus=outcome.EXIT_OK)
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except:
|
||||
er = py.code.ExceptionInfo().getrepr(funcargs=True, showlocals=True)
|
||||
self.sendevent("pytest_internalerror", excrepr=er)
|
||||
raise
|
||||
else:
|
||||
self.sendevent("slavefinished")
|
||||
|
||||
def run_single(self, item):
|
||||
call = self.runner.CallInfo(item._checkcollectable, when='setup')
|
||||
if call.excinfo:
|
||||
# likely it is not collectable here because of
|
||||
# platform/import-dependency induced skips
|
||||
# we fake a setup-error report with the obtained exception
|
||||
# and do not care about capturing or non-runner hooks
|
||||
rep = self.runner.pytest_runtest_makereport(item=item, call=call)
|
||||
self.pytest_runtest_logreport(rep)
|
||||
return
|
||||
item.config.hook.pytest_runtest_protocol(item=item)
|
||||
53
xdist/util.py
Normal file
53
xdist/util.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import py
|
||||
|
||||
class StatRecorder:
|
||||
def __init__(self, rootdirlist):
|
||||
self.rootdirlist = rootdirlist
|
||||
self.statcache = {}
|
||||
self.check() # snapshot state
|
||||
|
||||
def fil(self, p):
|
||||
return p.ext in ('.py', '.txt', '.c', '.h')
|
||||
def rec(self, p):
|
||||
return p.check(dotfile=0)
|
||||
|
||||
def waitonchange(self, checkinterval=1.0):
|
||||
while 1:
|
||||
changed = self.check()
|
||||
if changed:
|
||||
return
|
||||
py.std.time.sleep(checkinterval)
|
||||
|
||||
def check(self, removepycfiles=True):
|
||||
changed = False
|
||||
statcache = self.statcache
|
||||
newstat = {}
|
||||
for rootdir in self.rootdirlist:
|
||||
for path in rootdir.visit(self.fil, self.rec):
|
||||
oldstat = statcache.get(path, None)
|
||||
if oldstat is not None:
|
||||
del statcache[path]
|
||||
try:
|
||||
newstat[path] = curstat = path.stat()
|
||||
except py.error.ENOENT:
|
||||
if oldstat:
|
||||
del statcache[path]
|
||||
changed = True
|
||||
else:
|
||||
if oldstat:
|
||||
if oldstat.mtime != curstat.mtime or \
|
||||
oldstat.size != curstat.size:
|
||||
changed = True
|
||||
py.builtin.print_("# MODIFIED", path)
|
||||
if removepycfiles and path.ext == ".py":
|
||||
pycfile = path + "c"
|
||||
if pycfile.check():
|
||||
pycfile.remove()
|
||||
|
||||
else:
|
||||
changed = True
|
||||
if statcache:
|
||||
changed = True
|
||||
self.statcache = newstat
|
||||
return changed
|
||||
|
||||
Reference in New Issue
Block a user