IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rework dist-testing model: now all items are collected at the start. move back some tests to pytest-core

Browse Source
This commit is contained in:
holger krekel
2010-01-17 23:25:03 +01:00
parent 57c4e7300a
commit c607baca2b
6 changed files with 93 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
xdist/dsession.py
View File

@@ -54,7 +54,7 @@ class LoopState(object):
self.colitems.extend(pending[1:])
def pytest_rescheduleitems(self, items):
self.colitems.extend(items)
self.colitems[:] = items + self.colitems
self.dowork = False # avoid busywait
class DSession(Session):
@@ -62,8 +62,9 @@ class DSession(Session):
Session drives the collection and running of tests
and generates test events for reporters.
"""
MAXITEMSPERHOST = 15
LOAD_THRESHOLD_NEWITEMS = 5
ITEM_CHUNKSIZE = 10
def __init__(self, config):
self.queue = queue.Queue()
self.node2pending = {}
@@ -73,11 +74,20 @@ class DSession(Session):
def main(self, colitems):
self.sessionstarts()
self.setup()
exitstatus = self.loop(colitems)
allitems = self.collect_all_items(colitems)
self.nodemanager.wait_nodesready(5.0)
#for x in allitems:
# print x.listnames()
exitstatus = self.loop(allitems)
self.teardown()
self.sessionfinishes(exitstatus=exitstatus)
return exitstatus
def collect_all_items(self, colitems):
allitems = list(self.collect(colitems))
print ("collected %d items" %(len(allitems)))
return allitems
def loop_once(self, loopstate):
if loopstate.shuttingdown:
return self.loop_once_shutdown(loopstate)
@@ -180,15 +190,10 @@ class DSession(Session):
return pending
def triggertesting(self, colitems):
colitems = self.filteritems(colitems)
senditems = []
# for now we don't allow sending collectors
for next in colitems:
if isinstance(next, py.test.collect.Item):
senditems.append(next)
else:
self.config.hook.pytest_collectstart(collector=next)
colrep = self.config.hook.pytest_make_collect_report(collector=next)
self.queueevent("pytest_collectreport", report=colrep)
assert isinstance(next, py.test.collect.Item), next
senditems = list(colitems)
if self.config.option.dist == "each":
self.senditems_each(senditems)
else:
@@ -201,42 +206,36 @@ class DSession(Session):
def senditems_each(self, tosend):
if not tosend:
return
room = self.MAXITEMSPERHOST
for node, pending in self.node2pending.items():
room = min(self.MAXITEMSPERHOST - len(pending), room)
sending = tosend[:room]
if sending:
for node, pending in self.node2pending.items():
node.sendlist(sending)
pending.extend(sending)
for item in sending:
nodes = self.item2nodes.setdefault(item, [])
assert node not in nodes
nodes.append(node)
item.ihook.pytest_itemstart(item=item, node=node)
tosend[:] = tosend[room:] # update inplace
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
node.sendlist(tosend)
pending.extend(tosend)
for item in tosend:
nodes = self.item2nodes.setdefault(item, [])
assert node not in nodes
nodes.append(node)
item.ihook.pytest_itemstart(item=item, node=node)
tosend[:] = []
def senditems_load(self, tosend):
if not tosend:
return
available = []
for node, pending in self.node2pending.items():
room = self.MAXITEMSPERHOST - len(pending)
if room > 0:
sending = tosend[:room]
node.sendlist(sending)
for item in sending:
#assert item not in self.item2node, (
# "sending same item %r to multiple "
# "not implemented" %(item,))
self.item2nodes.setdefault(item, []).append(node)
item.ihook.pytest_itemstart(item=item, node=node)
pending.extend(sending)
tosend[:] = tosend[room:] # update inplace
if not tosend:
if len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
available.append((node, pending))
num_available = len(available)
max_one_round = num_available * self.ITEM_CHUNKSIZE -1
if num_available:
for i, item in enumerate(tosend):
nodeindex = i % num_available
node, pending = available[nodeindex]
node.send(item)
self.item2nodes.setdefault(item, []).append(node)
item.ihook.pytest_itemstart(item=item, node=node)
pending.append(item)
if i >= max_one_round:
break
del tosend[:i+1]
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
@@ -263,8 +262,6 @@ class DSession(Session):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def teardown(self):
""" teardown any resources after a test run. """

4
xdist/plugin.py
View File

@@ -3,8 +3,8 @@
The `pytest-xdist`_ plugin extends py.test with some unique
test execution modes:
* Looponfail: run your tests in a subprocess. After it finishes py.test
waits until a file in your project changes and then re-runs only the
* Looponfail: run your tests repeatedly in a subprocess. After each run py.test
waits until a file in your project changes and then re-runs the previously
failing tests. This is repeated until all tests pass after which again
a full run is performed.

16
xdist/remote.py
View File

@@ -63,12 +63,21 @@ class RemoteControl(object):
self.trace("setting up slave session")
self.gateway = self.initgateway()
self.channel = channel = self.gateway.remote_exec("""
import os
import os, sys
import py
chdir = channel.receive()
outchannel = channel.gateway.newchannel()
channel.send(outchannel)
# prune sys.path to not contain relative paths
newpaths = []
for p in sys.path:
if p:
if not os.path.isabs(p):
p = os.path.abspath(p)
newpaths.append(p)
sys.path[:] = newpaths
os.chdir(chdir) # unpickling config uses cwd as topdir
config_state = channel.receive()
fullwidth, hasmarkup = channel.receive()
py.test.config.__setstate__(config_state)
@@ -126,7 +135,10 @@ def slave_runsession(channel, config, fullwidth, hasmarkup):
#config.option.session = None
config.option.looponfail = False
config.option.usepdb = False
trails = channel.receive()
try:
trails = channel.receive()
except KeyboardInterrupt:
return # in the slave we can't do much about this
config.pluginmanager.do_configure(config)
DEBUG("SLAVE: initsession()")
session = config.initsession()

2
xdist/txnode.py
View File

@@ -152,7 +152,7 @@ class SlaveNode(object):
self.sendevent("slavefinished")
def run_single(self, item):
call = self.runner.CallInfo(item._checkcollectable, when='setup')
call = self.runner.CallInfo(item._reraiseunpicklingproblem, when='setup')
if call.excinfo:
# likely it is not collectable here because of
# platform/import-dependency induced skips