Compare commits

...

30 Commits
1.1 ... 1.3

Author SHA1 Message Date
holger krekel
faa03ad601 merge 2010-05-25 21:23:23 +02:00
holger krekel
43c0591c55 Added tag 1.3 for changeset e6c4ce20db4b 2010-05-25 21:26:09 +02:00
holger krekel
7bf3c7f029 bump version number 2010-05-25 21:26:06 +02:00
holger krekel
e0f61e4fa2 merge 2010-05-25 21:00:42 +02:00
holger krekel
f7be994848 adapt for py-trunk/1.3.1 changes wrt --exitfirst -> --maxfailures 2010-05-25 16:59:08 +02:00
holger krekel
7b19de5450 Added tag 1.2 for changeset 56d8e5280be2 2010-05-25 12:02:29 +02:00
holger krekel
0c4f5eced2 fix conftest related documentation 2010-05-18 18:40:48 +02:00
holger krekel
e593841a70 fix looponfailing issue to always run against the newest version of the source 2010-05-17 17:32:34 +02:00
holger krekel
43b693258f xdist is now somewhat stable after all the fixes, i'd say. 2010-05-05 21:21:39 +02:00
holger krekel
215e0e6149 modify CHANGELOG 2010-05-05 20:24:34 +02:00
holger krekel
a7ff9d751b require execnet-1.0.6 - it's release already 2010-05-05 20:19:30 +02:00
holger krekel
95a87e874d add python3 classifier 2010-05-05 14:25:59 +02:00
holger krekel
f18c78a118 adapt to new 1.3.0 hook registration 2010-05-02 16:44:59 +02:00
holger krekel
40277fbf7f require py-1.3.0 for xdist plugin 2010-05-02 15:07:00 +02:00
holger krekel
2b20e40805 update CHANGELOG 2010-04-30 16:28:10 +02:00
holger krekel
45f7787fe2 fixing a (likely) race condition when simultanous pickling/unpickling can leave
inconsistent memo states.  Rather than adding locks, the fix actually simplifies the code by
keeping the pickling/unpickling memo's in sync directly and more efficiently, also removing
a long outstanding XXX.
2010-04-30 15:56:48 +02:00
holger krekel
1bc7812dcc adapt for fnmatch_lines changes on trunk 2010-04-29 00:47:11 +02:00
holger krekel
bdc3f9bf53 fix python3 issues and add info to changelog 2010-04-27 21:26:35 +02:00
holger krekel
b6cdbf46a6 adapt for changed py-trunk, new outcome exceptions 2010-04-27 21:16:47 +02:00
holger krekel
bd254674f9 fix py3 print issue 2010-04-26 18:57:08 +02:00
holger krekel
9c74f59e55 fixes issue79 - call hooks more systematically on slave nodes and also in the case of SIGINT
reorganize internal session code to share more code with the "normal" non-distributed session
2010-04-26 18:35:53 +02:00
holger krekel
25cb514331 internally rename SlaveNode to SlaveSession which fits better 2010-04-26 15:48:31 +02:00
holger krekel
e993a6b079 add to changelog 2010-04-26 10:13:15 +02:00
Meme Dough
8eecf6e1d2 Fix description. 2010-04-26 14:54:55 +10:00
holger krekel
9abaae8778 * introduce explicit pytest_configure_node hook for node configuration,
simplifying implementation of slaveinput/slaveoutput data exchange
* only require the already released execnet-1.0.5
2010-04-22 17:00:03 +02:00
Meme Dough
8d6bd3ecde Slave data stored on config for slave and node for master. 2010-04-22 22:21:58 +10:00
holger krekel
a1ab548cb3 declare xdist-related hooks directly here (requires a recent py-trunk) 2010-04-22 11:58:13 +02:00
Meme Dough
e7a47ae911 Added support for exchanging data between master and slave. 2010-04-21 22:33:09 +10:00
holger krekel
3a8ead3c82 Added tag 1.1 for changeset eca7ce17eabf 2010-02-07 13:47:00 +01:00
holger krekel
8f2c3fb04e higher test-timeouts for slower nightly run machines 2010-01-21 20:07:41 +01:00
17 changed files with 268 additions and 118 deletions

View File

@@ -1 +1,4 @@
42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0 42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0
eca7ce17eabf296983c36812c8b8be901e7055a3 1.1
56d8e5280be224a0ad3220a9deed55334710bd23 1.2
e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3

View File

@@ -1,3 +1,25 @@
1.3
-------------------------
- fix --looponfailing - it would not actually run against the fully changed
source tree when initial conftest files load application state.
- adapt for py-1.3.1's new --maxfailure option
1.2
-------------------------
- fix issue79: sessionfinish/teardown hooks are now called systematically
on the slave side
- introduce a new data input/output mechanism to allow the master side
to send and receive data from a slave.
- fix race condition in underlying pickling/unpickling handling
- use and require new register hooks facility of py.test>=1.3.0
- require improved execnet>=1.0.6 because of various race conditions
that can arise in xdist testing modes.
- fix some python3 related pickling related race conditions
- fix PyPI description
1.1 1.1
------------------------- -------------------------

View File

@@ -7,11 +7,12 @@ for the new options.
""" """
from setuptools import setup from setuptools import setup
from xdist import __version__
setup( setup(
name="pytest-xdist", name="pytest-xdist",
version="1.1", version=__version__,
description='py.test figleaf coverage plugin', description='py.test xdist plugin for distributed testing and loop-on-failing modes',
long_description=__doc__, long_description=__doc__,
license='GPLv2 or later', license='GPLv2 or later',
author='holger krekel and contributors', author='holger krekel and contributors',
@@ -21,9 +22,9 @@ setup(
packages = ['xdist'], packages = ['xdist'],
entry_points = {'pytest11': ['xdist = xdist.plugin'],}, entry_points = {'pytest11': ['xdist = xdist.plugin'],},
zip_safe=False, zip_safe=False,
install_requires = ['execnet>=1.0.4', 'py>=1.2.0'], install_requires = ['execnet>=1.0.6', 'py>=1.3.1'],
classifiers=[ classifiers=[
'Development Status :: 4 - Beta', 'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers', 'Intended Audience :: Developers',
'License :: OSI Approved :: GNU General Public License (GPL)', 'License :: OSI Approved :: GNU General Public License (GPL)',
'Operating System :: POSIX', 'Operating System :: POSIX',
@@ -33,5 +34,6 @@ setup(
'Topic :: Software Development :: Quality Assurance', 'Topic :: Software Development :: Quality Assurance',
'Topic :: Utilities', 'Topic :: Utilities',
'Programming Language :: Python', 'Programming Language :: Python',
'Programming Language :: Python :: 3',
], ],
) )

View File

@@ -117,3 +117,58 @@ class TestDistribution:
s = result.stdout.str() s = result.stdout.str()
assert "2.4" in s assert "2.4" in s
assert "2.5" in s assert "2.5" in s
def test_data_exchange(self, testdir):
c1 = testdir.makeconftest("""
# This hook only called on master.
def pytest_configure_node(node):
node.slaveinput['a'] = 42
node.slaveinput['b'] = 7
# This hook only takes action on slave.
def pytest_configure(config):
if hasattr(config, 'slaveinput'):
a = config.slaveinput['a']
b = config.slaveinput['b']
r = a + b
config.slaveoutput['r'] = r
# This hook only called on master.
def pytest_testnodedown(node, error):
node.config.calc_result = node.slaveoutput['r']
# This hook only takes action on master.
def pytest_terminal_summary(terminalreporter):
if not hasattr(terminalreporter.config, 'slaveinput'):
calc_result = terminalreporter.config.calc_result
terminalreporter._tw.sep('-',
'calculated result is %s' % calc_result)
""")
p1 = testdir.makepyfile("def test_func(): pass")
result = testdir.runpytest(p1, '-d', '--tx=popen')
result.stdout.fnmatch_lines([
"*popen*Python*",
"*calculated result is 49*",
"*1 passed*"
])
assert result.ret == 0
def test_keyboardinterrupt_hooks_issue79(self, testdir):
testdir.makepyfile(__init__="", test_one="""
def test_hello():
raise KeyboardInterrupt()
""")
testdir.makeconftest("""
def pytest_sessionfinish(session):
if hasattr(session.config, 'slaveoutput'):
session.config.slaveoutput['s2'] = 42
def pytest_testnodedown(node, error):
assert node.slaveoutput['s2'] == 42
print ("s2call-finished")
""")
args = ["-n1"]
result = testdir.runpytest(*args)
s = result.stdout.str()
assert result.ret
assert 'SIGINT' in s
assert 's2call' in s

View File

@@ -8,7 +8,7 @@ def test_functional_boxed(testdir):
os.kill(os.getpid(), 15) os.kill(os.getpid(), 15)
""") """)
result = testdir.runpytest(p1, "--boxed") result = testdir.runpytest(p1, "--boxed")
assert result.stdout.fnmatch_lines([ result.stdout.fnmatch_lines([
"*CRASHED*", "*CRASHED*",
"*1 failed*" "*1 failed*"
]) ])

View File

@@ -1,5 +1,5 @@
from xdist.dsession import DSession from xdist.dsession import DSession
from py._test import outcome from py._test import session as outcome
import py import py
import execnet import execnet
@@ -259,7 +259,7 @@ class TestDSession:
def test_pass(): def test_pass():
pass pass
""") """)
modcol.config.option.exitfirst = True modcol.config.option.maxfail = 1
session = DSession(modcol.config) session = DSession(modcol.config)
node = MockNode() node = MockNode()
session.addnode(node) session.addnode(node)
@@ -271,12 +271,44 @@ class TestDSession:
# run tests ourselves and produce reports # run tests ourselves and produce reports
ev1 = run(items[0], node, "fail") ev1 = run(items[0], node, "fail")
ev2 = run(items[1], node, None) ev2 = run(items[1], node, None)
session.queueevent("pytest_runtest_logreport", report=ev1)
session.queueevent("pytest_runtest_logreport", report=ev2)
# now call the loop
loopstate = session._initloopstate(items)
py.test.raises(session.Interrupted, "session.loop_once(loopstate)")
assert loopstate.testsfailed
#assert loopstate.shuttingdown
def test_maxfail(self, testdir):
modcol = testdir.getmodulecol("""
def test_fail1():
assert 0
def test_fail2():
assert 0
def test_pass():
pass
""")
modcol.config.option.maxfail = 2
session = DSession(modcol.config)
node = MockNode()
session.addnode(node)
items = modcol.config.hook.pytest_make_collect_report(collector=modcol).result
# trigger testing - this sends tests to the node
session.triggertesting(items)
# run tests ourselves and produce reports
ev1 = run(items[0], node, "fail")
ev2 = run(items[1], node, "fail")
session.queueevent("pytest_runtest_logreport", report=ev1) # a failing one session.queueevent("pytest_runtest_logreport", report=ev1) # a failing one
session.queueevent("pytest_runtest_logreport", report=ev2) session.queueevent("pytest_runtest_logreport", report=ev2)
# now call the loop # now call the loop
loopstate = session._initloopstate(items) loopstate = session._initloopstate(items)
from xdist.dsession import ExitFirstInterrupt try:
py.test.raises(ExitFirstInterrupt, "session.loop_once(loopstate)") session.loop_once(loopstate)
except session.Interrupted:
py.test.fail("raised Interrupted but shouildn't")
py.test.raises(session.Interrupted, "session.loop_once(loopstate)")
assert loopstate.testsfailed assert loopstate.testsfailed
#assert loopstate.shuttingdown #assert loopstate.shuttingdown

View File

@@ -3,6 +3,7 @@ import os
from xdist.gwmanage import GatewayManager, HostRSync from xdist.gwmanage import GatewayManager, HostRSync
from py._test.pluginmanager import HookRelay, Registry from py._test.pluginmanager import HookRelay, Registry
from py._plugin import hookspec from py._plugin import hookspec
from xdist import newhooks
import execnet import execnet
def pytest_funcarg__hookrecorder(request): def pytest_funcarg__hookrecorder(request):
@@ -11,7 +12,7 @@ def pytest_funcarg__hookrecorder(request):
return _pytest.gethookrecorder(hook) return _pytest.gethookrecorder(hook)
def pytest_funcarg__hook(request): def pytest_funcarg__hook(request):
return HookRelay(hookspec, Registry()) return HookRelay([hookspec, newhooks], Registry())
class TestGatewayManagerPopen: class TestGatewayManagerPopen:
def test_popen_no_default_chdir(self, hook): def test_popen_no_default_chdir(self, hook):

View File

@@ -61,7 +61,6 @@ class TestLooponFailing:
""") """)
session = LooponfailingSession(modcol.config) session = LooponfailingSession(modcol.config)
loopstate = LoopState() loopstate = LoopState()
session.remotecontrol.setup()
session.loop_once(loopstate) session.loop_once(loopstate)
assert len(loopstate.colitems) == 1 assert len(loopstate.colitems) == 1
@@ -83,7 +82,6 @@ class TestLooponFailing:
""") """)
session = LooponfailingSession(modcol.config) session = LooponfailingSession(modcol.config)
loopstate = LoopState() loopstate = LoopState()
session.remotecontrol.setup()
loopstate.colitems = [] loopstate.colitems = []
session.loop_once(loopstate) session.loop_once(loopstate)
assert len(loopstate.colitems) == 1 assert len(loopstate.colitems) == 1
@@ -110,7 +108,6 @@ class TestLooponFailing:
""") """)
session = LooponfailingSession(modcol.config) session = LooponfailingSession(modcol.config)
loopstate = LoopState() loopstate = LoopState()
session.remotecontrol.setup()
loopstate.colitems = [] loopstate.colitems = []
session.loop_once(loopstate) session.loop_once(loopstate)
assert len(loopstate.colitems) == 2 assert len(loopstate.colitems) == 2

View File

@@ -12,7 +12,7 @@ class EventQueue:
self.queue = queue self.queue = queue
registry.register(self) registry.register(self)
def geteventargs(self, eventname, timeout=2.0): def geteventargs(self, eventname, timeout=10.0):
events = [] events = []
while 1: while 1:
try: try:
@@ -38,7 +38,7 @@ class MySetup:
self.id = 0 self.id = 0
self.request = request self.request = request
def geteventargs(self, eventname, timeout=2.0): def geteventargs(self, eventname, timeout=10.0):
eq = EventQueue(self.config.pluginmanager, self.queue) eq = EventQueue(self.config.pluginmanager, self.queue)
return eq.geteventargs(eventname, timeout=timeout) return eq.geteventargs(eventname, timeout=timeout)
@@ -52,7 +52,8 @@ class MySetup:
self.gateway = execnet.makegateway(self.xspec) self.gateway = execnet.makegateway(self.xspec)
self.id += 1 self.id += 1
self.gateway.id = str(self.id) self.gateway.id = str(self.id)
self.node = TXNode(self.gateway, self.config, putevent=self.queue.put) self.nodemanager = None
self.node = TXNode(self.nodemanager, self.gateway, self.config, putevent=self.queue.put)
assert not self.node.channel.isclosed() assert not self.node.channel.isclosed()
return self.node return self.node
@@ -80,7 +81,7 @@ class TestMasterSlaveConnection:
node.send(123) # invalid item node.send(123) # invalid item
kwargs = mysetup.geteventargs("pytest_testnodedown") kwargs = mysetup.geteventargs("pytest_testnodedown")
assert kwargs['node'] is node assert kwargs['node'] is node
assert isinstance(kwargs['error'], execnet.RemoteError) #assert isinstance(kwargs['error'], execnet.RemoteError)
def test_crash_killed(self, testdir, mysetup): def test_crash_killed(self, testdir, mysetup):
if not hasattr(py.std.os, 'kill'): if not hasattr(py.std.os, 'kill'):

View File

@@ -1,3 +1,3 @@
# #
__version__ = "1.1" __version__ = "1.3"

View File

@@ -1,6 +1,5 @@
import py import py
from py._test.session import Session from py._test import session
from py._test import outcome
from xdist.nodemanage import NodeManager from xdist.nodemanage import NodeManager
queue = py.builtin._tryimport('queue', 'Queue') queue = py.builtin._tryimport('queue', 'Queue')
@@ -21,7 +20,7 @@ class LoopState(object):
# waiting for a host to become ready. # waiting for a host to become ready.
self.dowork = True self.dowork = True
self.shuttingdown = False self.shuttingdown = False
self.testsfailed = False self.testsfailed = 0
def __repr__(self): def __repr__(self):
return "<LoopState exitstatus=%r shuttingdown=%r len(colitems)=%d>" % ( return "<LoopState exitstatus=%r shuttingdown=%r len(colitems)=%d>" % (
@@ -32,7 +31,7 @@ class LoopState(object):
if report.when != "teardown": # otherwise we already managed it if report.when != "teardown": # otherwise we already managed it
self.dsession.removeitem(report.item, report.node) self.dsession.removeitem(report.item, report.node)
if report.failed: if report.failed:
self.testsfailed = True self.testsfailed += 1
def pytest_collectreport(self, report): def pytest_collectreport(self, report):
if report.passed: if report.passed:
@@ -59,10 +58,7 @@ class LoopState(object):
if pending: if pending:
self.dowork = False # avoid busywait, nodes still have work self.dowork = False # avoid busywait, nodes still have work
class ExitFirstInterrupt(KeyboardInterrupt): class DSession(session.Session):
pass
class DSession(Session):
""" """
Session drives the collection and running of tests Session drives the collection and running of tests
and generates test events for reporters. and generates test events for reporters.
@@ -132,14 +128,18 @@ class DSession(Session):
call(**kwargs) call(**kwargs)
# termination conditions # termination conditions
if ((loopstate.testsfailed and self.config.option.exitfirst) or maxfail = self.config.getvalue("maxfail")
if (not self.node2pending or
(loopstate.testsfailed and maxfail and
loopstate.testsfailed >= maxfail) or
(not self.item2nodes and not colitems and not self.queue.qsize())): (not self.item2nodes and not colitems and not self.queue.qsize())):
if self.config.option.exitfirst: if maxfail and loopstate.testsfailed >= maxfail:
raise ExitFirstInterrupt() raise self.Interrupted("stopping after %d failures" % (
loopstate.testsfailed))
self.triggershutdown() self.triggershutdown()
loopstate.shuttingdown = True loopstate.shuttingdown = True
elif not self.node2pending: if not self.node2pending:
loopstate.exitstatus = outcome.EXIT_NOHOSTS loopstate.exitstatus = session.EXIT_NOHOSTS
def loop_once_shutdown(self, loopstate): def loop_once_shutdown(self, loopstate):
# once we are in shutdown mode we dont send # once we are in shutdown mode we dont send
@@ -153,16 +153,16 @@ class DSession(Session):
self.config.hook.pytest_runtest_logreport(**kwargs) self.config.hook.pytest_runtest_logreport(**kwargs)
elif eventname == "pytest_internalerror": elif eventname == "pytest_internalerror":
self.config.hook.pytest_internalerror(**kwargs) self.config.hook.pytest_internalerror(**kwargs)
loopstate.exitstatus = outcome.EXIT_INTERNALERROR loopstate.exitstatus = session.EXIT_INTERNALERROR
elif eventname == "pytest__teardown_final_logerror": elif eventname == "pytest__teardown_final_logerror":
self.config.hook.pytest__teardown_final_logerror(**kwargs) self.config.hook.pytest__teardown_final_logerror(**kwargs)
loopstate.exitstatus = outcome.EXIT_TESTSFAILED loopstate.exitstatus = session.EXIT_TESTSFAILED
if not self.node2pending: if not self.node2pending:
# finished # finished
if loopstate.testsfailed: if loopstate.testsfailed:
loopstate.exitstatus = outcome.EXIT_TESTSFAILED loopstate.exitstatus = session.EXIT_TESTSFAILED
else: else:
loopstate.exitstatus = outcome.EXIT_OK loopstate.exitstatus = session.EXIT_OK
#self.config.pluginmanager.unregister(loopstate) #self.config.pluginmanager.unregister(loopstate)
def _initloopstate(self, colitems): def _initloopstate(self, colitems):
@@ -181,17 +181,14 @@ class DSession(Session):
break break
except KeyboardInterrupt: except KeyboardInterrupt:
excinfo = py.code.ExceptionInfo() excinfo = py.code.ExceptionInfo()
if excinfo.errisinstance(ExitFirstInterrupt): self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
exitstatus = outcome.EXIT_TESTSFAILED exitstatus = session.EXIT_INTERRUPTED
else:
self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
exitstatus = outcome.EXIT_INTERRUPTED
except: except:
self.config.pluginmanager.notify_exception() self.config.pluginmanager.notify_exception()
exitstatus = outcome.EXIT_INTERNALERROR exitstatus = session.EXIT_INTERNALERROR
self.config.pluginmanager.unregister(loopstate) self.config.pluginmanager.unregister(loopstate)
if exitstatus == 0 and self._testsfailed: if exitstatus == 0 and self._testsfailed:
exitstatus = outcome.EXIT_TESTSFAILED exitstatus = session.EXIT_TESTSFAILED
return exitstatus return exitstatus
def triggershutdown(self): def triggershutdown(self):

View File

@@ -31,13 +31,12 @@ class MyPickler(Pickler):
""" Pickler with a custom memoize() """ Pickler with a custom memoize()
to take care of unique ID creation. to take care of unique ID creation.
See the usage in ImmutablePickler See the usage in ImmutablePickler
XXX we could probably extend Pickler
and Unpickler classes to directly
update the other'S memos.
""" """
def __init__(self, file, protocol, uneven): def __init__(self, immo, file, protocol, uneven):
Pickler.__init__(self, file, protocol) Pickler.__init__(self, file, protocol)
self.uneven = uneven self.uneven = uneven
self._unpicklememo = immo._unpicklememo
self.memo = immo._picklememo
def memoize(self, obj): def memoize(self, obj):
if self.fast: if self.fast:
@@ -47,6 +46,10 @@ class MyPickler(Pickler):
key = memo_len * 2 + self.uneven key = memo_len * 2 + self.uneven
self.write(self.put(key)) self.write(self.put(key))
self.memo[id(obj)] = key, obj self.memo[id(obj)] = key, obj
key = makekey(key)
if key in self._unpicklememo:
assert self._unpicklememo[key] is obj
dict.__setitem__(self._unpicklememo, key, obj)
#if sys.version_info < (3,0): #if sys.version_info < (3,0):
# def save_string(self, obj, pack=struct.pack): # def save_string(self, obj, pack=struct.pack):
@@ -54,6 +57,15 @@ class MyPickler(Pickler):
# self.save_unicode(obj, pack=pack) # self.save_unicode(obj, pack=pack)
# Pickler.dispatch[str] = save_string # Pickler.dispatch[str] = save_string
class UnpicklingDict(dict):
def __init__(self, picklememo):
super(UnpicklingDict, self).__init__()
self._picklememo = picklememo
def __setitem__(self, key, obj):
super(UnpicklingDict, self).__setitem__(key, obj)
self._picklememo[id(obj)] = (fromkey(key), obj)
class ImmutablePickler: class ImmutablePickler:
def __init__(self, uneven, protocol=0): def __init__(self, uneven, protocol=0):
""" ImmutablePicklers are instantiated in Pairs. """ ImmutablePicklers are instantiated in Pairs.
@@ -64,7 +76,7 @@ class ImmutablePickler:
parameter. parameter.
""" """
self._picklememo = {} self._picklememo = {}
self._unpicklememo = {} self._unpicklememo = UnpicklingDict(self._picklememo)
self._protocol = protocol self._protocol = protocol
self.uneven = uneven and 1 or 0 self.uneven = uneven and 1 or 0
@@ -73,18 +85,13 @@ class ImmutablePickler:
# which be the case e.g. if you want to pickle # which be the case e.g. if you want to pickle
# from a forked process back to the original # from a forked process back to the original
f = py.io.BytesIO() f = py.io.BytesIO()
pickler = MyPickler(f, self._protocol, uneven=self.uneven) pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
pickler.memo = self._picklememo
pickler.memoize(obj) pickler.memoize(obj)
self._updateunpicklememo()
def dumps(self, obj): def dumps(self, obj):
f = py.io.BytesIO() f = py.io.BytesIO()
pickler = MyPickler(f, self._protocol, uneven=self.uneven) pickler = MyPickler(self, f, self._protocol, uneven=self.uneven)
pickler.memo = self._picklememo
pickler.dump(obj) pickler.dump(obj)
if obj is not None:
self._updateunpicklememo()
#print >>debug, "dumped", obj #print >>debug, "dumped", obj
#print >>debug, "picklememo", self._picklememo #print >>debug, "picklememo", self._picklememo
return f.getvalue() return f.getvalue()
@@ -94,21 +101,10 @@ class ImmutablePickler:
unpickler = Unpickler(f) unpickler = Unpickler(f)
unpickler.memo = self._unpicklememo unpickler.memo = self._unpicklememo
res = unpickler.load() res = unpickler.load()
self._updatepicklememo()
#print >>debug, "loaded", res #print >>debug, "loaded", res
#print >>debug, "unpicklememo", self._unpicklememo #print >>debug, "unpicklememo", self._unpicklememo
return res return res
def _updatepicklememo(self):
for x, obj in self._unpicklememo.items():
self._picklememo[id(obj)] = (fromkey(x), obj)
def _updateunpicklememo(self):
for key,obj in self._picklememo.values():
key = makekey(key)
if key in self._unpicklememo:
assert self._unpicklememo[key] is obj
self._unpicklememo[key] = obj
NO_ENDMARKER_WANTED = object() NO_ENDMARKER_WANTED = object()

25
xdist/newhooks.py Normal file
View File

@@ -0,0 +1,25 @@
def pytest_gwmanage_newgateway(gateway, platinfo):
""" called on new raw gateway creation. """
def pytest_gwmanage_rsyncstart(source, gateways):
""" called before rsyncing a directory to remote gateways takes place. """
def pytest_gwmanage_rsyncfinish(source, gateways):
""" called after rsyncing a directory to remote gateways takes place. """
def pytest_configure_node(node):
""" configure node information before it gets instantiated. """
def pytest_testnodeready(node):
""" Test Node is ready to operate. """
def pytest_testnodedown(node, error):
""" Test Node is down. """
def pytest_rescheduleitems(items):
""" reschedule Items from a node that went down. """
def pytest_looponfailinfo(failreports, rootdirs):
""" info for repeating failing tests. """

View File

@@ -58,7 +58,7 @@ class NodeManager(object):
self.rsync_roots() self.rsync_roots()
self.trace("setting up nodes") self.trace("setting up nodes")
for gateway in self.gwmanager.group: for gateway in self.gwmanager.group:
node = TXNode(gateway, self.config, putevent) node = TXNode(self, gateway, self.config, putevent)
gateway.node = node # to keep node alive gateway.node = node # to keep node alive
self.trace("started node %r" % node) self.trace("started node %r" % node)

View File

@@ -120,11 +120,11 @@ Specifying test exec environments in a conftest.py
Instead of specifying command line options, you can Instead of specifying command line options, you can
put options values in a ``conftest.py`` file like this:: put options values in a ``conftest.py`` file like this::
pytest_option_tx = ['ssh=myhost//python=python2.5', 'popen//python=python2.5'] option_tx = ['ssh=myhost//python=python2.5', 'popen//python=python2.5']
pytest_option_dist = True option_dist = True
Any commandline ``--tx`` specifictions will add to the list of available execution Any commandline ``--tx`` specifictions will add to the list of
environments. available execution environments.
Specifying "rsync" dirs in a conftest.py Specifying "rsync" dirs in a conftest.py
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
@@ -173,6 +173,16 @@ def pytest_addoption(parser):
group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
help="add directory for rsyncing to remote tx nodes.") help="add directory for rsyncing to remote tx nodes.")
# -------------------------------------------------------------------------
# distributed testing hooks
# -------------------------------------------------------------------------
def pytest_addhooks(pluginmanager):
from xdist import newhooks
pluginmanager.addhooks(newhooks)
# -------------------------------------------------------------------------
# distributed testing initialization
# -------------------------------------------------------------------------
def pytest_configure(config): def pytest_configure(config):
if config.option.numprocesses: if config.option.numprocesses:
config.option.dist = "load" config.option.dist = "load"

View File

@@ -24,7 +24,6 @@ class LooponfailingSession(Session):
def main(self, initialitems): def main(self, initialitems):
try: try:
self.loopstate = loopstate = LoopState([]) self.loopstate = loopstate = LoopState([])
self.remotecontrol.setup()
while 1: while 1:
self.loop_once(loopstate) self.loop_once(loopstate)
if not loopstate.colitems and loopstate.wasfailing: if not loopstate.colitems and loopstate.wasfailing:
@@ -34,10 +33,10 @@ class LooponfailingSession(Session):
print print
def loop_once(self, loopstate): def loop_once(self, loopstate):
self.remotecontrol.setup()
colitems = loopstate.colitems colitems = loopstate.colitems
loopstate.wasfailing = colitems and len(colitems) loopstate.wasfailing = colitems and len(colitems)
loopstate.colitems = self.remotecontrol.runsession(colitems or ()) loopstate.colitems = self.remotecontrol.runsession(colitems or ())
self.remotecontrol.setup()
class LoopState: class LoopState:
def __init__(self, colitems=None): def __init__(self, colitems=None):

View File

@@ -3,7 +3,7 @@
""" """
import py import py
from xdist.mypickle import PickleChannel from xdist.mypickle import PickleChannel
from py._test import outcome from py._test.session import Session
class TXNode(object): class TXNode(object):
""" Represents a Test Execution environment in the controlling process. """ Represents a Test Execution environment in the controlling process.
@@ -13,11 +13,13 @@ class TXNode(object):
""" """
ENDMARK = -1 ENDMARK = -1
def __init__(self, gateway, config, putevent): def __init__(self, nodemanager, gateway, config, putevent):
self.nodemanager = nodemanager
self.config = config self.config = config
self.putevent = putevent self.putevent = putevent
self.gateway = gateway self.gateway = gateway
self.channel = install_slave(gateway, config) self.slaveinput = {}
self.channel = install_slave(self)
self.channel.setcallback(self.callback, endmarker=self.ENDMARK) self.channel.setcallback(self.callback, endmarker=self.ENDMARK)
self._down = False self._down = False
@@ -43,7 +45,7 @@ class TXNode(object):
err = self.channel._getremoteerror() err = self.channel._getremoteerror()
if not self._down: if not self._down:
if not err or isinstance(err, EOFError): if not err or isinstance(err, EOFError):
err = "Not properly terminated" err = "Not properly terminated" # lost connection?
self.notify("pytest_testnodedown", node=self, error=err) self.notify("pytest_testnodedown", node=self, error=err)
self._down = True self._down = True
return return
@@ -52,7 +54,9 @@ class TXNode(object):
self.notify("pytest_testnodeready", node=self) self.notify("pytest_testnodeready", node=self)
elif eventname == "slavefinished": elif eventname == "slavefinished":
self._down = True self._down = True
self.notify("pytest_testnodedown", error=None, node=self) self.slaveoutput = kwargs['slaveoutput']
error = kwargs['error']
self.notify("pytest_testnodedown", error=error, node=self)
elif eventname in ("pytest_runtest_logreport", elif eventname in ("pytest_runtest_logreport",
"pytest__teardown_final_logerror"): "pytest__teardown_final_logerror"):
kwargs['report'].node = self kwargs['report'].node = self
@@ -80,32 +84,44 @@ class TXNode(object):
else: else:
self.channel.send(None) self.channel.send(None)
# setting up slave code # configuring and setting up slave node
def install_slave(gateway, config): def install_slave(node):
channel = gateway.remote_exec(source=""" channel = node.gateway.remote_exec(source="""
import os, sys import os, sys
sys.path.insert(0, os.getcwd()) sys.path.insert(0, os.getcwd())
from xdist.mypickle import PickleChannel from xdist.mypickle import PickleChannel
from xdist.txnode import SlaveNode from xdist.txnode import SlaveSession
channel.send("basicimport") channel.send("basicimport")
channel = PickleChannel(channel) channel = PickleChannel(channel)
slavenode = SlaveNode(channel) import py
slavenode.run() config, slaveinput, basetemp, nodeid = channel.receive()
config.slaveinput = slaveinput
config.slaveoutput = {}
if basetemp:
config.basetemp = py.path.local(basetemp)
config.nodeid = nodeid
config.pluginmanager.do_configure(config)
session = SlaveSession(config, channel, nodeid)
session.dist_main()
""") """)
channel.receive() channel.receive()
channel = PickleChannel(channel) channel = PickleChannel(channel)
basetemp = None basetemp = None
if gateway.spec.popen: config = node.config
config.hook.pytest_configure_node(node=node)
if node.gateway.spec.popen:
popenbase = config.ensuretemp("popen") popenbase = config.ensuretemp("popen")
basetemp = py.path.local.make_numbered_dir(prefix="slave-", basetemp = py.path.local.make_numbered_dir(prefix="slave-",
keep=0, rootdir=popenbase) keep=0, rootdir=popenbase)
basetemp = str(basetemp) basetemp = str(basetemp)
channel.send((config, basetemp, gateway.id)) channel.send((config, node.slaveinput, basetemp, node.gateway.id))
return channel return channel
class SlaveNode(object): class SlaveSession(Session):
def __init__(self, channel): def __init__(self, config, channel, nodeid):
self.channel = channel self.channel = channel
self.nodeid = nodeid
super(SlaveSession, self).__init__(config=config)
def __repr__(self): def __repr__(self):
return "<%s channel=%s>" %(self.__class__.__name__, self.channel) return "<%s channel=%s>" %(self.__class__.__name__, self.channel)
@@ -119,37 +135,31 @@ class SlaveNode(object):
def pytest__teardown_final_logerror(self, report): def pytest__teardown_final_logerror(self, report):
self.sendevent("pytest__teardown_final_logerror", report=report) self.sendevent("pytest__teardown_final_logerror", report=report)
def run(self): def pytest_keyboard_interrupt(self, excinfo):
channel = self.channel self._slaveerror = "SIGINT"
self.config, basetemp, self.nodeid = channel.receive()
if basetemp: def pytest_internalerror(self, excrepr):
self.config.basetemp = py.path.local(basetemp) self._slaveerror = "internal-error"
self.config.pluginmanager.do_configure(self.config) self.sendevent("pytest_internalerror", excrepr=excrepr)
self.config.pluginmanager.register(self)
def dist_main(self):
self.runner = self.config.pluginmanager.getplugin("pytest_runner") self.runner = self.config.pluginmanager.getplugin("pytest_runner")
self.sendevent("slaveready") self.sendevent("slaveready")
try: self.main(None)
self.config.hook.pytest_sessionstart(session=self) error = getattr(self, '_slaveerror', None)
while 1: self.sendevent("slavefinished", error=error,
task = channel.receive() slaveoutput=self.config.slaveoutput)
if task is None:
break def _mainloop(self, colitems):
if isinstance(task, list): while 1:
for item in task: task = self.channel.receive()
self.run_single(item=item) if task is None:
else: break
self.run_single(item=task) if isinstance(task, list):
self.config.hook.pytest_sessionfinish( for item in task:
session=self, self.run_single(item=item)
exitstatus=outcome.EXIT_OK) else:
except KeyboardInterrupt: self.run_single(item=task)
raise
except:
er = py.code.ExceptionInfo().getrepr(funcargs=True, showlocals=True)
self.sendevent("pytest_internalerror", excrepr=er)
raise
else:
self.sendevent("slavefinished")
def run_single(self, item): def run_single(self, item):
call = self.runner.CallInfo(item._reraiseunpicklingproblem, when='setup') call = self.runner.CallInfo(item._reraiseunpicklingproblem, when='setup')