Compare commits

...

25 Commits
1.9 ... 1.10

Author SHA1 Message Date
holger krekel
b0732f99a6 final 2014-01-29 13:10:20 +01:00
holger krekel
ab987d512c make pexpect specific environments so that the canonical ones also run on windows. Also add a little sleep to a pexpect-related tests in the hope it helps the test consistently passing (Bah). 2014-01-27 19:47:37 +01:00
holger krekel
ffa156ced5 refine distribution fractions and make it dependend on the duration
of the last test in a node's set.
2014-01-27 12:39:43 +01:00
holger krekel
e26d4486fc send multiple "to test" indices in one network message to a slave
and improve heuristics for sending chunks where the chunksize
depends on the number of remaining tests rather than fixed numbers.
This reduces the number of master -> node messages (but not the
reverse direction)
2014-01-27 11:37:40 +01:00
holger krekel
18a30fab7d fix issue419: work with collection indices instead of node ids.
This reduces network message size.
2014-01-27 11:37:33 +01:00
holger krekel
dd73d132b1 remove unneccesary item2nodes data structure for load scheduling 2014-01-27 11:37:27 +01:00
holger krekel
a22e0384a1 a failing test for pytest issue419 2014-01-27 11:37:26 +01:00
holger krekel
8eabc7d5df Merged in lukaszb/pytest-xdist/lukaszb/readmetxt-typo-1388766852691 (pull request #7)
README.txt typo
2014-01-04 21:41:34 +01:00
Lukasz Balcerzak
dbf7932514 README.txt typo 2014-01-03 16:34:15 +00:00
holger krekel
6355c3e8d0 add glob support for rsyncignores, add command line option to pass
additional rsyncignores. Thanks Anatoly Bubenkov.
2013-12-06 13:26:09 +01:00
holger krekel
c505f00320 Merged in paylogic/pytest-xdist/rsyncignore-option (pull request #5)
add glob support for rsyncignore. add command line option for rsyncignore
2013-12-06 13:25:19 +01:00
Anatoly Bubenkov
7e066be35e docs fixed 2013-12-06 10:21:32 +01:00
Anatoly Bubenkov
27f1d8049b merge with mainline 2013-12-06 10:20:03 +01:00
Anatoly Bubenkov
bcf1c85f44 docs fixed 2013-12-06 10:19:18 +01:00
Anatoly Bubenkov
a3fac52e85 more readability for return 2013-12-05 23:22:49 +01:00
Anatoly Bubenkov
152f965a70 fix ignores mixing 2013-12-05 16:35:56 +01:00
Anatoly Bubenkov
027b3f51ed change metavars 2013-12-05 15:31:52 +01:00
Anatoly Bubenkov
f4b86f0127 add glob support for rsyncignore. add command line option for rsyncignore 2013-12-05 15:19:31 +01:00
holger krekel
8146b27671 now that pytest_runtest_logstart is sent to the master again,
disable showing of filenames explicitely with the terminalreporter
2013-11-21 15:00:06 +01:00
holger krekel
a488cc5add merge 2013-11-19 12:21:31 +01:00
holger krekel
aad1aace52 merge fix pytest issue382 - produce "pytest_runtest_logstart" event again
in master. Thanks Aron Curzon.
2013-11-19 12:08:55 +01:00
curzona
82d357ab1a Uncomment logstart hook in remote.py and slavemanage.py 2013-11-09 12:10:13 -08:00
holger krekel
ca54911481 ignore directories and pyc files for changes (editors write tmp/swap
files etc., which also affects mtime of directory)
2013-10-07 10:28:08 +02:00
holger krekel
8e88261c15 ignore dot files for file changes (editors write tmp/swap files etc.) 2013-10-07 10:20:24 +02:00
holger krekel
6045195f16 Added tag 1.9 for changeset 5c5cb6d59e12 2013-10-04 14:35:34 +02:00
18 changed files with 226 additions and 140 deletions

View File

@@ -14,8 +14,14 @@ syntax:glob
*.class *.class
*.orig *.orig
*.sublime-*
.Python
build/ build/
dist/ dist/
include/
lib/
bin/
pytest_xdist.egg-info pytest_xdist.egg-info
issue/ issue/
3rdparty/ 3rdparty/

View File

@@ -13,3 +13,4 @@ cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8 0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8 0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
1d27987c267577899350a25ba5828d55d87083ad 1.8 1d27987c267577899350a25ba5828d55d87083ad 1.8
5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9

View File

@@ -1,3 +1,23 @@
1.10
-------------------------
- add glob support for rsyncignores, add command line option to pass
additional rsyncignores. Thanks Anatoly Bubenkov.
- fix pytest issue382 - produce "pytest_runtest_logstart" event again
in master. Thanks Aron Curzon.
- fix pytest issue419 by sending/receiving indices into the test
collection instead of node ids (which are not neccessarily unique
for functions parametrized with duplicate values)
- send multiple "to test" indices in one network message to a slave
and improve heuristics for sending chunks where the chunksize
depends on the number of remaining tests rather than fixed numbers.
This reduces the number of master -> node messages (but not the
reverse direction)
1.9 1.9
------------------------- -------------------------

View File

@@ -80,7 +80,7 @@ Running tests in a boxed subprocess
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
If you have tests involving C or C++ libraries you might have to deal If you have tests involving C or C++ libraries you might have to deal
with tests crashing the process. For this case you max use the boxing with tests crashing the process. For this case you may use the boxing
options:: options::
py.test --boxed py.test --boxed
@@ -122,6 +122,13 @@ py.test references tests as a fully qualified python
module path. **You will otherwise get strange errors** module path. **You will otherwise get strange errors**
during setup of the remote side. during setup of the remote side.
You can specify multiple ``--rsyncignore`` glob-patterns
to be ignored when file are sent to the remote side.
There are also internal ignores: .*, *.pyc, *.pyo, *~
Those you cannot override using rsyncignore command-line or
ini-file option(s).
Sending tests to remote Socket Servers Sending tests to remote Socket Servers
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

View File

@@ -2,7 +2,7 @@ from setuptools import setup
setup( setup(
name="pytest-xdist", name="pytest-xdist",
version='1.9', version='1.10',
description='py.test xdist plugin for distributed testing and loop-on-failing modes', description='py.test xdist plugin for distributed testing and loop-on-failing modes',
long_description=open('README.txt').read(), long_description=open('README.txt').read(),
license='MIT', license='MIT',
@@ -13,7 +13,7 @@ setup(
packages = ['xdist'], packages = ['xdist'],
entry_points = {'pytest11': ['xdist = xdist.plugin'],}, entry_points = {'pytest11': ['xdist = xdist.plugin'],},
zip_safe=False, zip_safe=False,
install_requires = ['execnet>=1.1', 'pytest>=2.3.5'], install_requires = ['execnet>=1.1', 'pytest>=2.4.2'],
classifiers=[ classifiers=[
'Development Status :: 5 - Production/Stable', 'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers', 'Intended Audience :: Developers',

View File

@@ -256,6 +256,7 @@ class TestDistribution:
time.sleep(10) time.sleep(10)
""") """)
child = testdir.spawn_pytest("-n1") child = testdir.spawn_pytest("-n1")
py.std.time.sleep(0.1)
child.expect(".*test session starts.*") child.expect(".*test session starts.*")
child.kill(2) # keyboard interrupt child.kill(2) # keyboard interrupt
child.expect(".*KeyboardInterrupt.*") child.expect(".*KeyboardInterrupt.*")

View File

@@ -6,6 +6,7 @@ from xdist.dsession import (
) )
from _pytest import main as outcome from _pytest import main as outcome
import py import py
import pytest
import execnet import execnet
XSpec = execnet.XSpec XSpec = execnet.XSpec
@@ -28,15 +29,12 @@ class MockNode:
self.sent = [] self.sent = []
self.gateway = MockGateway() self.gateway = MockGateway()
def send_runtest(self, nodeid): def send_runtest_some(self, indices):
self.sent.append(nodeid) self.sent.extend(indices)
def send_runtest_all(self): def send_runtest_all(self):
self.sent.append("ALL") self.sent.append("ALL")
def sendlist(self, items):
self.sent.extend(items)
def shutdown(self): def shutdown(self):
self._shutdown=True self._shutdown=True
@@ -63,9 +61,9 @@ class TestEachScheduling:
assert sched.tests_finished() assert sched.tests_finished()
assert node1.sent == ['ALL'] assert node1.sent == ['ALL']
assert node2.sent == ['ALL'] assert node2.sent == ['ALL']
sched.remove_item(node1, collection[0]) sched.remove_item(node1, 0)
assert sched.tests_finished() assert sched.tests_finished()
sched.remove_item(node2, collection[0]) sched.remove_item(node2, 0)
assert sched.tests_finished() assert sched.tests_finished()
def test_schedule_remove_node(self): def test_schedule_remove_node(self):
@@ -104,7 +102,7 @@ class TestLoadScheduling:
assert len(node1.sent) == 1 assert len(node1.sent) == 1
assert len(node2.sent) == 1 assert len(node2.sent) == 1
x = sorted(node1.sent + node2.sent) x = sorted(node1.sent + node2.sent)
assert x == collection assert x == [0, 1]
sched.remove_item(node1, node1.sent[0]) sched.remove_item(node1, node1.sent[0])
sched.remove_item(node2, node2.sent[0]) sched.remove_item(node2, node2.sent[0])
assert sched.tests_finished() assert sched.tests_finished()
@@ -116,23 +114,22 @@ class TestLoadScheduling:
node2 = MockNode() node2 = MockNode()
sched.addnode(node1) sched.addnode(node1)
sched.addnode(node2) sched.addnode(node2)
sched.ITEM_CHUNKSIZE = 2 col = ["xyz"] * (3)
col = ["xyz"] * (2*sched.ITEM_CHUNKSIZE +1)
sched.addnode_collection(node1, col) sched.addnode_collection(node1, col)
sched.addnode_collection(node2, col) sched.addnode_collection(node2, col)
sched.init_distribute() sched.init_distribute()
#assert not sched.tests_finished() #assert not sched.tests_finished()
sent1 = node1.sent sent1 = node1.sent
sent2 = node2.sent sent2 = node2.sent
chunkitems = col[:sched.ITEM_CHUNKSIZE] chunkitems = col[:1]
assert sent1 == chunkitems assert (sent1 == [0] and sent2 == [1]) or (
assert sent2 == chunkitems sent1 == [1] and sent2 == [0])
assert sched.node2pending[node1] == sent1 assert sched.node2pending[node1] == sent1
assert sched.node2pending[node2] == sent2 assert sched.node2pending[node2] == sent2
assert len(sched.pending) == 1 assert len(sched.pending) == 1
for node in (node1, node2): for node in (node1, node2):
for i in range(sched.ITEM_CHUNKSIZE): for i in sched.node2pending[node]:
sched.remove_item(node, "xyz") sched.remove_item(node, i)
assert not sched.pending assert not sched.pending
def test_add_remove_node(self): def test_add_remove_node(self):
@@ -209,3 +206,16 @@ def test_report_collection_diff_different():
report_collection_diff(from_collection, to_collection, 1, 2) report_collection_diff(from_collection, to_collection, 1, 2)
except AssertionError as e: except AssertionError as e:
assert py.builtin._totext(e) == error_message assert py.builtin._totext(e) == error_message
@pytest.mark.xfail(reason="duplicate test ids not supported yet")
def test_pytest_issue419(testdir):
testdir.makepyfile("""
import pytest
@pytest.mark.parametrize('birth_year', [1988, 1988, ])
def test_2011_table(birth_year):
pass
""")
reprec = testdir.inline_run("-n1")
reprec.assertoutcome(passed=2)
assert 0

View File

@@ -14,6 +14,10 @@ class TestStatRecorder:
changed = sd.check() changed = sd.check()
assert changed assert changed
(hello + "c").write("hello")
changed = sd.check()
assert not changed
p = tmp.ensure("new.py") p = tmp.ensure("new.py")
changed = sd.check() changed = sd.check()
assert changed assert changed
@@ -36,6 +40,12 @@ class TestStatRecorder:
changed = sd.check() changed = sd.check()
assert changed assert changed
def test_dirchange(self, tmpdir):
tmp = tmpdir
hello = tmp.ensure("dir", "hello.py")
sd = StatRecorder([tmp])
assert not sd.fil(tmp.join("dir"))
def test_filechange_deletion_race(self, tmpdir, monkeypatch): def test_filechange_deletion_race(self, tmpdir, monkeypatch):
tmp = tmpdir tmp = tmpdir
sd = StatRecorder([tmp]) sd = StatRecorder([tmp])
@@ -63,12 +73,10 @@ class TestStatRecorder:
pycfile = hello + "c" pycfile = hello + "c"
pycfile.ensure() pycfile.ensure()
changed = sd.check()
assert changed
hello.write("world") hello.write("world")
changed = sd.check() changed = sd.check()
assert changed assert changed
assert not pycfile.check()
def test_waitonchange(self, tmpdir, monkeypatch): def test_waitonchange(self, tmpdir, monkeypatch):
tmp = tmpdir tmp = tmpdir

View File

@@ -46,6 +46,11 @@ class TestDistOptions:
assert nm.roots assert nm.roots
assert testdir.tmpdir in nm.roots assert testdir.tmpdir in nm.roots
def test_getrsyncignore(self, testdir):
config = testdir.parseconfigure('--rsyncignore=fo*')
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert 'fo*' in nm.rsyncoptions['ignores']
def test_getrsyncdirs_with_conftest(self, testdir): def test_getrsyncdirs_with_conftest(self, testdir):
p = py.path.local() p = py.path.local()
for bn in 'x y z'.split(): for bn in 'x y z'.split():

View File

@@ -154,8 +154,11 @@ class TestSlaveInteractor:
assert ev.kwargs['topdir'] == slave.testdir.tmpdir assert ev.kwargs['topdir'] == slave.testdir.tmpdir
ids = ev.kwargs['ids'] ids = ev.kwargs['ids']
assert len(ids) == 1 assert len(ids) == 1
slave.sendcommand("runtests", ids=ids) slave.sendcommand("runtests", indices=list(range(len(ids))))
slave.sendcommand("shutdown") slave.sendcommand("shutdown")
ev = slave.popevent("logstart")
assert ev.kwargs["nodeid"].endswith("test_func")
assert len(ev.kwargs["location"]) == 3
ev = slave.popevent("testreport") # setup ev = slave.popevent("testreport") # setup
ev = slave.popevent("testreport") ev = slave.popevent("testreport")
assert ev.name == "testreport" assert ev.name == "testreport"

View File

@@ -113,7 +113,7 @@ class TestHRSync:
source.ensure(".svn", "entries") source.ensure(".svn", "entries")
source.ensure(".somedotfile", "moreentries") source.ensure(".somedotfile", "moreentries")
source.ensure("somedir", "editfile~") source.ensure("somedir", "editfile~")
syncer = HostRSync(source) syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
l = list(source.visit(rec=syncer.filter, l = list(source.visit(rec=syncer.filter,
fil=syncer.filter)) fil=syncer.filter))
assert len(l) == 3 assert len(l) == 3
@@ -197,12 +197,15 @@ class TestNodeManager:
dir5 = source.ensure("dir5", "dir6", "bogus") dir5 = source.ensure("dir5", "dir6", "bogus")
dirf = source.ensure("dir5", "file") dirf = source.ensure("dir5", "file")
dir2.ensure("hello") dir2.ensure("hello")
dirfoo = source.ensure("foo", "bar")
dirbar = source.ensure("bar", "foo")
source.join("tox.ini").write(py.std.textwrap.dedent(""" source.join("tox.ini").write(py.std.textwrap.dedent("""
[pytest] [pytest]
rsyncdirs = dir1 dir5 rsyncdirs = dir1 dir5
rsyncignore = dir1/dir2 dir5/dir6 rsyncignore = dir1/dir2 dir5/dir6 foo*
""")) """))
config = testdir.parseconfig(source) config = testdir.parseconfig(source)
config.option.rsyncignore = ['bar']
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.makegateways() nodemanager.makegateways()
nodemanager.rsync_roots() nodemanager.rsync_roots()
@@ -210,6 +213,8 @@ class TestNodeManager:
assert not dest.join("dir1", "dir2").check() assert not dest.join("dir1", "dir2").check()
assert dest.join("dir5","file").check() assert dest.join("dir5","file").check()
assert not dest.join("dir6").check() assert not dest.join("dir6").check()
assert not dest.join('foo').check()
assert not dest.join('bar').check()
def test_optimise_popen(self, testdir, mysetup): def test_optimise_popen(self, testdir, mysetup):
source, dest = mysetup.source, mysetup.dest source, dest = mysetup.source, mysetup.dest

17
tox.ini
View File

@@ -1,25 +1,26 @@
[tox] [tox]
envlist=py26,py32,py33,py27,py26,py26-old,py33-old envlist=py26,py32,py33,py27,py27-pexpect,py33-pexpect,py26,py26-old,py33-old
[testenv] [testenv]
changedir=testing changedir=testing
deps=pytest>=2.4.2 deps=pytest>=2.5.1
commands= py.test --junitxml={envlogdir}/junit-{envname}.xml [] commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
[testenv:py27] [testenv:py27-pexpect]
deps= deps={[testenv]deps}
pytest>=2.4.2 pexpect
[testenv:py33-pexpect]
deps={[testenv]deps}
pexpect pexpect
[testenv:py26-old] [testenv:py26-old]
deps= deps=
pytest==2.3.5 pytest==2.4.2
pexpect
[testenv:py33-old] [testenv:py33-old]
basepython = python3.3 basepython = python3.3
deps= deps=
pytest==2.3.5 pytest==2.4.2
[pytest] [pytest]
addopts = -rsfxX addopts = -rsfxX

View File

@@ -1,2 +1,2 @@
# #
__version__ = '1.9' __version__ = '1.10'

View File

@@ -40,15 +40,15 @@ class EachScheduling:
if len(self.node2pending) >= self.numnodes: if len(self.node2pending) >= self.numnodes:
self.collection_is_completed = True self.collection_is_completed = True
def remove_item(self, node, item): def remove_item(self, node, item_index, duration=0):
self.node2pending[node].remove(item) self.node2pending[node].remove(item_index)
def remove_node(self, node): def remove_node(self, node):
# KeyError if we didn't get an addnode() yet # KeyError if we didn't get an addnode() yet
pending = self.node2pending.pop(node) pending = self.node2pending.pop(node)
if not pending: if not pending:
return return
crashitem = pending.pop(0) crashitem = self.node2collection[node][pending.pop(0)]
# XXX what about the rest of pending? # XXX what about the rest of pending?
return crashitem return crashitem
@@ -56,12 +56,9 @@ class EachScheduling:
assert self.collection_is_completed assert self.collection_is_completed
for node, pending in self.node2pending.items(): for node, pending in self.node2pending.items():
node.send_runtest_all() node.send_runtest_all()
pending[:] = self.node2collection[node] pending[:] = range(len(self.node2collection[node]))
class LoadScheduling: class LoadScheduling:
LOAD_THRESHOLD_NEWITEMS = 5
ITEM_CHUNKSIZE = 10
def __init__(self, numnodes, log=None): def __init__(self, numnodes, log=None):
self.numnodes = numnodes self.numnodes = numnodes
self.node2pending = {} self.node2pending = {}
@@ -94,47 +91,46 @@ class LoadScheduling:
if len(self.node2collection) >= self.numnodes: if len(self.node2collection) >= self.numnodes:
self.collection_is_completed = True self.collection_is_completed = True
def remove_item(self, node, item): def remove_item(self, node, item_index, duration=0):
if item not in self.item2nodes: node_pending = self.node2pending[node]
raise AssertionError(item, self.item2nodes) node_pending.remove(item_index)
nodes = self.item2nodes[item]
if node in nodes: # the node might have gone down already
nodes.remove(node)
#if not nodes:
# del self.item2nodes[item]
pending = self.node2pending[node]
pending.remove(item)
# pre-load items-to-test if the node may become ready # pre-load items-to-test if the node may become ready
if self.pending and len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
item = self.pending.pop(0) if self.pending:
pending.append(item) if duration >= 0.1 and node_pending:
self.item2nodes.setdefault(item, []).append(node) # seems the node is doing long-running tests
node.send_runtest(item) # so let's rather wait with sending new items
self.log("items waiting for node: %d" %(len(self.pending))) return
#self.log("item2pending still executing: %s" %(self.item2nodes,)) # how many nodes do we have remaining per node roughly?
#self.log("node2pending: %s" %(self.node2pending,)) num_nodes = len(self.node2pending)
# if our node goes below a heuristic minimum, fill it out to
# heuristic maximum
items_per_node_min = max(
1, len(self.pending) // num_nodes // 4)
items_per_node_max = max(
1, len(self.pending) // num_nodes // 2)
if len(node_pending) <= items_per_node_min:
num_send = items_per_node_max - len(node_pending) + 1
self._send_tests(node, num_send)
self.log("num items waiting for node:", len(self.pending))
#self.log("node2pending:", self.node2pending)
def remove_node(self, node): def remove_node(self, node):
pending = self.node2pending.pop(node) pending = self.node2pending.pop(node)
# KeyError if we didn't get an addnode() yet
for item in pending:
l = self.item2nodes[item]
l.remove(node)
if not l:
del self.item2nodes[item]
if not pending: if not pending:
return return
crashitem = pending.pop(0) # the node must have crashed on the item if there are pending ones
crashitem = self.collection[pending.pop(0)]
self.pending.extend(pending) self.pending.extend(pending)
return crashitem return crashitem
def init_distribute(self): def init_distribute(self):
assert self.collection_is_completed assert self.collection_is_completed
assert not hasattr(self, 'item2nodes')
self.item2nodes = {}
# XXX allow nodes to have different collections # XXX allow nodes to have different collections
first_node, col = list(self.node2collection.items())[0] node_collection_items = list(self.node2collection.items())
for node, collection in self.node2collection.items(): first_node, col = node_collection_items[0]
for node, collection in node_collection_items[1:]:
report_collection_diff( report_collection_diff(
col, col,
collection, collection,
@@ -142,22 +138,28 @@ class LoadScheduling:
node.gateway.id, node.gateway.id,
) )
self.pending = col # all collections are the same, good.
# we now create an index
self.collection = col
self.pending[:] = range(len(col))
if not col: if not col:
return return
available = list(self.node2pending.items()) # how many items per node do we have about?
num_available = self.numnodes items_per_node = len(self.collection) // len(self.node2pending)
max_one_round = num_available * self.ITEM_CHUNKSIZE - 1 # take a fraction of tests for initial distribution
for i, item in enumerate(self.pending): node_chunksize = max(items_per_node // 4, 1)
nodeindex = i % num_available # and initialize each node with a chunk of tests
node, pending = available[nodeindex] for node in self.node2pending:
node.send_runtest(item) self._send_tests(node, node_chunksize)
self.item2nodes.setdefault(item, []).append(node)
pending.append(item)
if i >= max_one_round:
break
del self.pending[:i + 1]
#f = open("/tmp/sent", "w")
def _send_tests(self, node, num):
tests_per_node = self.pending[:num]
#print >>self.f, "sent", node, tests_per_node
if tests_per_node:
del self.pending[:num]
self.node2pending[node].extend(tests_per_node)
node.send_runtest_some(tests_per_node)
def report_collection_diff(from_collection, to_collection, from_id, to_id): def report_collection_diff(from_collection, to_collection, from_id, to_id):
"""Report the collected test difference between two nodes. """Report the collected test difference between two nodes.
@@ -256,7 +258,7 @@ class DSession:
assert callname, kwargs assert callname, kwargs
method = "slave_" + callname method = "slave_" + callname
call = getattr(self, method) call = getattr(self, method)
self.log("calling method: %s(**%s)" % (method, kwargs)) self.log("calling method", method, kwargs)
call(**kwargs) call(**kwargs)
if self.sched.tests_finished(): if self.sched.tests_finished():
self.triggershutdown() self.triggershutdown()
@@ -319,7 +321,7 @@ class DSession:
def slave_testreport(self, node, rep): def slave_testreport(self, node, rep):
if not (rep.passed and rep.when != "call"): if not (rep.passed and rep.when != "call"):
if rep.when in ("setup", "call"): if rep.when in ("setup", "call"):
self.sched.remove_item(node, rep.nodeid) self.sched.remove_item(node, rep.item_index, rep.duration)
#self.report_line("testreport %s: %s" %(rep.id, rep.status)) #self.report_line("testreport %s: %s" %(rep.id, rep.status))
rep.node = node rep.node = node
self.config.hook.pytest_runtest_logreport(report=rep) self.config.hook.pytest_runtest_logreport(report=rep)

View File

@@ -188,7 +188,7 @@ class StatRecorder:
self.check() # snapshot state self.check() # snapshot state
def fil(self, p): def fil(self, p):
return True # we are sensitive to all file changes since 1.9 return p.check(file=1, dotfile=0) and p.ext != ".pyc"
def rec(self, p): def rec(self, p):
return p.check(dotfile=0) return p.check(dotfile=0)

View File

@@ -1,5 +1,5 @@
import sys import py
import py, pytest import pytest
def pytest_addoption(parser): def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing") group = parser.getgroup("xdist", "distributed and subprocess testing")
@@ -28,12 +28,14 @@ def pytest_addoption(parser):
group._addoption('-d', group._addoption('-d',
action="store_true", dest="distload", default=False, action="store_true", dest="distload", default=False,
help="load-balance tests. shortcut for '--dist=load'") help="load-balance tests. shortcut for '--dist=load'")
group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
help="add directory for rsyncing to remote tx nodes.") help="add directory for rsyncing to remote tx nodes.")
group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
help="add expression for ignores when rsyncing to remote tx nodes.")
parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
' remote distributed testing.', type="pathlist") ' remote distributed testing.', type="pathlist")
parser.addini('rsyncignore', 'list of (relative) paths to be ignored ' parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
'for rsyncing.', type="pathlist") 'for rsyncing.', type="pathlist")
parser.addini("looponfailroots", type="pathlist", parser.addini("looponfailroots", type="pathlist",
help="directories to check for changes", default=[py.path.local()]) help="directories to check for changes", default=[py.path.local()])
@@ -62,6 +64,8 @@ def pytest_configure(config, __multicall__):
from xdist.dsession import DSession from xdist.dsession import DSession
session = DSession(config) session = DSession(config)
config.pluginmanager.register(session, "dsession") config.pluginmanager.register(session, "dsession")
tr = config.pluginmanager.getplugin("terminalreporter")
tr.showfspath = False
def check_options(config): def check_options(config):
if config.option.numprocesses: if config.option.numprocesses:

View File

@@ -24,7 +24,7 @@ class SlaveInteractor:
def pytest_internalerror(self, excrepr): def pytest_internalerror(self, excrepr):
for line in str(excrepr).split("\n"): for line in str(excrepr).split("\n"):
self.log("IERROR> " + line) self.log("IERROR>", line)
def pytest_sessionstart(self, session): def pytest_sessionstart(self, session):
self.session = session self.session = session
@@ -45,41 +45,41 @@ class SlaveInteractor:
torun = [] torun = []
while 1: while 1:
name, kwargs = self.channel.receive() name, kwargs = self.channel.receive()
self.log("received command %s(**%s)" % (name, kwargs)) self.log("received command", name, kwargs)
if name == "runtests": if name == "runtests":
ids = kwargs['ids'] torun.extend(kwargs['indices'])
for nodeid in ids:
torun.append(self._id2item[nodeid])
elif name == "runtests_all": elif name == "runtests_all":
torun.extend(session.items) torun.extend(range(len(session.items)))
self.log("items to run: %s" %(len(torun))) self.log("items to run:", torun)
while len(torun) >= 2:
item = torun.pop(0)
nextitem = torun[0]
self.config.hook.pytest_runtest_protocol(item=item,
nextitem=nextitem)
if name == "shutdown":
while torun: while torun:
self.config.hook.pytest_runtest_protocol( self.run_tests(torun)
item=torun.pop(0), nextitem=None) if name == "shutdown":
break break
return True return True
def run_tests(self, torun):
items = self.session.items
self.item_index = torun.pop(0)
if torun:
nextitem = items[torun[0]]
else:
nextitem = None
self.config.hook.pytest_runtest_protocol(
item=items[self.item_index],
nextitem=nextitem)
def pytest_collection_finish(self, session): def pytest_collection_finish(self, session):
self._id2item = {}
ids = []
for item in session.items:
self._id2item[item.nodeid] = item
ids.append(item.nodeid)
self.sendevent("collectionfinish", self.sendevent("collectionfinish",
topdir=str(session.fspath), topdir=str(session.fspath),
ids=ids) ids=[item.nodeid for item in session.items])
#def pytest_runtest_logstart(self, nodeid, location, fspath): def pytest_runtest_logstart(self, nodeid, location):
# self.sendevent("logstart", nodeid=nodeid, location=location) self.sendevent("logstart", nodeid=nodeid, location=location)
def pytest_runtest_logreport(self, report): def pytest_runtest_logreport(self, report):
data = serialize_report(report) data = serialize_report(report)
data["item_index"] = self.item_index
assert self.session.items[self.item_index].nodeid == report.nodeid
self.sendevent("testreport", data=data) self.sendevent("testreport", data=data)
def pytest_collectreport(self, report): def pytest_collectreport(self, report):

View File

@@ -1,5 +1,8 @@
import py, pytest import fnmatch
import sys, os import os
import py
import pytest
import execnet import execnet
import xdist.remote import xdist.remote
@@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically
class NodeManager(object): class NodeManager(object):
EXIT_TIMEOUT = 10 EXIT_TIMEOUT = 10
DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config self.config = config
self._nodesready = py.std.threading.Event() self._nodesready = py.std.threading.Event()
@@ -23,20 +27,17 @@ class NodeManager(object):
self.group.allocate_id(spec) self.group.allocate_id(spec)
self.specs.append(spec) self.specs.append(spec)
self.roots = self._getrsyncdirs() self.roots = self._getrsyncdirs()
self.rsyncoptions = self._getrsyncoptions()
def rsync_roots(self): def rsync_roots(self):
""" make sure that all remote gateways """ make sure that all remote gateways
have the same set of roots in their have the same set of roots in their
current directory. current directory.
""" """
options = {
'ignores': self.config.getini("rsyncignore"),
'verbose': self.config.option.verbose,
}
if self.roots: if self.roots:
# send each rsync root # send each rsync root
for root in self.roots: for root in self.roots:
self.rsync(root, **options) self.rsync(root, **self.rsyncoptions)
def makegateways(self): def makegateways(self):
assert not list(self.group) assert not list(self.group)
@@ -98,6 +99,18 @@ class NodeManager(object):
roots.append(root) roots.append(root)
return roots return roots
def _getrsyncoptions(self):
"""Get options to be passed for rsync."""
ignores = list(self.DEFAULT_IGNORES)
ignores += self.config.option.rsyncignore
ignores += self.config.getini("rsyncignore")
return {
'ignores': ignores,
'verbose': self.config.option.verbose,
}
def rsync(self, source, notify=None, verbose=False, ignores=None): def rsync(self, source, notify=None, verbose=False, ignores=None):
""" perform rsync to all remote hosts. """ perform rsync to all remote hosts.
""" """
@@ -144,12 +157,10 @@ class HostRSync(execnet.RSync):
def filter(self, path): def filter(self, path):
path = py.path.local(path) path = py.path.local(path)
if not path.ext in ('.pyc', '.pyo'):
if not path.basename.endswith('~'):
if path.check(dotfile=0):
for x in self._ignores: for x in self._ignores:
if path == x: x = getattr(x, 'strpath', x)
break if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
return False
else: else:
return True return True
@@ -230,8 +241,8 @@ class SlaveController(object):
self.gateway.exit() self.gateway.exit()
#del self.gateway #del self.gateway
def send_runtest(self, nodeid): def send_runtest_some(self, indices):
self.sendcommand("runtests", ids=[nodeid]) self.sendcommand("runtests", indices=indices)
def send_runtest_all(self): def send_runtest_all(self):
self.sendcommand("runtests_all",) self.sendcommand("runtests_all",)
@@ -278,10 +289,13 @@ class SlaveController(object):
self._down = True self._down = True
self.slaveoutput = kwargs['slaveoutput'] self.slaveoutput = kwargs['slaveoutput']
self.notify_inproc("slavefinished", node=self) self.notify_inproc("slavefinished", node=self)
#elif eventname == "logstart": elif eventname == "logstart":
# self.notify_inproc(eventname, node=self, **kwargs) self.notify_inproc(eventname, node=self, **kwargs)
elif eventname in ("testreport", "collectreport", "teardownreport"): elif eventname in ("testreport", "collectreport", "teardownreport"):
item_index = kwargs.pop("item_index", None)
rep = unserialize_report(eventname, kwargs['data']) rep = unserialize_report(eventname, kwargs['data'])
if item_index is not None:
rep.item_index = item_index
self.notify_inproc(eventname, node=self, rep=rep) self.notify_inproc(eventname, node=self, rep=rep)
elif eventname == "collectionfinish": elif eventname == "collectionfinish":
self.notify_inproc(eventname, node=self, ids=kwargs['ids']) self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
@@ -296,8 +310,7 @@ class SlaveController(object):
self.config.pluginmanager.notify_exception(excinfo) self.config.pluginmanager.notify_exception(excinfo)
def unserialize_report(name, reportdict): def unserialize_report(name, reportdict):
d = reportdict
if name == "testreport": if name == "testreport":
return runner.TestReport(**d) return runner.TestReport(**reportdict)
elif name == "collectreport": elif name == "collectreport":
return runner.CollectReport(**d) return runner.CollectReport(**reportdict)