Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b0732f99a6 | ||
|
|
ab987d512c | ||
|
|
ffa156ced5 | ||
|
|
e26d4486fc | ||
|
|
18a30fab7d | ||
|
|
dd73d132b1 | ||
|
|
a22e0384a1 | ||
|
|
8eabc7d5df | ||
|
|
dbf7932514 | ||
|
|
6355c3e8d0 | ||
|
|
c505f00320 | ||
|
|
7e066be35e | ||
|
|
27f1d8049b | ||
|
|
bcf1c85f44 | ||
|
|
a3fac52e85 | ||
|
|
152f965a70 | ||
|
|
027b3f51ed | ||
|
|
f4b86f0127 | ||
|
|
8146b27671 | ||
|
|
a488cc5add | ||
|
|
aad1aace52 | ||
|
|
82d357ab1a | ||
|
|
ca54911481 | ||
|
|
8e88261c15 | ||
|
|
6045195f16 |
@@ -14,8 +14,14 @@ syntax:glob
|
|||||||
*.class
|
*.class
|
||||||
*.orig
|
*.orig
|
||||||
|
|
||||||
|
*.sublime-*
|
||||||
|
.Python
|
||||||
|
|
||||||
build/
|
build/
|
||||||
dist/
|
dist/
|
||||||
|
include/
|
||||||
|
lib/
|
||||||
|
bin/
|
||||||
pytest_xdist.egg-info
|
pytest_xdist.egg-info
|
||||||
issue/
|
issue/
|
||||||
3rdparty/
|
3rdparty/
|
||||||
|
|||||||
1
.hgtags
1
.hgtags
@@ -13,3 +13,4 @@ cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
|
|||||||
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
|
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
|
||||||
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
|
0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
|
||||||
1d27987c267577899350a25ba5828d55d87083ad 1.8
|
1d27987c267577899350a25ba5828d55d87083ad 1.8
|
||||||
|
5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9
|
||||||
|
|||||||
20
CHANGELOG
20
CHANGELOG
@@ -1,3 +1,23 @@
|
|||||||
|
1.10
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
- add glob support for rsyncignores, add command line option to pass
|
||||||
|
additional rsyncignores. Thanks Anatoly Bubenkov.
|
||||||
|
|
||||||
|
- fix pytest issue382 - produce "pytest_runtest_logstart" event again
|
||||||
|
in master. Thanks Aron Curzon.
|
||||||
|
|
||||||
|
- fix pytest issue419 by sending/receiving indices into the test
|
||||||
|
collection instead of node ids (which are not neccessarily unique
|
||||||
|
for functions parametrized with duplicate values)
|
||||||
|
|
||||||
|
- send multiple "to test" indices in one network message to a slave
|
||||||
|
and improve heuristics for sending chunks where the chunksize
|
||||||
|
depends on the number of remaining tests rather than fixed numbers.
|
||||||
|
This reduces the number of master -> node messages (but not the
|
||||||
|
reverse direction)
|
||||||
|
|
||||||
|
|
||||||
1.9
|
1.9
|
||||||
-------------------------
|
-------------------------
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ Running tests in a boxed subprocess
|
|||||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
|
||||||
If you have tests involving C or C++ libraries you might have to deal
|
If you have tests involving C or C++ libraries you might have to deal
|
||||||
with tests crashing the process. For this case you max use the boxing
|
with tests crashing the process. For this case you may use the boxing
|
||||||
options::
|
options::
|
||||||
|
|
||||||
py.test --boxed
|
py.test --boxed
|
||||||
@@ -122,6 +122,13 @@ py.test references tests as a fully qualified python
|
|||||||
module path. **You will otherwise get strange errors**
|
module path. **You will otherwise get strange errors**
|
||||||
during setup of the remote side.
|
during setup of the remote side.
|
||||||
|
|
||||||
|
You can specify multiple ``--rsyncignore`` glob-patterns
|
||||||
|
to be ignored when file are sent to the remote side.
|
||||||
|
There are also internal ignores: .*, *.pyc, *.pyo, *~
|
||||||
|
Those you cannot override using rsyncignore command-line or
|
||||||
|
ini-file option(s).
|
||||||
|
|
||||||
|
|
||||||
Sending tests to remote Socket Servers
|
Sending tests to remote Socket Servers
|
||||||
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
|
||||||
|
|||||||
4
setup.py
4
setup.py
@@ -2,7 +2,7 @@ from setuptools import setup
|
|||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="pytest-xdist",
|
name="pytest-xdist",
|
||||||
version='1.9',
|
version='1.10',
|
||||||
description='py.test xdist plugin for distributed testing and loop-on-failing modes',
|
description='py.test xdist plugin for distributed testing and loop-on-failing modes',
|
||||||
long_description=open('README.txt').read(),
|
long_description=open('README.txt').read(),
|
||||||
license='MIT',
|
license='MIT',
|
||||||
@@ -13,7 +13,7 @@ setup(
|
|||||||
packages = ['xdist'],
|
packages = ['xdist'],
|
||||||
entry_points = {'pytest11': ['xdist = xdist.plugin'],},
|
entry_points = {'pytest11': ['xdist = xdist.plugin'],},
|
||||||
zip_safe=False,
|
zip_safe=False,
|
||||||
install_requires = ['execnet>=1.1', 'pytest>=2.3.5'],
|
install_requires = ['execnet>=1.1', 'pytest>=2.4.2'],
|
||||||
classifiers=[
|
classifiers=[
|
||||||
'Development Status :: 5 - Production/Stable',
|
'Development Status :: 5 - Production/Stable',
|
||||||
'Intended Audience :: Developers',
|
'Intended Audience :: Developers',
|
||||||
|
|||||||
@@ -256,6 +256,7 @@ class TestDistribution:
|
|||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
""")
|
""")
|
||||||
child = testdir.spawn_pytest("-n1")
|
child = testdir.spawn_pytest("-n1")
|
||||||
|
py.std.time.sleep(0.1)
|
||||||
child.expect(".*test session starts.*")
|
child.expect(".*test session starts.*")
|
||||||
child.kill(2) # keyboard interrupt
|
child.kill(2) # keyboard interrupt
|
||||||
child.expect(".*KeyboardInterrupt.*")
|
child.expect(".*KeyboardInterrupt.*")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from xdist.dsession import (
|
|||||||
)
|
)
|
||||||
from _pytest import main as outcome
|
from _pytest import main as outcome
|
||||||
import py
|
import py
|
||||||
|
import pytest
|
||||||
import execnet
|
import execnet
|
||||||
|
|
||||||
XSpec = execnet.XSpec
|
XSpec = execnet.XSpec
|
||||||
@@ -28,15 +29,12 @@ class MockNode:
|
|||||||
self.sent = []
|
self.sent = []
|
||||||
self.gateway = MockGateway()
|
self.gateway = MockGateway()
|
||||||
|
|
||||||
def send_runtest(self, nodeid):
|
def send_runtest_some(self, indices):
|
||||||
self.sent.append(nodeid)
|
self.sent.extend(indices)
|
||||||
|
|
||||||
def send_runtest_all(self):
|
def send_runtest_all(self):
|
||||||
self.sent.append("ALL")
|
self.sent.append("ALL")
|
||||||
|
|
||||||
def sendlist(self, items):
|
|
||||||
self.sent.extend(items)
|
|
||||||
|
|
||||||
def shutdown(self):
|
def shutdown(self):
|
||||||
self._shutdown=True
|
self._shutdown=True
|
||||||
|
|
||||||
@@ -63,9 +61,9 @@ class TestEachScheduling:
|
|||||||
assert sched.tests_finished()
|
assert sched.tests_finished()
|
||||||
assert node1.sent == ['ALL']
|
assert node1.sent == ['ALL']
|
||||||
assert node2.sent == ['ALL']
|
assert node2.sent == ['ALL']
|
||||||
sched.remove_item(node1, collection[0])
|
sched.remove_item(node1, 0)
|
||||||
assert sched.tests_finished()
|
assert sched.tests_finished()
|
||||||
sched.remove_item(node2, collection[0])
|
sched.remove_item(node2, 0)
|
||||||
assert sched.tests_finished()
|
assert sched.tests_finished()
|
||||||
|
|
||||||
def test_schedule_remove_node(self):
|
def test_schedule_remove_node(self):
|
||||||
@@ -104,7 +102,7 @@ class TestLoadScheduling:
|
|||||||
assert len(node1.sent) == 1
|
assert len(node1.sent) == 1
|
||||||
assert len(node2.sent) == 1
|
assert len(node2.sent) == 1
|
||||||
x = sorted(node1.sent + node2.sent)
|
x = sorted(node1.sent + node2.sent)
|
||||||
assert x == collection
|
assert x == [0, 1]
|
||||||
sched.remove_item(node1, node1.sent[0])
|
sched.remove_item(node1, node1.sent[0])
|
||||||
sched.remove_item(node2, node2.sent[0])
|
sched.remove_item(node2, node2.sent[0])
|
||||||
assert sched.tests_finished()
|
assert sched.tests_finished()
|
||||||
@@ -116,23 +114,22 @@ class TestLoadScheduling:
|
|||||||
node2 = MockNode()
|
node2 = MockNode()
|
||||||
sched.addnode(node1)
|
sched.addnode(node1)
|
||||||
sched.addnode(node2)
|
sched.addnode(node2)
|
||||||
sched.ITEM_CHUNKSIZE = 2
|
col = ["xyz"] * (3)
|
||||||
col = ["xyz"] * (2*sched.ITEM_CHUNKSIZE +1)
|
|
||||||
sched.addnode_collection(node1, col)
|
sched.addnode_collection(node1, col)
|
||||||
sched.addnode_collection(node2, col)
|
sched.addnode_collection(node2, col)
|
||||||
sched.init_distribute()
|
sched.init_distribute()
|
||||||
#assert not sched.tests_finished()
|
#assert not sched.tests_finished()
|
||||||
sent1 = node1.sent
|
sent1 = node1.sent
|
||||||
sent2 = node2.sent
|
sent2 = node2.sent
|
||||||
chunkitems = col[:sched.ITEM_CHUNKSIZE]
|
chunkitems = col[:1]
|
||||||
assert sent1 == chunkitems
|
assert (sent1 == [0] and sent2 == [1]) or (
|
||||||
assert sent2 == chunkitems
|
sent1 == [1] and sent2 == [0])
|
||||||
assert sched.node2pending[node1] == sent1
|
assert sched.node2pending[node1] == sent1
|
||||||
assert sched.node2pending[node2] == sent2
|
assert sched.node2pending[node2] == sent2
|
||||||
assert len(sched.pending) == 1
|
assert len(sched.pending) == 1
|
||||||
for node in (node1, node2):
|
for node in (node1, node2):
|
||||||
for i in range(sched.ITEM_CHUNKSIZE):
|
for i in sched.node2pending[node]:
|
||||||
sched.remove_item(node, "xyz")
|
sched.remove_item(node, i)
|
||||||
assert not sched.pending
|
assert not sched.pending
|
||||||
|
|
||||||
def test_add_remove_node(self):
|
def test_add_remove_node(self):
|
||||||
@@ -209,3 +206,16 @@ def test_report_collection_diff_different():
|
|||||||
report_collection_diff(from_collection, to_collection, 1, 2)
|
report_collection_diff(from_collection, to_collection, 1, 2)
|
||||||
except AssertionError as e:
|
except AssertionError as e:
|
||||||
assert py.builtin._totext(e) == error_message
|
assert py.builtin._totext(e) == error_message
|
||||||
|
|
||||||
|
@pytest.mark.xfail(reason="duplicate test ids not supported yet")
|
||||||
|
def test_pytest_issue419(testdir):
|
||||||
|
testdir.makepyfile("""
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
@pytest.mark.parametrize('birth_year', [1988, 1988, ])
|
||||||
|
def test_2011_table(birth_year):
|
||||||
|
pass
|
||||||
|
""")
|
||||||
|
reprec = testdir.inline_run("-n1")
|
||||||
|
reprec.assertoutcome(passed=2)
|
||||||
|
assert 0
|
||||||
|
|||||||
@@ -14,6 +14,10 @@ class TestStatRecorder:
|
|||||||
changed = sd.check()
|
changed = sd.check()
|
||||||
assert changed
|
assert changed
|
||||||
|
|
||||||
|
(hello + "c").write("hello")
|
||||||
|
changed = sd.check()
|
||||||
|
assert not changed
|
||||||
|
|
||||||
p = tmp.ensure("new.py")
|
p = tmp.ensure("new.py")
|
||||||
changed = sd.check()
|
changed = sd.check()
|
||||||
assert changed
|
assert changed
|
||||||
@@ -36,6 +40,12 @@ class TestStatRecorder:
|
|||||||
changed = sd.check()
|
changed = sd.check()
|
||||||
assert changed
|
assert changed
|
||||||
|
|
||||||
|
def test_dirchange(self, tmpdir):
|
||||||
|
tmp = tmpdir
|
||||||
|
hello = tmp.ensure("dir", "hello.py")
|
||||||
|
sd = StatRecorder([tmp])
|
||||||
|
assert not sd.fil(tmp.join("dir"))
|
||||||
|
|
||||||
def test_filechange_deletion_race(self, tmpdir, monkeypatch):
|
def test_filechange_deletion_race(self, tmpdir, monkeypatch):
|
||||||
tmp = tmpdir
|
tmp = tmpdir
|
||||||
sd = StatRecorder([tmp])
|
sd = StatRecorder([tmp])
|
||||||
@@ -63,12 +73,10 @@ class TestStatRecorder:
|
|||||||
|
|
||||||
pycfile = hello + "c"
|
pycfile = hello + "c"
|
||||||
pycfile.ensure()
|
pycfile.ensure()
|
||||||
changed = sd.check()
|
|
||||||
assert changed
|
|
||||||
|
|
||||||
hello.write("world")
|
hello.write("world")
|
||||||
changed = sd.check()
|
changed = sd.check()
|
||||||
assert changed
|
assert changed
|
||||||
|
assert not pycfile.check()
|
||||||
|
|
||||||
def test_waitonchange(self, tmpdir, monkeypatch):
|
def test_waitonchange(self, tmpdir, monkeypatch):
|
||||||
tmp = tmpdir
|
tmp = tmpdir
|
||||||
|
|||||||
@@ -46,6 +46,11 @@ class TestDistOptions:
|
|||||||
assert nm.roots
|
assert nm.roots
|
||||||
assert testdir.tmpdir in nm.roots
|
assert testdir.tmpdir in nm.roots
|
||||||
|
|
||||||
|
def test_getrsyncignore(self, testdir):
|
||||||
|
config = testdir.parseconfigure('--rsyncignore=fo*')
|
||||||
|
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
|
||||||
|
assert 'fo*' in nm.rsyncoptions['ignores']
|
||||||
|
|
||||||
def test_getrsyncdirs_with_conftest(self, testdir):
|
def test_getrsyncdirs_with_conftest(self, testdir):
|
||||||
p = py.path.local()
|
p = py.path.local()
|
||||||
for bn in 'x y z'.split():
|
for bn in 'x y z'.split():
|
||||||
|
|||||||
@@ -154,8 +154,11 @@ class TestSlaveInteractor:
|
|||||||
assert ev.kwargs['topdir'] == slave.testdir.tmpdir
|
assert ev.kwargs['topdir'] == slave.testdir.tmpdir
|
||||||
ids = ev.kwargs['ids']
|
ids = ev.kwargs['ids']
|
||||||
assert len(ids) == 1
|
assert len(ids) == 1
|
||||||
slave.sendcommand("runtests", ids=ids)
|
slave.sendcommand("runtests", indices=list(range(len(ids))))
|
||||||
slave.sendcommand("shutdown")
|
slave.sendcommand("shutdown")
|
||||||
|
ev = slave.popevent("logstart")
|
||||||
|
assert ev.kwargs["nodeid"].endswith("test_func")
|
||||||
|
assert len(ev.kwargs["location"]) == 3
|
||||||
ev = slave.popevent("testreport") # setup
|
ev = slave.popevent("testreport") # setup
|
||||||
ev = slave.popevent("testreport")
|
ev = slave.popevent("testreport")
|
||||||
assert ev.name == "testreport"
|
assert ev.name == "testreport"
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ class TestHRSync:
|
|||||||
source.ensure(".svn", "entries")
|
source.ensure(".svn", "entries")
|
||||||
source.ensure(".somedotfile", "moreentries")
|
source.ensure(".somedotfile", "moreentries")
|
||||||
source.ensure("somedir", "editfile~")
|
source.ensure("somedir", "editfile~")
|
||||||
syncer = HostRSync(source)
|
syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
|
||||||
l = list(source.visit(rec=syncer.filter,
|
l = list(source.visit(rec=syncer.filter,
|
||||||
fil=syncer.filter))
|
fil=syncer.filter))
|
||||||
assert len(l) == 3
|
assert len(l) == 3
|
||||||
@@ -197,12 +197,15 @@ class TestNodeManager:
|
|||||||
dir5 = source.ensure("dir5", "dir6", "bogus")
|
dir5 = source.ensure("dir5", "dir6", "bogus")
|
||||||
dirf = source.ensure("dir5", "file")
|
dirf = source.ensure("dir5", "file")
|
||||||
dir2.ensure("hello")
|
dir2.ensure("hello")
|
||||||
|
dirfoo = source.ensure("foo", "bar")
|
||||||
|
dirbar = source.ensure("bar", "foo")
|
||||||
source.join("tox.ini").write(py.std.textwrap.dedent("""
|
source.join("tox.ini").write(py.std.textwrap.dedent("""
|
||||||
[pytest]
|
[pytest]
|
||||||
rsyncdirs = dir1 dir5
|
rsyncdirs = dir1 dir5
|
||||||
rsyncignore = dir1/dir2 dir5/dir6
|
rsyncignore = dir1/dir2 dir5/dir6 foo*
|
||||||
"""))
|
"""))
|
||||||
config = testdir.parseconfig(source)
|
config = testdir.parseconfig(source)
|
||||||
|
config.option.rsyncignore = ['bar']
|
||||||
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
|
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
|
||||||
nodemanager.makegateways()
|
nodemanager.makegateways()
|
||||||
nodemanager.rsync_roots()
|
nodemanager.rsync_roots()
|
||||||
@@ -210,6 +213,8 @@ class TestNodeManager:
|
|||||||
assert not dest.join("dir1", "dir2").check()
|
assert not dest.join("dir1", "dir2").check()
|
||||||
assert dest.join("dir5","file").check()
|
assert dest.join("dir5","file").check()
|
||||||
assert not dest.join("dir6").check()
|
assert not dest.join("dir6").check()
|
||||||
|
assert not dest.join('foo').check()
|
||||||
|
assert not dest.join('bar').check()
|
||||||
|
|
||||||
def test_optimise_popen(self, testdir, mysetup):
|
def test_optimise_popen(self, testdir, mysetup):
|
||||||
source, dest = mysetup.source, mysetup.dest
|
source, dest = mysetup.source, mysetup.dest
|
||||||
|
|||||||
17
tox.ini
17
tox.ini
@@ -1,25 +1,26 @@
|
|||||||
[tox]
|
[tox]
|
||||||
envlist=py26,py32,py33,py27,py26,py26-old,py33-old
|
envlist=py26,py32,py33,py27,py27-pexpect,py33-pexpect,py26,py26-old,py33-old
|
||||||
|
|
||||||
[testenv]
|
[testenv]
|
||||||
changedir=testing
|
changedir=testing
|
||||||
deps=pytest>=2.4.2
|
deps=pytest>=2.5.1
|
||||||
commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
|
commands= py.test --junitxml={envlogdir}/junit-{envname}.xml []
|
||||||
|
|
||||||
[testenv:py27]
|
[testenv:py27-pexpect]
|
||||||
deps=
|
deps={[testenv]deps}
|
||||||
pytest>=2.4.2
|
pexpect
|
||||||
|
[testenv:py33-pexpect]
|
||||||
|
deps={[testenv]deps}
|
||||||
pexpect
|
pexpect
|
||||||
|
|
||||||
[testenv:py26-old]
|
[testenv:py26-old]
|
||||||
deps=
|
deps=
|
||||||
pytest==2.3.5
|
pytest==2.4.2
|
||||||
pexpect
|
|
||||||
|
|
||||||
[testenv:py33-old]
|
[testenv:py33-old]
|
||||||
basepython = python3.3
|
basepython = python3.3
|
||||||
deps=
|
deps=
|
||||||
pytest==2.3.5
|
pytest==2.4.2
|
||||||
|
|
||||||
[pytest]
|
[pytest]
|
||||||
addopts = -rsfxX
|
addopts = -rsfxX
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
#
|
#
|
||||||
__version__ = '1.9'
|
__version__ = '1.10'
|
||||||
|
|||||||
@@ -40,15 +40,15 @@ class EachScheduling:
|
|||||||
if len(self.node2pending) >= self.numnodes:
|
if len(self.node2pending) >= self.numnodes:
|
||||||
self.collection_is_completed = True
|
self.collection_is_completed = True
|
||||||
|
|
||||||
def remove_item(self, node, item):
|
def remove_item(self, node, item_index, duration=0):
|
||||||
self.node2pending[node].remove(item)
|
self.node2pending[node].remove(item_index)
|
||||||
|
|
||||||
def remove_node(self, node):
|
def remove_node(self, node):
|
||||||
# KeyError if we didn't get an addnode() yet
|
# KeyError if we didn't get an addnode() yet
|
||||||
pending = self.node2pending.pop(node)
|
pending = self.node2pending.pop(node)
|
||||||
if not pending:
|
if not pending:
|
||||||
return
|
return
|
||||||
crashitem = pending.pop(0)
|
crashitem = self.node2collection[node][pending.pop(0)]
|
||||||
# XXX what about the rest of pending?
|
# XXX what about the rest of pending?
|
||||||
return crashitem
|
return crashitem
|
||||||
|
|
||||||
@@ -56,12 +56,9 @@ class EachScheduling:
|
|||||||
assert self.collection_is_completed
|
assert self.collection_is_completed
|
||||||
for node, pending in self.node2pending.items():
|
for node, pending in self.node2pending.items():
|
||||||
node.send_runtest_all()
|
node.send_runtest_all()
|
||||||
pending[:] = self.node2collection[node]
|
pending[:] = range(len(self.node2collection[node]))
|
||||||
|
|
||||||
class LoadScheduling:
|
class LoadScheduling:
|
||||||
LOAD_THRESHOLD_NEWITEMS = 5
|
|
||||||
ITEM_CHUNKSIZE = 10
|
|
||||||
|
|
||||||
def __init__(self, numnodes, log=None):
|
def __init__(self, numnodes, log=None):
|
||||||
self.numnodes = numnodes
|
self.numnodes = numnodes
|
||||||
self.node2pending = {}
|
self.node2pending = {}
|
||||||
@@ -94,47 +91,46 @@ class LoadScheduling:
|
|||||||
if len(self.node2collection) >= self.numnodes:
|
if len(self.node2collection) >= self.numnodes:
|
||||||
self.collection_is_completed = True
|
self.collection_is_completed = True
|
||||||
|
|
||||||
def remove_item(self, node, item):
|
def remove_item(self, node, item_index, duration=0):
|
||||||
if item not in self.item2nodes:
|
node_pending = self.node2pending[node]
|
||||||
raise AssertionError(item, self.item2nodes)
|
node_pending.remove(item_index)
|
||||||
nodes = self.item2nodes[item]
|
|
||||||
if node in nodes: # the node might have gone down already
|
|
||||||
nodes.remove(node)
|
|
||||||
#if not nodes:
|
|
||||||
# del self.item2nodes[item]
|
|
||||||
pending = self.node2pending[node]
|
|
||||||
pending.remove(item)
|
|
||||||
# pre-load items-to-test if the node may become ready
|
# pre-load items-to-test if the node may become ready
|
||||||
if self.pending and len(pending) < self.LOAD_THRESHOLD_NEWITEMS:
|
|
||||||
item = self.pending.pop(0)
|
if self.pending:
|
||||||
pending.append(item)
|
if duration >= 0.1 and node_pending:
|
||||||
self.item2nodes.setdefault(item, []).append(node)
|
# seems the node is doing long-running tests
|
||||||
node.send_runtest(item)
|
# so let's rather wait with sending new items
|
||||||
self.log("items waiting for node: %d" %(len(self.pending)))
|
return
|
||||||
#self.log("item2pending still executing: %s" %(self.item2nodes,))
|
# how many nodes do we have remaining per node roughly?
|
||||||
#self.log("node2pending: %s" %(self.node2pending,))
|
num_nodes = len(self.node2pending)
|
||||||
|
# if our node goes below a heuristic minimum, fill it out to
|
||||||
|
# heuristic maximum
|
||||||
|
items_per_node_min = max(
|
||||||
|
1, len(self.pending) // num_nodes // 4)
|
||||||
|
items_per_node_max = max(
|
||||||
|
1, len(self.pending) // num_nodes // 2)
|
||||||
|
if len(node_pending) <= items_per_node_min:
|
||||||
|
num_send = items_per_node_max - len(node_pending) + 1
|
||||||
|
self._send_tests(node, num_send)
|
||||||
|
|
||||||
|
self.log("num items waiting for node:", len(self.pending))
|
||||||
|
#self.log("node2pending:", self.node2pending)
|
||||||
|
|
||||||
def remove_node(self, node):
|
def remove_node(self, node):
|
||||||
pending = self.node2pending.pop(node)
|
pending = self.node2pending.pop(node)
|
||||||
# KeyError if we didn't get an addnode() yet
|
|
||||||
for item in pending:
|
|
||||||
l = self.item2nodes[item]
|
|
||||||
l.remove(node)
|
|
||||||
if not l:
|
|
||||||
del self.item2nodes[item]
|
|
||||||
if not pending:
|
if not pending:
|
||||||
return
|
return
|
||||||
crashitem = pending.pop(0)
|
# the node must have crashed on the item if there are pending ones
|
||||||
|
crashitem = self.collection[pending.pop(0)]
|
||||||
self.pending.extend(pending)
|
self.pending.extend(pending)
|
||||||
return crashitem
|
return crashitem
|
||||||
|
|
||||||
def init_distribute(self):
|
def init_distribute(self):
|
||||||
assert self.collection_is_completed
|
assert self.collection_is_completed
|
||||||
assert not hasattr(self, 'item2nodes')
|
|
||||||
self.item2nodes = {}
|
|
||||||
# XXX allow nodes to have different collections
|
# XXX allow nodes to have different collections
|
||||||
first_node, col = list(self.node2collection.items())[0]
|
node_collection_items = list(self.node2collection.items())
|
||||||
for node, collection in self.node2collection.items():
|
first_node, col = node_collection_items[0]
|
||||||
|
for node, collection in node_collection_items[1:]:
|
||||||
report_collection_diff(
|
report_collection_diff(
|
||||||
col,
|
col,
|
||||||
collection,
|
collection,
|
||||||
@@ -142,22 +138,28 @@ class LoadScheduling:
|
|||||||
node.gateway.id,
|
node.gateway.id,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.pending = col
|
# all collections are the same, good.
|
||||||
|
# we now create an index
|
||||||
|
self.collection = col
|
||||||
|
self.pending[:] = range(len(col))
|
||||||
if not col:
|
if not col:
|
||||||
return
|
return
|
||||||
available = list(self.node2pending.items())
|
# how many items per node do we have about?
|
||||||
num_available = self.numnodes
|
items_per_node = len(self.collection) // len(self.node2pending)
|
||||||
max_one_round = num_available * self.ITEM_CHUNKSIZE - 1
|
# take a fraction of tests for initial distribution
|
||||||
for i, item in enumerate(self.pending):
|
node_chunksize = max(items_per_node // 4, 1)
|
||||||
nodeindex = i % num_available
|
# and initialize each node with a chunk of tests
|
||||||
node, pending = available[nodeindex]
|
for node in self.node2pending:
|
||||||
node.send_runtest(item)
|
self._send_tests(node, node_chunksize)
|
||||||
self.item2nodes.setdefault(item, []).append(node)
|
|
||||||
pending.append(item)
|
|
||||||
if i >= max_one_round:
|
|
||||||
break
|
|
||||||
del self.pending[:i + 1]
|
|
||||||
|
|
||||||
|
#f = open("/tmp/sent", "w")
|
||||||
|
def _send_tests(self, node, num):
|
||||||
|
tests_per_node = self.pending[:num]
|
||||||
|
#print >>self.f, "sent", node, tests_per_node
|
||||||
|
if tests_per_node:
|
||||||
|
del self.pending[:num]
|
||||||
|
self.node2pending[node].extend(tests_per_node)
|
||||||
|
node.send_runtest_some(tests_per_node)
|
||||||
|
|
||||||
def report_collection_diff(from_collection, to_collection, from_id, to_id):
|
def report_collection_diff(from_collection, to_collection, from_id, to_id):
|
||||||
"""Report the collected test difference between two nodes.
|
"""Report the collected test difference between two nodes.
|
||||||
@@ -256,7 +258,7 @@ class DSession:
|
|||||||
assert callname, kwargs
|
assert callname, kwargs
|
||||||
method = "slave_" + callname
|
method = "slave_" + callname
|
||||||
call = getattr(self, method)
|
call = getattr(self, method)
|
||||||
self.log("calling method: %s(**%s)" % (method, kwargs))
|
self.log("calling method", method, kwargs)
|
||||||
call(**kwargs)
|
call(**kwargs)
|
||||||
if self.sched.tests_finished():
|
if self.sched.tests_finished():
|
||||||
self.triggershutdown()
|
self.triggershutdown()
|
||||||
@@ -319,7 +321,7 @@ class DSession:
|
|||||||
def slave_testreport(self, node, rep):
|
def slave_testreport(self, node, rep):
|
||||||
if not (rep.passed and rep.when != "call"):
|
if not (rep.passed and rep.when != "call"):
|
||||||
if rep.when in ("setup", "call"):
|
if rep.when in ("setup", "call"):
|
||||||
self.sched.remove_item(node, rep.nodeid)
|
self.sched.remove_item(node, rep.item_index, rep.duration)
|
||||||
#self.report_line("testreport %s: %s" %(rep.id, rep.status))
|
#self.report_line("testreport %s: %s" %(rep.id, rep.status))
|
||||||
rep.node = node
|
rep.node = node
|
||||||
self.config.hook.pytest_runtest_logreport(report=rep)
|
self.config.hook.pytest_runtest_logreport(report=rep)
|
||||||
|
|||||||
@@ -188,7 +188,7 @@ class StatRecorder:
|
|||||||
self.check() # snapshot state
|
self.check() # snapshot state
|
||||||
|
|
||||||
def fil(self, p):
|
def fil(self, p):
|
||||||
return True # we are sensitive to all file changes since 1.9
|
return p.check(file=1, dotfile=0) and p.ext != ".pyc"
|
||||||
def rec(self, p):
|
def rec(self, p):
|
||||||
return p.check(dotfile=0)
|
return p.check(dotfile=0)
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import sys
|
import py
|
||||||
import py, pytest
|
import pytest
|
||||||
|
|
||||||
def pytest_addoption(parser):
|
def pytest_addoption(parser):
|
||||||
group = parser.getgroup("xdist", "distributed and subprocess testing")
|
group = parser.getgroup("xdist", "distributed and subprocess testing")
|
||||||
@@ -28,12 +28,14 @@ def pytest_addoption(parser):
|
|||||||
group._addoption('-d',
|
group._addoption('-d',
|
||||||
action="store_true", dest="distload", default=False,
|
action="store_true", dest="distload", default=False,
|
||||||
help="load-balance tests. shortcut for '--dist=load'")
|
help="load-balance tests. shortcut for '--dist=load'")
|
||||||
group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
|
group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
|
||||||
help="add directory for rsyncing to remote tx nodes.")
|
help="add directory for rsyncing to remote tx nodes.")
|
||||||
|
group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
|
||||||
|
help="add expression for ignores when rsyncing to remote tx nodes.")
|
||||||
|
|
||||||
parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
|
parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
|
||||||
' remote distributed testing.', type="pathlist")
|
' remote distributed testing.', type="pathlist")
|
||||||
parser.addini('rsyncignore', 'list of (relative) paths to be ignored '
|
parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
|
||||||
'for rsyncing.', type="pathlist")
|
'for rsyncing.', type="pathlist")
|
||||||
parser.addini("looponfailroots", type="pathlist",
|
parser.addini("looponfailroots", type="pathlist",
|
||||||
help="directories to check for changes", default=[py.path.local()])
|
help="directories to check for changes", default=[py.path.local()])
|
||||||
@@ -62,6 +64,8 @@ def pytest_configure(config, __multicall__):
|
|||||||
from xdist.dsession import DSession
|
from xdist.dsession import DSession
|
||||||
session = DSession(config)
|
session = DSession(config)
|
||||||
config.pluginmanager.register(session, "dsession")
|
config.pluginmanager.register(session, "dsession")
|
||||||
|
tr = config.pluginmanager.getplugin("terminalreporter")
|
||||||
|
tr.showfspath = False
|
||||||
|
|
||||||
def check_options(config):
|
def check_options(config):
|
||||||
if config.option.numprocesses:
|
if config.option.numprocesses:
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class SlaveInteractor:
|
|||||||
|
|
||||||
def pytest_internalerror(self, excrepr):
|
def pytest_internalerror(self, excrepr):
|
||||||
for line in str(excrepr).split("\n"):
|
for line in str(excrepr).split("\n"):
|
||||||
self.log("IERROR> " + line)
|
self.log("IERROR>", line)
|
||||||
|
|
||||||
def pytest_sessionstart(self, session):
|
def pytest_sessionstart(self, session):
|
||||||
self.session = session
|
self.session = session
|
||||||
@@ -45,41 +45,41 @@ class SlaveInteractor:
|
|||||||
torun = []
|
torun = []
|
||||||
while 1:
|
while 1:
|
||||||
name, kwargs = self.channel.receive()
|
name, kwargs = self.channel.receive()
|
||||||
self.log("received command %s(**%s)" % (name, kwargs))
|
self.log("received command", name, kwargs)
|
||||||
if name == "runtests":
|
if name == "runtests":
|
||||||
ids = kwargs['ids']
|
torun.extend(kwargs['indices'])
|
||||||
for nodeid in ids:
|
|
||||||
torun.append(self._id2item[nodeid])
|
|
||||||
elif name == "runtests_all":
|
elif name == "runtests_all":
|
||||||
torun.extend(session.items)
|
torun.extend(range(len(session.items)))
|
||||||
self.log("items to run: %s" %(len(torun)))
|
self.log("items to run:", torun)
|
||||||
while len(torun) >= 2:
|
|
||||||
item = torun.pop(0)
|
|
||||||
nextitem = torun[0]
|
|
||||||
self.config.hook.pytest_runtest_protocol(item=item,
|
|
||||||
nextitem=nextitem)
|
|
||||||
if name == "shutdown":
|
|
||||||
while torun:
|
while torun:
|
||||||
self.config.hook.pytest_runtest_protocol(
|
self.run_tests(torun)
|
||||||
item=torun.pop(0), nextitem=None)
|
if name == "shutdown":
|
||||||
break
|
break
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def run_tests(self, torun):
|
||||||
|
items = self.session.items
|
||||||
|
self.item_index = torun.pop(0)
|
||||||
|
if torun:
|
||||||
|
nextitem = items[torun[0]]
|
||||||
|
else:
|
||||||
|
nextitem = None
|
||||||
|
self.config.hook.pytest_runtest_protocol(
|
||||||
|
item=items[self.item_index],
|
||||||
|
nextitem=nextitem)
|
||||||
|
|
||||||
def pytest_collection_finish(self, session):
|
def pytest_collection_finish(self, session):
|
||||||
self._id2item = {}
|
|
||||||
ids = []
|
|
||||||
for item in session.items:
|
|
||||||
self._id2item[item.nodeid] = item
|
|
||||||
ids.append(item.nodeid)
|
|
||||||
self.sendevent("collectionfinish",
|
self.sendevent("collectionfinish",
|
||||||
topdir=str(session.fspath),
|
topdir=str(session.fspath),
|
||||||
ids=ids)
|
ids=[item.nodeid for item in session.items])
|
||||||
|
|
||||||
#def pytest_runtest_logstart(self, nodeid, location, fspath):
|
def pytest_runtest_logstart(self, nodeid, location):
|
||||||
# self.sendevent("logstart", nodeid=nodeid, location=location)
|
self.sendevent("logstart", nodeid=nodeid, location=location)
|
||||||
|
|
||||||
def pytest_runtest_logreport(self, report):
|
def pytest_runtest_logreport(self, report):
|
||||||
data = serialize_report(report)
|
data = serialize_report(report)
|
||||||
|
data["item_index"] = self.item_index
|
||||||
|
assert self.session.items[self.item_index].nodeid == report.nodeid
|
||||||
self.sendevent("testreport", data=data)
|
self.sendevent("testreport", data=data)
|
||||||
|
|
||||||
def pytest_collectreport(self, report):
|
def pytest_collectreport(self, report):
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
import py, pytest
|
import fnmatch
|
||||||
import sys, os
|
import os
|
||||||
|
|
||||||
|
import py
|
||||||
|
import pytest
|
||||||
import execnet
|
import execnet
|
||||||
import xdist.remote
|
import xdist.remote
|
||||||
|
|
||||||
@@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically
|
|||||||
|
|
||||||
class NodeManager(object):
|
class NodeManager(object):
|
||||||
EXIT_TIMEOUT = 10
|
EXIT_TIMEOUT = 10
|
||||||
|
DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
|
||||||
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
|
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
|
||||||
self.config = config
|
self.config = config
|
||||||
self._nodesready = py.std.threading.Event()
|
self._nodesready = py.std.threading.Event()
|
||||||
@@ -23,20 +27,17 @@ class NodeManager(object):
|
|||||||
self.group.allocate_id(spec)
|
self.group.allocate_id(spec)
|
||||||
self.specs.append(spec)
|
self.specs.append(spec)
|
||||||
self.roots = self._getrsyncdirs()
|
self.roots = self._getrsyncdirs()
|
||||||
|
self.rsyncoptions = self._getrsyncoptions()
|
||||||
|
|
||||||
def rsync_roots(self):
|
def rsync_roots(self):
|
||||||
""" make sure that all remote gateways
|
""" make sure that all remote gateways
|
||||||
have the same set of roots in their
|
have the same set of roots in their
|
||||||
current directory.
|
current directory.
|
||||||
"""
|
"""
|
||||||
options = {
|
|
||||||
'ignores': self.config.getini("rsyncignore"),
|
|
||||||
'verbose': self.config.option.verbose,
|
|
||||||
}
|
|
||||||
if self.roots:
|
if self.roots:
|
||||||
# send each rsync root
|
# send each rsync root
|
||||||
for root in self.roots:
|
for root in self.roots:
|
||||||
self.rsync(root, **options)
|
self.rsync(root, **self.rsyncoptions)
|
||||||
|
|
||||||
def makegateways(self):
|
def makegateways(self):
|
||||||
assert not list(self.group)
|
assert not list(self.group)
|
||||||
@@ -98,6 +99,18 @@ class NodeManager(object):
|
|||||||
roots.append(root)
|
roots.append(root)
|
||||||
return roots
|
return roots
|
||||||
|
|
||||||
|
def _getrsyncoptions(self):
|
||||||
|
"""Get options to be passed for rsync."""
|
||||||
|
ignores = list(self.DEFAULT_IGNORES)
|
||||||
|
ignores += self.config.option.rsyncignore
|
||||||
|
ignores += self.config.getini("rsyncignore")
|
||||||
|
|
||||||
|
return {
|
||||||
|
'ignores': ignores,
|
||||||
|
'verbose': self.config.option.verbose,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def rsync(self, source, notify=None, verbose=False, ignores=None):
|
def rsync(self, source, notify=None, verbose=False, ignores=None):
|
||||||
""" perform rsync to all remote hosts.
|
""" perform rsync to all remote hosts.
|
||||||
"""
|
"""
|
||||||
@@ -144,12 +157,10 @@ class HostRSync(execnet.RSync):
|
|||||||
|
|
||||||
def filter(self, path):
|
def filter(self, path):
|
||||||
path = py.path.local(path)
|
path = py.path.local(path)
|
||||||
if not path.ext in ('.pyc', '.pyo'):
|
|
||||||
if not path.basename.endswith('~'):
|
|
||||||
if path.check(dotfile=0):
|
|
||||||
for x in self._ignores:
|
for x in self._ignores:
|
||||||
if path == x:
|
x = getattr(x, 'strpath', x)
|
||||||
break
|
if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
|
||||||
|
return False
|
||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -230,8 +241,8 @@ class SlaveController(object):
|
|||||||
self.gateway.exit()
|
self.gateway.exit()
|
||||||
#del self.gateway
|
#del self.gateway
|
||||||
|
|
||||||
def send_runtest(self, nodeid):
|
def send_runtest_some(self, indices):
|
||||||
self.sendcommand("runtests", ids=[nodeid])
|
self.sendcommand("runtests", indices=indices)
|
||||||
|
|
||||||
def send_runtest_all(self):
|
def send_runtest_all(self):
|
||||||
self.sendcommand("runtests_all",)
|
self.sendcommand("runtests_all",)
|
||||||
@@ -278,10 +289,13 @@ class SlaveController(object):
|
|||||||
self._down = True
|
self._down = True
|
||||||
self.slaveoutput = kwargs['slaveoutput']
|
self.slaveoutput = kwargs['slaveoutput']
|
||||||
self.notify_inproc("slavefinished", node=self)
|
self.notify_inproc("slavefinished", node=self)
|
||||||
#elif eventname == "logstart":
|
elif eventname == "logstart":
|
||||||
# self.notify_inproc(eventname, node=self, **kwargs)
|
self.notify_inproc(eventname, node=self, **kwargs)
|
||||||
elif eventname in ("testreport", "collectreport", "teardownreport"):
|
elif eventname in ("testreport", "collectreport", "teardownreport"):
|
||||||
|
item_index = kwargs.pop("item_index", None)
|
||||||
rep = unserialize_report(eventname, kwargs['data'])
|
rep = unserialize_report(eventname, kwargs['data'])
|
||||||
|
if item_index is not None:
|
||||||
|
rep.item_index = item_index
|
||||||
self.notify_inproc(eventname, node=self, rep=rep)
|
self.notify_inproc(eventname, node=self, rep=rep)
|
||||||
elif eventname == "collectionfinish":
|
elif eventname == "collectionfinish":
|
||||||
self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
|
self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
|
||||||
@@ -296,8 +310,7 @@ class SlaveController(object):
|
|||||||
self.config.pluginmanager.notify_exception(excinfo)
|
self.config.pluginmanager.notify_exception(excinfo)
|
||||||
|
|
||||||
def unserialize_report(name, reportdict):
|
def unserialize_report(name, reportdict):
|
||||||
d = reportdict
|
|
||||||
if name == "testreport":
|
if name == "testreport":
|
||||||
return runner.TestReport(**d)
|
return runner.TestReport(**reportdict)
|
||||||
elif name == "collectreport":
|
elif name == "collectreport":
|
||||||
return runner.CollectReport(**d)
|
return runner.CollectReport(**reportdict)
|
||||||
|
|||||||
Reference in New Issue
Block a user