IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

flake8 cleanup

Browse Source
This commit is contained in:
Ronny Pfannschmidt
2015-09-01 22:37:51 +02:00
parent fa1ec6742a
commit 0f5ef95be8
18 changed files with 298 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@@ -25,7 +25,7 @@ env:
- TESTENV=py27-pytest27-pexpect
- TESTENV=py34-pytest27-pexpect
- TESTENV=py35-pytest27
# - TESTENV=py35-pytest27
- TESTENV=pypy-pytest27
script: tox --recreate -e $TESTENV

3
setup.py
View File

@@ -3,7 +3,8 @@ from setuptools import setup
setup(
name="pytest-xdist",
use_scm_version={'write_to': 'xdist/_version.py'},
description='py.test xdist plugin for distributed testing and loop-on-failing modes',
description='py.test xdist plugin for distributed testing'
' and loop-on-failing modes',
long_description=open('README.rst').read(),
license='MIT',
author='holger krekel and contributors',

90
testing/acceptance_test.py
View File

@@ -10,9 +10,7 @@ class TestDistribution:
""")
result = testdir.runpytest(p1, "-n1")
assert result.ret == 0
result.stdout.fnmatch_lines([
"*1 passed*",
])
result.stdout.fnmatch_lines(["*1 passed*", ])
def test_n1_fail(self, testdir):
p1 = testdir.makepyfile("""
@@ -21,9 +19,7 @@ class TestDistribution:
""")
result = testdir.runpytest(p1, "-n1")
assert result.ret == 1
result.stdout.fnmatch_lines([
"*1 failed*",
])
result.stdout.fnmatch_lines(["*1 failed*", ])
def test_n1_import_error(self, testdir):
p1 = testdir.makepyfile("""
@@ -57,9 +53,7 @@ class TestDistribution:
""")
result = testdir.runpytest(p1, "-n1")
assert result.ret == 0
result.stdout.fnmatch_lines([
"*1 skipped*",
])
result.stdout.fnmatch_lines(["*1 skipped*", ])
def test_manytests_to_one_import_error(self, testdir):
p1 = testdir.makepyfile("""
@@ -84,8 +78,7 @@ class TestDistribution:
pass
def test_skip():
py.test.skip("hello")
""",
)
""", )
result = testdir.runpytest(p1, "-v", '-d', '--tx=popen', '--tx=popen')
result.stdout.fnmatch_lines([
"*1*Python*",
@@ -115,9 +108,7 @@ class TestDistribution:
""" % str(testdir.tmpdir))
result = testdir.runpytest_subprocess(p1, "-n1")
assert result.ret == 0
result.stdout.fnmatch_lines([
"*1 passed*",
])
result.stdout.fnmatch_lines(["*1 passed*", ])
def test_dist_ini_specified(self, testdir):
p1 = testdir.makepyfile("""
@@ -130,8 +121,7 @@ class TestDistribution:
pass
def test_skip():
py.test.skip("hello")
""",
)
""", )
testdir.makeini("""
[pytest]
addopts = --tx=3*popen
@@ -163,13 +153,10 @@ class TestDistribution:
import os
time.sleep(0.5)
os.kill(os.getpid(), 15)
"""
)
""")
result = testdir.runpytest(p1, "-v", '-d', '-n1')
result.stdout.fnmatch_lines([
"*Python*",
"*PASS**test_ok*",
"*node*down*",
"*Python*", "*PASS**test_ok*", "*node*down*",
"*3 failed, 1 passed, 1 skipped*"
])
assert result.ret == 1
@@ -221,14 +208,13 @@ class TestDistribution:
p1 = testdir.makepyfile("def test_func(): pass")
result = testdir.runpytest("-v", p1, '-d', '--tx=popen')
result.stdout.fnmatch_lines([
"*0*Python*",
"*calculated result is 49*",
"*1 passed*"
"*0*Python*", "*calculated result is 49*", "*1 passed*"
])
assert result.ret == 0
def test_keyboardinterrupt_hooks_issue79(self, testdir):
testdir.makepyfile(__init__="", test_one="""
testdir.makepyfile(__init__="",
test_one="""
def test_hello():
raise KeyboardInterrupt()
""")
@@ -264,17 +250,20 @@ class TestDistribution:
child.close()
# assert ret == 2
class TestDistEach:
def test_simple(self, testdir):
testdir.makepyfile("""
def test_hello():
pass
""")
result = testdir.runpytest_subprocess("--debug", "--dist=each", "--tx=2*popen")
result = testdir.runpytest_subprocess("--debug", "--dist=each",
"--tx=2*popen")
assert not result.ret
result.stdout.fnmatch_lines(["*2 pass*"])
@py.test.mark.xfail(run=False,
@py.test.mark.xfail(
run=False,
reason="other python versions might not have py.test installed")
def test_simple_diffoutput(self, testdir):
interpreters = []
@@ -284,7 +273,8 @@ class TestDistEach:
py.test.skip("%s not found" % name)
interpreters.append(interp)
testdir.makepyfile(__init__="", test_one="""
testdir.makepyfile(__init__="",
test_one="""
import sys
def test_hello():
print("%s...%s" % sys.version_info[:2])
@@ -298,6 +288,7 @@ class TestDistEach:
assert "2...5" in s
assert "2...6" in s
class TestTerminalReporting:
def test_pass_skip_fail(self, testdir):
testdir.makepyfile("""
@@ -335,6 +326,7 @@ class TestTerminalReporting:
"E assert 0",
])
def test_teardownfails_one_function(testdir):
p = testdir.makepyfile("""
def test_func():
@@ -344,10 +336,10 @@ def test_teardownfails_one_function(testdir):
""")
result = testdir.runpytest(p, '-n1', '--tx=popen')
result.stdout.fnmatch_lines([
"*def teardown_function(function):*",
"*1 passed*1 error*"
"*def teardown_function(function):*", "*1 passed*1 error*"
])
@py.test.mark.xfail
def test_terminate_on_hangingnode(testdir):
p = testdir.makeconftest("""
@@ -358,9 +350,7 @@ def test_terminate_on_hangingnode(testdir):
""")
result = testdir.runpytest(p, '--dist=each', '--tx=popen//id=my')
assert result.duration < 2.0
result.stdout.fnmatch_lines([
"*killed*my*",
])
result.stdout.fnmatch_lines(["*killed*my*", ])
def test_auto_detect_cpus(testdir, monkeypatch):
@@ -400,10 +390,7 @@ def test_session_hooks(testdir):
assert hasattr(sys, 'pytestsessionhooks')
""")
result = testdir.runpytest(p, "--dist=each", "--tx=popen")
result.stdout.fnmatch_lines([
"*ValueError*",
"*1 passed*",
])
result.stdout.fnmatch_lines(["*ValueError*", "*1 passed*", ])
assert not result.ret
d = result.parseoutcomes()
assert d['passed'] == 1
@@ -446,12 +433,10 @@ def test_funcarg_teardown_failure(testdir):
pass
""")
result = testdir.runpytest_subprocess("--debug", p) # , "-n1")
result.stdout.fnmatch_lines([
"*ValueError*42*",
"*1 passed*1 error*",
])
result.stdout.fnmatch_lines(["*ValueError*42*", "*1 passed*1 error*", ])
assert result.ret
def test_crashing_item(testdir):
p = testdir.makepyfile("""
import py
@@ -463,12 +448,10 @@ def test_crashing_item(testdir):
""")
result = testdir.runpytest("-n2", p)
result.stdout.fnmatch_lines([
"*crashed*test_crash*",
"*1 failed*1 passed*"
"*crashed*test_crash*", "*1 failed*1 passed*"
])
def test_skipping(testdir):
p = testdir.makepyfile("""
import pytest
@@ -477,10 +460,8 @@ def test_skipping(testdir):
""")
result = testdir.runpytest("-n1", '-rs', p)
assert result.ret == 0
result.stdout.fnmatch_lines([
"*hello*",
"*1 skipped*"
])
result.stdout.fnmatch_lines(["*hello*", "*1 skipped*"])
def test_issue34_pluginloading_in_subprocess(testdir):
testdir.tmpdir.join("plugin123.py").write(py.code.Source("""
@@ -494,9 +475,7 @@ def test_issue34_pluginloading_in_subprocess(testdir):
""")
result = testdir.runpytest_subprocess("-n1", "-p", "plugin123")
assert result.ret == 0
result.stdout.fnmatch_lines([
"*1 passed*",
])
result.stdout.fnmatch_lines(["*1 passed*", ])
def test_fixture_scope_caching_issue503(testdir):
@@ -505,7 +484,8 @@ def test_fixture_scope_caching_issue503(testdir):
@pytest.fixture(scope='session')
def fix():
assert fix.counter == 0, 'session fixture was invoked multiple times'
assert fix.counter == 0, \
'session fixture was invoked multiple times'
fix.counter += 1
fix.counter = 0
@@ -517,9 +497,7 @@ def test_fixture_scope_caching_issue503(testdir):
""")
result = testdir.runpytest(p1, '-v', '-n1')
assert result.ret == 0
result.stdout.fnmatch_lines([
"*2 passed*",
])
result.stdout.fnmatch_lines(["*2 passed*", ])
def test_issue_594_random_parametrize(testdir):
@@ -545,7 +523,6 @@ def test_issue_594_random_parametrize(testdir):
class TestNodeFailure:
def test_load_single(self, testdir):
f = testdir.makepyfile("""
import os
@@ -617,7 +594,6 @@ class TestNodeFailure:
"*2 failed*2 passed*",
])
def test_disable_restart(self, testdir):
f = testdir.makepyfile("""
import os

18
testing/conftest.py
View File

@@ -2,6 +2,7 @@ import py
import pytest
import execnet
@pytest.fixture(scope="session", autouse=True)
def _ensure_imports():
# we import some modules because pytest-2.8's testdir fixture
@@ -10,28 +11,36 @@ def _ensure_imports():
execnet.Group
execnet.makegateway
pytest_plugins = "pytester"
# rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
@pytest.fixture(autouse=True)
def _divert_atexit(request, monkeypatch):
import atexit
l = []
def finish():
while l:
l.pop()()
monkeypatch.setattr(atexit, "register", l.append)
request.addfinalizer(finish)
def pytest_addoption(parser):
parser.addoption('--gx',
action="append", dest="gspecs",
action="append",
dest="gspecs",
help=("add a global test environment, XSpec-syntax. "))
def pytest_funcarg__specssh(request):
return getspecssh(request.config)
@pytest.fixture
def testdir(testdir):
# pytest before 2.8 did not have a runpytest_subprocess
@@ -39,10 +48,11 @@ def testdir(testdir):
testdir.runpytest_subprocess = testdir.runpytest
return testdir
# configuration information for tests
def getgspecs(config):
return [execnet.XSpec(spec)
for spec in config.getvalueorskip("gspecs")]
return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
def getspecssh(config):
xspecs = getgspecs(config)
@@ -53,10 +63,10 @@ def getspecssh(config):
return str(spec)
py.test.skip("need '--gx ssh=...'")
def getsocketspec(config):
xspecs = getgspecs(config)
for spec in xspecs:
if spec.socket:
return spec
py.test.skip("need '--gx socket=...'")

4
testing/test_boxed.py
View File

@@ -4,6 +4,7 @@ import os
needsfork = pytest.mark.skipif(not hasattr(os, "fork"),
reason="os.fork required")
@needsfork
def test_functional_boxed(testdir):
p1 = testdir.makepyfile("""
@@ -17,6 +18,7 @@ def test_functional_boxed(testdir):
"*1 failed*"
])
@needsfork
@pytest.mark.parametrize("capmode", [
"no",
@@ -41,6 +43,7 @@ def test_functional_boxed_capturing(testdir, capmode):
*1 failed*
""")
class TestOptionEffects:
def test_boxed_option_default(self, testdir):
tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
@@ -53,4 +56,3 @@ class TestOptionEffects:
def test_is_not_boxed_by_default(self, testdir):
config = testdir.parseconfig(testdir.tmpdir)
assert not config.option.boxed

27
testing/test_dsession.py
View File

@@ -1,8 +1,5 @@
from xdist.dsession import (
DSession,
LoadScheduling,
EachScheduling,
report_collection_diff,
DSession, LoadScheduling, EachScheduling, report_collection_diff,
)
import py
import pytest
@@ -10,19 +7,22 @@ import execnet
XSpec = execnet.XSpec
def run(item, node, excinfo=None):
runner = item.config.pluginmanager.getplugin("runner")
rep = runner.ItemTestReport(item=item,
excinfo=excinfo, when="call")
rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call")
rep.node = node
return rep
class MockGateway:
_count = 0
def __init__(self):
self.id = str(self._count)
self._count += 1
class MockNode:
def __init__(self):
self.sent = []
@@ -37,10 +37,12 @@ class MockNode:
def shutdown(self):
self._shutdown = True
def dumpqueue(queue):
while queue.qsize():
print(queue.get())
class TestEachScheduling:
def test_schedule_load_simple(self):
node1 = MockNode()
@@ -81,6 +83,7 @@ class TestEachScheduling:
assert sched.tests_finished()
assert not sched.hasnodes()
class TestLoadScheduling:
def test_schedule_load_simple(self):
sched = LoadScheduling(2)
@@ -149,6 +152,7 @@ class TestLoadScheduling:
Test that LoadScheduling is reporting collection errors when
different test ids are collected by slaves.
"""
class CollectHook(object):
"""
Dummy hook that stores collection reports.
@@ -177,7 +181,6 @@ class TestLoadScheduling:
class TestDistReporter:
@py.test.mark.xfail
def test_rsync_printing(self, testdir, linecomp):
config = testdir.parseconfig()
@@ -185,9 +188,11 @@ class TestDistReporter:
rep = TerminalReporter(config, file=linecomp.stringio)
config.pluginmanager.register(rep, "terminalreporter")
dsession = DSession(config)
class gw1:
id = "X1"
spec = execnet.XSpec("popen")
class gw2:
id = "X2"
spec = execnet.XSpec("popen")
@@ -202,9 +207,7 @@ class TestDistReporter:
# "*X1*popen*xyz*2.5*"
# ])
dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
linecomp.assert_contains_lines([
"[X1,X2] rsyncing: hello",
])
linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello", ])
def test_report_collection_diff_equal():
@@ -230,12 +233,12 @@ def test_report_collection_diff_different():
' bbb\n'
'+XXX\n'
' ccc\n'
'-YYY'
)
'-YYY')
msg = report_collection_diff(from_collection, to_collection, 1, 2)
assert msg == error_message
@pytest.mark.xfail(reason="duplicate test ids not supported yet")
def test_pytest_issue419(testdir):
testdir.makepyfile("""

5
testing/test_looponfail.py
View File

@@ -2,6 +2,7 @@ import py
from xdist.looponfail import RemoteControl
from xdist.looponfail import StatRecorder
class TestStatRecorder:
def test_filechange(self, tmpdir):
tmp = tmpdir
@@ -87,6 +88,7 @@ class TestStatRecorder:
sd.waitonchange(checkinterval=0.2)
assert not l
class TestRemoteControl:
def test_nofailures(self, testdir):
item = testdir.getitem("def test_func(): pass\n")
@@ -143,6 +145,7 @@ class TestRemoteControl:
control.loop_once()
assert control.failures
class TestLooponFailing:
def test_looponfail_from_fail_to_ok(self, testdir):
modcol = testdir.getmodulecol("""
@@ -269,6 +272,7 @@ class TestFunctional:
child.expect("waiting for changes")
child.kill(15)
def removepyc(path):
# XXX damn those pyc files
pyc = path + "c"
@@ -277,4 +281,3 @@ def removepyc(path):
c = path.dirpath("__pycache__")
if c.check():
c.remove()

3
testing/test_plugin.py
View File

@@ -2,6 +2,7 @@ import py
import execnet
from xdist.slavemanage import NodeManager
def test_dist_incompatibility_messages(testdir):
result = testdir.runpytest("--pdb", "--looponfail")
assert result.ret != 0
@@ -12,6 +13,7 @@ def test_dist_incompatibility_messages(testdir):
assert result.ret != 0
assert "incompatible" in result.stderr.str()
def test_dist_options(testdir):
from xdist.plugin import pytest_cmdline_main as check_options
config = testdir.parseconfigure("-n 2")
@@ -22,6 +24,7 @@ def test_dist_options(testdir):
check_options(config)
assert config.option.dist == "load"
class TestDistOptions:
def test_getxspecs(self, testdir):
config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")

7
testing/test_remote.py
View File

@@ -7,6 +7,7 @@ import marshal
WAIT_TIMEOUT = 10.0
def check_marshallable(d):
try:
marshal.dumps(d)
@@ -14,6 +15,7 @@ def check_marshallable(d):
py.std.pprint.pprint(d)
raise ValueError("not marshallable")
class EventCall:
def __init__(self, eventcall):
self.name, self.kwargs = eventcall
@@ -21,6 +23,7 @@ class EventCall:
def __str__(self):
return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
class SlaveSetup:
use_callback = False
@@ -53,9 +56,11 @@ class SlaveSetup:
def sendcommand(self, name, **kwargs):
self.slp.sendcommand(name, **kwargs)
def pytest_funcarg__slave(request):
return SlaveSetup(request)
def test_remoteinitconfig(testdir):
from xdist.remote import remote_initconfig
config1 = testdir.parseconfig()
@@ -63,6 +68,7 @@ def test_remoteinitconfig(testdir):
assert config2.option.__dict__ == config1.option.__dict__
assert config2.pluginmanager.getplugin("terminal") in (-1, None)
class TestReportSerialization:
def test_itemreport_outcomes(self, testdir):
reprec = testdir.inline_runsource("""
@@ -245,4 +251,3 @@ class TestSlaveInteractor:
("pytest_pycollect_makeitem", "name == 'test_func'"),
("pytest_collectreport", "report.collector.fspath == bbb"),
])

38
testing/test_slavemanage.py
View File

@@ -7,6 +7,7 @@ from xdist.slavemanage import HostRSync, NodeManager
pytest_plugins = "pytester"
def pytest_funcarg__hookrecorder(request, config):
hookrecorder = HookRecorder(config.pluginmanager)
if hasattr(hookrecorder, "start_recording"):
@@ -14,23 +15,32 @@ def pytest_funcarg__hookrecorder(request, config):
request.addfinalizer(hookrecorder.finish_recording)
return hookrecorder
def pytest_funcarg__config(testdir):
return testdir.parseconfig()
def pytest_funcarg__mysetup(tmpdir):
class mysetup:
source = tmpdir.mkdir("source")
dest = tmpdir.mkdir("dest")
return mysetup()
@pytest.fixture
def slavecontroller(monkeypatch):
class MockController(object):
def __init__(self, *args): pass
def setup(self): pass
def __init__(self, *args):
pass
def setup(self):
pass
monkeypatch.setattr(slavemanage, 'SlaveController', MockController)
return MockController
class TestNodeManagerPopen:
def test_popen_no_default_chdir(self, config):
gm = NodeManager(config, ["popen"])
@@ -43,7 +53,8 @@ class TestNodeManagerPopen:
for spec in NodeManager(config, l, defaultchdir="abc").specs:
assert spec.chdir == "abc"
def test_popen_makegateway_events(self, config, hookrecorder, slavecontroller):
def test_popen_makegateway_events(self, config, hookrecorder,
slavecontroller):
hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None)
call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -64,12 +75,16 @@ class TestNodeManagerPopen:
hm.setup_nodes(None)
assert len(hm.group) == 2
for gw in hm.group:
class pseudoexec:
args = []
def __init__(self, *args):
self.args.extend(args)
def waitclose(self):
pass
gw.remote_exec = pseudoexec
l = []
for gw in hm.group:
@@ -95,8 +110,8 @@ class TestNodeManagerPopen:
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
def test_rsync_same_popen_twice(self, config, mysetup,
hookrecorder, slavecontroller):
def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder,
slavecontroller):
source, dest = mysetup.source, mysetup.dest
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
hm.roots = []
@@ -110,6 +125,7 @@ class TestNodeManagerPopen:
assert call.gateways[0] in hm.group
call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
class TestHRSync:
def test_hrsync_filter(self, mysetup):
source, _ = mysetup.source, mysetup.dest # noqa
@@ -118,8 +134,7 @@ class TestHRSync:
source.ensure(".somedotfile", "moreentries")
source.ensure("somedir", "editfile~")
syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
l = list(source.visit(rec=syncer.filter,
fil=syncer.filter))
l = list(source.visit(rec=syncer.filter, fil=syncer.filter))
assert len(l) == 3
basenames = [x.basename for x in l]
assert 'dir' in basenames
@@ -164,10 +179,8 @@ class TestNodeManager:
for rsyncroot in (dir1, source):
dest.remove()
nodemanager = NodeManager(testdir.parseconfig(
"--tx", "popen//chdir=%s" % dest,
"--rsyncdir", rsyncroot,
source,
))
"--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot,
source, ))
nodemanager.setup_nodes(None) # calls .rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
@@ -230,7 +243,8 @@ class TestNodeManager:
assert not gwspec.chdir
def test_ssh_setup_nodes(self, specssh, testdir):
testdir.makepyfile(__init__="", test_x="""
testdir.makepyfile(__init__="",
test_x="""
def test_one():
pass
""")

4
tox.ini
View File

@@ -21,8 +21,8 @@ commands=
[testenv:flakes]
changedir=
deps = pytest-flakes>=0.2
commands = py.test --flakes -m flakes testing xdist
deps = flake8
commands = flake8 setup.py testing xdist
[testenv:readme]
changedir =

6
xdist/boxed.py
View File

@@ -4,7 +4,8 @@ import py
def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing")
group.addoption('--boxed',
group.addoption(
'--boxed',
action="store_true", dest="boxed", default=False,
help="box each test run in a separate process (unix)")
@@ -16,6 +17,7 @@ def pytest_runtest_protocol(item):
item.ihook.pytest_runtest_logreport(report=rep)
return True
def forked_run_report(item):
# for now, we run setup/teardown in the subprocess
# XXX optionally allow sharing of setup/teardown
@@ -24,6 +26,7 @@ def forked_run_report(item):
import marshal
from xdist.remote import serialize_report
from xdist.slavemanage import unserialize_report
def runforked():
try:
reports = runtestprotocol(item, log=False)
@@ -41,6 +44,7 @@ def forked_run_report(item):
py.test.exit("forked test item %s raised Exit" % (item,))
return [report_process_crash(item, result)]
def report_process_crash(item, result):
path, lineno = item._getfslineno()
info = ("%s:%s: running the test CRASHED with signal %d" %

9
xdist/dsession.py
View File

@@ -257,8 +257,9 @@ class LoadScheduling:
assert node in self.node2pending
if self.collection_is_completed:
# A new node has been added later, perhaps an original one died.
assert self.collection # .init_distribute() should have
# .init_distribute() should have
# been called by now
assert self.collection
if collection != self.collection:
other_node = next(iter(self.node2collection.keys()))
msg = report_collection_diff(self.collection,
@@ -398,8 +399,9 @@ class LoadScheduling:
same_collection = False
self.log(msg)
if self.config is not None:
rep = CollectReport(node.gateway.id, 'failed', longrepr=msg,
result=[])
rep = CollectReport(
node.gateway.id, 'failed',
longrepr=msg, result=[])
self.config.hook.pytest_collectreport(report=rep)
return same_collection
@@ -769,4 +771,3 @@ class TerminalDistReporter:
# def pytest_xdist_rsyncfinish(self, source, gateways):
# targets = ", ".join(["[%s]" % gw.id for gw in gateways])
# self.write_line("rsyncfinish: %s -> %s" %(source, targets))

29
xdist/looponfail.py
View File

@@ -7,17 +7,21 @@
the controlling process which should best never happen.
"""
import py, pytest
import py
import pytest
import sys
import execnet
def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing")
group._addoption('-f', '--looponfail',
group._addoption(
'-f', '--looponfail',
action="store_true", dest="looponfail", default=False,
help="run tests in subprocess, wait for modified files "
"and re-run failing test set until all pass.")
def pytest_cmdline_main(config):
if config.getoption("looponfail"):
@@ -25,7 +29,6 @@ def pytest_cmdline_main(config):
return 2 # looponfail only can get stop with ctrl-C anyway
def looponfail_main(config):
remotecontrol = RemoteControl(config)
rootdirs = config.getini("looponfailroots")
@@ -34,7 +37,8 @@ def looponfail_main(config):
while 1:
remotecontrol.loop_once()
if not remotecontrol.failures and remotecontrol.wasfailing:
continue # the last failures passed, let's immediately rerun all
# the last failures passed, let's immediately rerun all
continue
repr_pytest_looponfailinfo(
failreports=remotecontrol.failures,
rootdirs=rootdirs)
@@ -42,6 +46,7 @@ def looponfail_main(config):
except KeyboardInterrupt:
print()
class RemoteControl(object):
def __init__(self, config):
self.config = config
@@ -62,11 +67,13 @@ class RemoteControl(object):
raise ValueError("already have gateway %r" % self.gateway)
self.trace("setting up slave session")
self.gateway = self.initgateway()
self.channel = channel = self.gateway.remote_exec(init_slave_session,
self.channel = channel = self.gateway.remote_exec(
init_slave_session,
args=self.config.args,
option_dict=vars(self.config.option),
)
remote_outchannel = channel.receive()
def write(s):
out._file.write(s)
out._file.flush()
@@ -110,6 +117,7 @@ class RemoteControl(object):
uniq_failures.append(failure)
self.failures = uniq_failures
def repr_pytest_looponfailinfo(failreports, rootdirs):
tr = py.io.TerminalWriter()
if failreports:
@@ -123,7 +131,8 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
def init_slave_session(channel, args, option_dict):
import os, sys
import os
import sys
outchannel = channel.gateway.newchannel()
sys.stdout = sys.stderr = outchannel.makefile('w')
channel.send(outchannel)
@@ -143,6 +152,7 @@ def init_slave_session(channel, args, option_dict):
from xdist.looponfail import SlaveFailSession
SlaveFailSession(config, channel).main()
class SlaveFailSession:
def __init__(self, config, channel):
self.config = config
@@ -165,7 +175,8 @@ class SlaveFailSession:
items = session.perform_collect(self.trails or None)
except pytest.UsageError:
items = session.perform_collect(None)
hook.pytest_collection_modifyitems(session=session, config=session.config, items=items)
hook.pytest_collection_modifyitems(
session=session, config=session.config, items=items)
hook.pytest_collection_finish(session=session)
return True
@@ -195,6 +206,7 @@ class SlaveFailSession:
failreports.append(loc)
self.channel.send((trails, failreports, self.collection_failed))
class StatRecorder:
def __init__(self, rootdirlist):
self.rootdirlist = rootdirlist
@@ -203,6 +215,7 @@ class StatRecorder:
def fil(self, p):
return p.check(file=1, dotfile=0) and p.ext != ".pyc"
def rec(self, p):
return p.check(dotfile=0)
@@ -213,7 +226,7 @@ class StatRecorder:
return
py.std.time.sleep(checkinterval)
def check(self, removepycfiles=True):
def check(self, removepycfiles=True): # noqa, too complex
changed = False
statcache = self.statcache
newstat = {}

6
xdist/newhooks.py
View File

@@ -2,20 +2,26 @@
def pytest_xdist_setupnodes(config, specs):
""" called before any remote node is set up. """
def pytest_xdist_newgateway(gateway):
""" called on new raw gateway creation. """
def pytest_xdist_rsyncstart(source, gateways):
""" called before rsyncing a directory to remote gateways takes place. """
def pytest_xdist_rsyncfinish(source, gateways):
""" called after rsyncing a directory to remote gateways takes place. """
def pytest_configure_node(node):
""" configure node information before it gets instantiated. """
def pytest_testnodeready(node):
""" Test Node is ready to operate. """
def pytest_testnodedown(node, error):
""" Test Node is down. """

36
xdist/plugin.py
View File

@@ -12,7 +12,8 @@ def parse_numprocesses(s):
def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing")
group._addoption('-n', dest="numprocesses", metavar="numprocesses",
group._addoption(
'-n', dest="numprocesses", metavar="numprocesses",
action="store",
type=parse_numprocesses,
help="shortcut for '--dist=load --tx=NUM*popen', "
@@ -21,36 +22,46 @@ def pytest_addoption(parser):
group._addoption('--max-slave-restart', action="store", default=None,
help="maximum number of slaves that can be restarted "
"when crashed (set to zero to disable this feature)")
group._addoption('--dist', metavar="distmode",
group._addoption(
'--dist', metavar="distmode",
action="store", choices=['load', 'each', 'no'],
type="choice", dest="dist", default="no",
help=("set mode for distributing tests to exec environments.\n\n"
"each: send each test to each available environment.\n\n"
"load: send each test to available environment.\n\n"
"(default) no: run tests inprocess, don't distribute."))
group._addoption('--tx', dest="tx", action="append", default=[],
group._addoption(
'--tx', dest="tx", action="append", default=[],
metavar="xspec",
help=("add a test execution environment. some examples: "
"--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
"--tx ssh=user@codespeak.net//chdir=testcache"))
group._addoption('-d',
group._addoption(
'-d',
action="store_true", dest="distload", default=False,
help="load-balance tests. shortcut for '--dist=load'")
group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
group.addoption(
'--rsyncdir', action="append", default=[], metavar="DIR",
help="add directory for rsyncing to remote tx nodes.")
group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
group.addoption(
'--rsyncignore', action="append", default=[], metavar="GLOB",
help="add expression for ignores when rsyncing to remote tx nodes.")
parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
parser.addini(
'rsyncdirs', 'list of (relative) paths to be rsynced for'
' remote distributed testing.', type="pathlist")
parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
parser.addini(
'rsyncignore', 'list of (relative) glob-style paths to be ignored '
'for rsyncing.', type="pathlist")
parser.addini("looponfailroots", type="pathlist",
parser.addini(
"looponfailroots", type="pathlist",
help="directories to check for changes", default=[py.path.local()])
# -------------------------------------------------------------------------
# distributed testing hooks
# -------------------------------------------------------------------------
def pytest_addhooks(pluginmanager):
from xdist import newhooks
# avoid warnings with pytest-2.8
@@ -73,6 +84,7 @@ def pytest_configure(config):
tr = config.pluginmanager.getplugin("terminalreporter")
tr.showfspath = False
@pytest.mark.tryfirst
def pytest_cmdline_main(config):
if config.option.numprocesses:
@@ -85,7 +97,9 @@ def pytest_cmdline_main(config):
usepdb = config.option.usepdb # a core option
if val("looponfail"):
if usepdb:
raise pytest.UsageError("--pdb incompatible with --looponfail.")
raise pytest.UsageError(
"--pdb incompatible with --looponfail.")
elif val("dist") != "no":
if usepdb:
raise pytest.UsageError("--pdb incompatible with distributing tests.")
raise pytest.UsageError(
"--pdb incompatible with distributing tests.")

13
xdist/remote.py
View File

@@ -6,7 +6,9 @@
needs not to be installed in remote environments.
"""
import sys, os
import sys
import os
class SlaveInteractor:
def __init__(self, config, channel):
@@ -72,7 +74,8 @@ class SlaveInteractor:
nextitem=nextitem)
def pytest_collection_finish(self, session):
self.sendevent("collectionfinish",
self.sendevent(
"collectionfinish",
topdir=str(session.fspath),
ids=[item.nodeid for item in session.items])
@@ -89,6 +92,7 @@ class SlaveInteractor:
data = serialize_report(report)
self.sendevent("collectreport", data=data)
def serialize_report(rep):
import py
d = rep.__dict__.copy()
@@ -103,6 +107,7 @@ def serialize_report(rep):
d[name] = None # for now
return d
def getinfodict():
import platform
return dict(
@@ -114,6 +119,7 @@ def getinfodict():
cwd=os.getcwd(),
)
def remote_initconfig(option_dict, args):
from _pytest.config import Config
option_dict['plugins'].append("no:terminal")
@@ -136,7 +142,8 @@ if __name__ == '__channelexec__':
slaveinput, args, option_dict = channel.receive()
importpath = os.getcwd()
sys.path.insert(0, importpath) # XXX only for remote situations
os.environ['PYTHONPATH'] = (importpath + os.pathsep +
os.environ['PYTHONPATH'] = (
importpath + os.pathsep +
os.environ.get('PYTHONPATH', ''))
# os.environ['PYTHONPATH'] = importpath
import py

29
xdist/slavemanage.py
View File

@@ -1,5 +1,6 @@
import fnmatch
import os
import re
import py
import pytest
@@ -8,9 +9,11 @@ import xdist.remote
from _pytest import runner # XXX load dynamically
class NodeManager(object):
EXIT_TIMEOUT = 10
DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
self._nodesready = py.std.threading.Event()
@@ -79,7 +82,8 @@ class NodeManager(object):
break
else:
return []
import pytest, _pytest
import pytest
import _pytest
pytestpath = pytest.__file__.rstrip("co")
pytestdir = py.path.local(_pytest.__file__).dirpath()
config = self.config
@@ -124,6 +128,7 @@ class NodeManager(object):
return
if (spec, source) in self._rsynced_specs:
return
def finished():
if notify:
notify("rsyncrootready", spec, source)
@@ -139,19 +144,23 @@ class NodeManager(object):
gateways=[gateway],
)
class HostRSync(execnet.RSync):
""" RSyncer that filters out common files
"""
def __init__(self, sourcedir, *args, **kwargs):
self._synced = {}
self._ignores = kwargs.pop('ignores', None) or []
self._ignores = []
ignores = kwargs.pop('ignores', None) or []
for x in ignores:
x = getattr(x, 'strpath', x)
self.ignores.append(re.compile(fnmatch.translate(x)))
super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
def filter(self, path):
path = py.path.local(path)
for x in self._ignores:
x = getattr(x, 'strpath', x)
if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
for check in self._ignores:
if check(path.basename) or check(path.strpath):
return False
else:
return True
@@ -187,6 +196,7 @@ def make_reltoroot(roots, args):
l.append(splitcode.join(parts))
return l
class SlaveController(object):
ENDMARK = -1
@@ -219,7 +229,8 @@ class SlaveController(object):
self.channel = self.gateway.remote_exec(xdist.remote)
self.channel.send((self.slaveinput, args, option_dict))
if self.putevent:
self.channel.setcallback(self.process_from_remote,
self.channel.setcallback(
self.process_from_remote,
endmarker=self.ENDMARK)
def ensure_teardown(self):
@@ -255,7 +266,7 @@ class SlaveController(object):
self.log("queuing %s(**%s)" % (eventname, kwargs))
self.putevent((eventname, kwargs))
def process_from_remote(self, eventcall):
def process_from_remote(self, eventcall): # noqa too complex
""" this gets called for each object we receive from
the other side and if the channel closes.
@@ -283,7 +294,8 @@ class SlaveController(object):
self.notify_inproc("slavefinished", node=self)
elif eventname == "logstart":
self.notify_inproc(eventname, node=self, **kwargs)
elif eventname in ("testreport", "collectreport", "teardownreport"):
elif eventname in (
"testreport", "collectreport", "teardownreport"):
item_index = kwargs.pop("item_index", None)
rep = unserialize_report(eventname, kwargs['data'])
if item_index is not None:
@@ -301,6 +313,7 @@ class SlaveController(object):
py.builtin.print_("!" * 20, excinfo)
self.config.pluginmanager.notify_exception(excinfo)
def unserialize_report(name, reportdict):
if name == "testreport":
return runner.TestReport(**reportdict)