From 0f5ef95be80bbccd0f6a8e45a121e768f79a8166 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 1 Sep 2015 22:37:51 +0200 Subject: [PATCH] flake8 cleanup --- .travis.yml | 2 +- setup.py | 7 ++- testing/acceptance_test.py | 106 ++++++++++++++---------------------- testing/conftest.py | 22 ++++++-- testing/test_boxed.py | 4 +- testing/test_dsession.py | 41 +++++++------- testing/test_looponfail.py | 13 +++-- testing/test_plugin.py | 9 ++- testing/test_remote.py | 17 ++++-- testing/test_slavemanage.py | 40 +++++++++----- tox.ini | 4 +- xdist/boxed.py | 12 ++-- xdist/dsession.py | 25 +++++---- xdist/looponfail.py | 53 +++++++++++------- xdist/newhooks.py | 6 ++ xdist/plugin.py | 78 +++++++++++++++----------- xdist/remote.py | 35 +++++++----- xdist/slavemanage.py | 47 ++++++++++------ 18 files changed, 298 insertions(+), 223 deletions(-) diff --git a/.travis.yml b/.travis.yml index d98b0ae..3443209 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ env: - TESTENV=py27-pytest27-pexpect - TESTENV=py34-pytest27-pexpect - - TESTENV=py35-pytest27 +# - TESTENV=py35-pytest27 - TESTENV=pypy-pytest27 script: tox --recreate -e $TESTENV diff --git a/setup.py b/setup.py index 9c2c7b4..d4ee54f 100644 --- a/setup.py +++ b/setup.py @@ -3,15 +3,16 @@ from setuptools import setup setup( name="pytest-xdist", use_scm_version={'write_to': 'xdist/_version.py'}, - description='py.test xdist plugin for distributed testing and loop-on-failing modes', + description='py.test xdist plugin for distributed testing' + ' and loop-on-failing modes', long_description=open('README.rst').read(), license='MIT', author='holger krekel and contributors', author_email='pytest-dev@python.org,holger@merlinux.eu', url='http://bitbucket.org/hpk42/pytest-xdist', platforms=['linux', 'osx', 'win32'], - packages = ['xdist'], - entry_points = { + packages=['xdist'], + entry_points={ 'pytest11': [ 'xdist = xdist.plugin', 'xdist.looponfail = xdist.looponfail', diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index ec53c67..9297bfb 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -10,9 +10,7 @@ class TestDistribution: """) result = testdir.runpytest(p1, "-n1") assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*1 passed*", - ]) + result.stdout.fnmatch_lines(["*1 passed*", ]) def test_n1_fail(self, testdir): p1 = testdir.makepyfile(""" @@ -21,9 +19,7 @@ class TestDistribution: """) result = testdir.runpytest(p1, "-n1") assert result.ret == 1 - result.stdout.fnmatch_lines([ - "*1 failed*", - ]) + result.stdout.fnmatch_lines(["*1 failed*", ]) def test_n1_import_error(self, testdir): p1 = testdir.makepyfile(""" @@ -57,9 +53,7 @@ class TestDistribution: """) result = testdir.runpytest(p1, "-n1") assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*1 skipped*", - ]) + result.stdout.fnmatch_lines(["*1 skipped*", ]) def test_manytests_to_one_import_error(self, testdir): p1 = testdir.makepyfile(""" @@ -84,8 +78,7 @@ class TestDistribution: pass def test_skip(): py.test.skip("hello") - """, - ) + """, ) result = testdir.runpytest(p1, "-v", '-d', '--tx=popen', '--tx=popen') result.stdout.fnmatch_lines([ "*1*Python*", @@ -115,9 +108,7 @@ class TestDistribution: """ % str(testdir.tmpdir)) result = testdir.runpytest_subprocess(p1, "-n1") assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*1 passed*", - ]) + result.stdout.fnmatch_lines(["*1 passed*", ]) def test_dist_ini_specified(self, testdir): p1 = testdir.makepyfile(""" @@ -130,8 +121,7 @@ class TestDistribution: pass def test_skip(): py.test.skip("hello") - """, - ) + """, ) testdir.makeini(""" [pytest] addopts = --tx=3*popen @@ -163,13 +153,10 @@ class TestDistribution: import os time.sleep(0.5) os.kill(os.getpid(), 15) - """ - ) + """) result = testdir.runpytest(p1, "-v", '-d', '-n1') result.stdout.fnmatch_lines([ - "*Python*", - "*PASS**test_ok*", - "*node*down*", + "*Python*", "*PASS**test_ok*", "*node*down*", "*3 failed, 1 passed, 1 skipped*" ]) assert result.ret == 1 @@ -182,13 +169,13 @@ class TestDistribution: p = subdir.join("test_one.py") p.write("def test_5():\n assert not __file__.startswith(%r)" % str(p)) result = testdir.runpytest("-v", "-d", - "--rsyncdir=%(subdir)s" % locals(), - "--tx=popen//chdir=%(dest)s" % locals(), p) + "--rsyncdir=%(subdir)s" % locals(), + "--tx=popen//chdir=%(dest)s" % locals(), p) assert result.ret == 0 result.stdout.fnmatch_lines([ "*0* *cwd*", - #"RSyncStart: [G1]", - #"RSyncFinished: [G1]", + # "RSyncStart: [G1]", + # "RSyncFinished: [G1]", "*1 passed*" ]) assert dest.join(subdir.basename).check(dir=1) @@ -221,14 +208,13 @@ class TestDistribution: p1 = testdir.makepyfile("def test_func(): pass") result = testdir.runpytest("-v", p1, '-d', '--tx=popen') result.stdout.fnmatch_lines([ - "*0*Python*", - "*calculated result is 49*", - "*1 passed*" + "*0*Python*", "*calculated result is 49*", "*1 passed*" ]) assert result.ret == 0 def test_keyboardinterrupt_hooks_issue79(self, testdir): - testdir.makepyfile(__init__="", test_one=""" + testdir.makepyfile(__init__="", + test_one=""" def test_hello(): raise KeyboardInterrupt() """) @@ -258,11 +244,12 @@ class TestDistribution: """) child = testdir.spawn_pytest("-n1 -v") child.expect(".*test_sleep.*") - child.kill(2) # keyboard interrupt + child.kill(2) # keyboard interrupt child.expect(".*KeyboardInterrupt.*") - #child.expect(".*seconds.*") + # child.expect(".*seconds.*") child.close() - #assert ret == 2 + # assert ret == 2 + class TestDistEach: def test_simple(self, testdir): @@ -270,11 +257,13 @@ class TestDistEach: def test_hello(): pass """) - result = testdir.runpytest_subprocess("--debug", "--dist=each", "--tx=2*popen") + result = testdir.runpytest_subprocess("--debug", "--dist=each", + "--tx=2*popen") assert not result.ret result.stdout.fnmatch_lines(["*2 pass*"]) - @py.test.mark.xfail(run=False, + @py.test.mark.xfail( + run=False, reason="other python versions might not have py.test installed") def test_simple_diffoutput(self, testdir): interpreters = [] @@ -284,7 +273,8 @@ class TestDistEach: py.test.skip("%s not found" % name) interpreters.append(interp) - testdir.makepyfile(__init__="", test_one=""" + testdir.makepyfile(__init__="", + test_one=""" import sys def test_hello(): print("%s...%s" % sys.version_info[:2]) @@ -298,6 +288,7 @@ class TestDistEach: assert "2...5" in s assert "2...6" in s + class TestTerminalReporting: def test_pass_skip_fail(self, testdir): testdir.makepyfile(""" @@ -335,6 +326,7 @@ class TestTerminalReporting: "E assert 0", ]) + def test_teardownfails_one_function(testdir): p = testdir.makepyfile(""" def test_func(): @@ -344,10 +336,10 @@ def test_teardownfails_one_function(testdir): """) result = testdir.runpytest(p, '-n1', '--tx=popen') result.stdout.fnmatch_lines([ - "*def teardown_function(function):*", - "*1 passed*1 error*" + "*def teardown_function(function):*", "*1 passed*1 error*" ]) + @py.test.mark.xfail def test_terminate_on_hangingnode(testdir): p = testdir.makeconftest(""" @@ -358,9 +350,7 @@ def test_terminate_on_hangingnode(testdir): """) result = testdir.runpytest(p, '--dist=each', '--tx=popen//id=my') assert result.duration < 2.0 - result.stdout.fnmatch_lines([ - "*killed*my*", - ]) + result.stdout.fnmatch_lines(["*killed*my*", ]) def test_auto_detect_cpus(testdir, monkeypatch): @@ -400,10 +390,7 @@ def test_session_hooks(testdir): assert hasattr(sys, 'pytestsessionhooks') """) result = testdir.runpytest(p, "--dist=each", "--tx=popen") - result.stdout.fnmatch_lines([ - "*ValueError*", - "*1 passed*", - ]) + result.stdout.fnmatch_lines(["*ValueError*", "*1 passed*", ]) assert not result.ret d = result.parseoutcomes() assert d['passed'] == 1 @@ -445,13 +432,11 @@ def test_funcarg_teardown_failure(testdir): def test_hello(myarg): pass """) - result = testdir.runpytest_subprocess("--debug", p) # , "-n1") - result.stdout.fnmatch_lines([ - "*ValueError*42*", - "*1 passed*1 error*", - ]) + result = testdir.runpytest_subprocess("--debug", p) # , "-n1") + result.stdout.fnmatch_lines(["*ValueError*42*", "*1 passed*1 error*", ]) assert result.ret + def test_crashing_item(testdir): p = testdir.makepyfile(""" import py @@ -463,12 +448,10 @@ def test_crashing_item(testdir): """) result = testdir.runpytest("-n2", p) result.stdout.fnmatch_lines([ - "*crashed*test_crash*", - "*1 failed*1 passed*" + "*crashed*test_crash*", "*1 failed*1 passed*" ]) - def test_skipping(testdir): p = testdir.makepyfile(""" import pytest @@ -477,10 +460,8 @@ def test_skipping(testdir): """) result = testdir.runpytest("-n1", '-rs', p) assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*hello*", - "*1 skipped*" - ]) + result.stdout.fnmatch_lines(["*hello*", "*1 skipped*"]) + def test_issue34_pluginloading_in_subprocess(testdir): testdir.tmpdir.join("plugin123.py").write(py.code.Source(""" @@ -494,9 +475,7 @@ def test_issue34_pluginloading_in_subprocess(testdir): """) result = testdir.runpytest_subprocess("-n1", "-p", "plugin123") assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*1 passed*", - ]) + result.stdout.fnmatch_lines(["*1 passed*", ]) def test_fixture_scope_caching_issue503(testdir): @@ -505,7 +484,8 @@ def test_fixture_scope_caching_issue503(testdir): @pytest.fixture(scope='session') def fix(): - assert fix.counter == 0, 'session fixture was invoked multiple times' + assert fix.counter == 0, \ + 'session fixture was invoked multiple times' fix.counter += 1 fix.counter = 0 @@ -517,9 +497,7 @@ def test_fixture_scope_caching_issue503(testdir): """) result = testdir.runpytest(p1, '-v', '-n1') assert result.ret == 0 - result.stdout.fnmatch_lines([ - "*2 passed*", - ]) + result.stdout.fnmatch_lines(["*2 passed*", ]) def test_issue_594_random_parametrize(testdir): @@ -545,7 +523,6 @@ def test_issue_594_random_parametrize(testdir): class TestNodeFailure: - def test_load_single(self, testdir): f = testdir.makepyfile(""" import os @@ -617,7 +594,6 @@ class TestNodeFailure: "*2 failed*2 passed*", ]) - def test_disable_restart(self, testdir): f = testdir.makepyfile(""" import os diff --git a/testing/conftest.py b/testing/conftest.py index 9f31a31..70537cd 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -2,6 +2,7 @@ import py import pytest import execnet + @pytest.fixture(scope="session", autouse=True) def _ensure_imports(): # we import some modules because pytest-2.8's testdir fixture @@ -10,28 +11,36 @@ def _ensure_imports(): execnet.Group execnet.makegateway + pytest_plugins = "pytester" -#rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()] +# rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()] + @pytest.fixture(autouse=True) def _divert_atexit(request, monkeypatch): import atexit l = [] + def finish(): while l: l.pop()() + monkeypatch.setattr(atexit, "register", l.append) request.addfinalizer(finish) + def pytest_addoption(parser): parser.addoption('--gx', - action="append", dest="gspecs", - help=("add a global test environment, XSpec-syntax. ")) + action="append", + dest="gspecs", + help=("add a global test environment, XSpec-syntax. ")) + def pytest_funcarg__specssh(request): return getspecssh(request.config) + @pytest.fixture def testdir(testdir): # pytest before 2.8 did not have a runpytest_subprocess @@ -39,10 +48,11 @@ def testdir(testdir): testdir.runpytest_subprocess = testdir.runpytest return testdir + # configuration information for tests def getgspecs(config): - return [execnet.XSpec(spec) - for spec in config.getvalueorskip("gspecs")] + return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")] + def getspecssh(config): xspecs = getgspecs(config) @@ -53,10 +63,10 @@ def getspecssh(config): return str(spec) py.test.skip("need '--gx ssh=...'") + def getsocketspec(config): xspecs = getgspecs(config) for spec in xspecs: if spec.socket: return spec py.test.skip("need '--gx socket=...'") - diff --git a/testing/test_boxed.py b/testing/test_boxed.py index bee9367..d624a95 100644 --- a/testing/test_boxed.py +++ b/testing/test_boxed.py @@ -4,6 +4,7 @@ import os needsfork = pytest.mark.skipif(not hasattr(os, "fork"), reason="os.fork required") + @needsfork def test_functional_boxed(testdir): p1 = testdir.makepyfile(""" @@ -17,6 +18,7 @@ def test_functional_boxed(testdir): "*1 failed*" ]) + @needsfork @pytest.mark.parametrize("capmode", [ "no", @@ -41,6 +43,7 @@ def test_functional_boxed_capturing(testdir, capmode): *1 failed* """) + class TestOptionEffects: def test_boxed_option_default(self, testdir): tmpdir = testdir.tmpdir.ensure("subdir", dir=1) @@ -53,4 +56,3 @@ class TestOptionEffects: def test_is_not_boxed_by_default(self, testdir): config = testdir.parseconfig(testdir.tmpdir) assert not config.option.boxed - diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 4785018..234f94f 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -1,8 +1,5 @@ from xdist.dsession import ( - DSession, - LoadScheduling, - EachScheduling, - report_collection_diff, + DSession, LoadScheduling, EachScheduling, report_collection_diff, ) import py import pytest @@ -10,19 +7,22 @@ import execnet XSpec = execnet.XSpec + def run(item, node, excinfo=None): runner = item.config.pluginmanager.getplugin("runner") - rep = runner.ItemTestReport(item=item, - excinfo=excinfo, when="call") + rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call") rep.node = node return rep + class MockGateway: _count = 0 + def __init__(self): self.id = str(self._count) self._count += 1 + class MockNode: def __init__(self): self.sent = [] @@ -35,12 +35,14 @@ class MockNode: self.sent.append("ALL") def shutdown(self): - self._shutdown=True + self._shutdown = True + def dumpqueue(queue): while queue.qsize(): print(queue.get()) + class TestEachScheduling: def test_schedule_load_simple(self): node1 = MockNode() @@ -81,6 +83,7 @@ class TestEachScheduling: assert sched.tests_finished() assert not sched.hasnodes() + class TestLoadScheduling: def test_schedule_load_simple(self): sched = LoadScheduling(2) @@ -115,7 +118,7 @@ class TestLoadScheduling: sched.addnode_collection(node1, col) sched.addnode_collection(node2, col) sched.init_distribute() - #assert not sched.tests_finished() + # assert not sched.tests_finished() sent1 = node1.sent sent2 = node2.sent assert sent1 == [0, 1] @@ -149,6 +152,7 @@ class TestLoadScheduling: Test that LoadScheduling is reporting collection errors when different test ids are collected by slaves. """ + class CollectHook(object): """ Dummy hook that stores collection reports. @@ -177,7 +181,6 @@ class TestLoadScheduling: class TestDistReporter: - @py.test.mark.xfail def test_rsync_printing(self, testdir, linecomp): config = testdir.parseconfig() @@ -185,26 +188,26 @@ class TestDistReporter: rep = TerminalReporter(config, file=linecomp.stringio) config.pluginmanager.register(rep, "terminalreporter") dsession = DSession(config) + class gw1: id = "X1" spec = execnet.XSpec("popen") + class gw2: id = "X2" spec = execnet.XSpec("popen") - #class rinfo: + # class rinfo: # version_info = (2, 5, 1, 'final', 0) # executable = "hello" # platform = "xyz" # cwd = "qwe" - #dsession.pytest_xdist_newgateway(gw1, rinfo) - #linecomp.assert_contains_lines([ - # "*X1*popen*xyz*2.5*" - #]) + # dsession.pytest_xdist_newgateway(gw1, rinfo) + # linecomp.assert_contains_lines([ + # "*X1*popen*xyz*2.5*" + # ]) dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2]) - linecomp.assert_contains_lines([ - "[X1,X2] rsyncing: hello", - ]) + linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello", ]) def test_report_collection_diff_equal(): @@ -230,12 +233,12 @@ def test_report_collection_diff_different(): ' bbb\n' '+XXX\n' ' ccc\n' - '-YYY' - ) + '-YYY') msg = report_collection_diff(from_collection, to_collection, 1, 2) assert msg == error_message + @pytest.mark.xfail(reason="duplicate test ids not supported yet") def test_pytest_issue419(testdir): testdir.makepyfile(""" diff --git a/testing/test_looponfail.py b/testing/test_looponfail.py index 4aadb32..cad56c3 100644 --- a/testing/test_looponfail.py +++ b/testing/test_looponfail.py @@ -2,6 +2,7 @@ import py from xdist.looponfail import RemoteControl from xdist.looponfail import StatRecorder + class TestStatRecorder: def test_filechange(self, tmpdir): tmp = tmpdir @@ -87,6 +88,7 @@ class TestStatRecorder: sd.waitonchange(checkinterval=0.2) assert not l + class TestRemoteControl: def test_nofailures(self, testdir): item = testdir.getitem("def test_func(): pass\n") @@ -136,13 +138,14 @@ class TestRemoteControl: parent = modcol.fspath.dirpath().dirpath() parent.chdir() modcol.config.args = [py.path.local(x).relto(parent) - for x in modcol.config.args] + for x in modcol.config.args] control = RemoteControl(modcol.config) control.loop_once() assert control.failures control.loop_once() assert control.failures + class TestLooponFailing: def test_looponfail_from_fail_to_ok(self, testdir): modcol = testdir.getmodulecol(""" @@ -225,7 +228,7 @@ class TestLooponFailing: def runsession_dups(): # twisted.trial test cases may report multiple errors. failures, reports, collection_failed = orig_runsession() - print (failures) + print(failures) return failures * 2, reports, collection_failed monkeypatch.setattr(remotecontrol, 'runsession', runsession_dups) @@ -240,8 +243,8 @@ class TestFunctional: x = 0 assert x == 1 """) - #p = testdir.mkdir("sub").join(p1.basename) - #p1.move(p) + # p = testdir.mkdir("sub").join(p1.basename) + # p1.move(p) child = testdir.spawn_pytest("-f %s --traceconfig" % p) child.expect("def test_one") child.expect("x == 1") @@ -269,6 +272,7 @@ class TestFunctional: child.expect("waiting for changes") child.kill(15) + def removepyc(path): # XXX damn those pyc files pyc = path + "c" @@ -277,4 +281,3 @@ def removepyc(path): c = path.dirpath("__pycache__") if c.check(): c.remove() - diff --git a/testing/test_plugin.py b/testing/test_plugin.py index b856cde..c3e87c7 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -2,6 +2,7 @@ import py import execnet from xdist.slavemanage import NodeManager + def test_dist_incompatibility_messages(testdir): result = testdir.runpytest("--pdb", "--looponfail") assert result.ret != 0 @@ -12,6 +13,7 @@ def test_dist_incompatibility_messages(testdir): assert result.ret != 0 assert "incompatible" in result.stderr.str() + def test_dist_options(testdir): from xdist.plugin import pytest_cmdline_main as check_options config = testdir.parseconfigure("-n 2") @@ -22,6 +24,7 @@ def test_dist_options(testdir): check_options(config) assert config.option.dist == "load" + class TestDistOptions: def test_getxspecs(self, testdir): config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz") @@ -33,7 +36,7 @@ class TestDistOptions: assert xspecs[1].ssh == "xyz" def test_xspecs_multiplied(self, testdir): - config = testdir.parseconfigure("--tx=3*popen",) + config = testdir.parseconfigure("--tx=3*popen", ) xspecs = NodeManager(config)._getxspecs() assert len(xspecs) == 3 assert xspecs[1].popen @@ -60,10 +63,10 @@ class TestDistOptions: rsyncdirs= x """) config = testdir.parseconfigure( - testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z') + testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z') nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")]) roots = nm._getrsyncdirs() - #assert len(roots) == 3 + 1 # pylib + # assert len(roots) == 3 + 1 # pylib assert py.path.local('y') in roots assert py.path.local('z') in roots assert testdir.tmpdir.join('x') in roots diff --git a/testing/test_remote.py b/testing/test_remote.py index 4a043e7..53dc2f2 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -7,6 +7,7 @@ import marshal WAIT_TIMEOUT = 10.0 + def check_marshallable(d): try: marshal.dumps(d) @@ -14,12 +15,14 @@ def check_marshallable(d): py.std.pprint.pprint(d) raise ValueError("not marshallable") + class EventCall: def __init__(self, eventcall): self.name, self.kwargs = eventcall def __str__(self): - return "" %(self.name, self.kwargs) + return "" % (self.name, self.kwargs) + class SlaveSetup: use_callback = False @@ -31,7 +34,7 @@ class SlaveSetup: def setup(self, ): self.testdir.chdir() - #import os ; os.environ['EXECNET_DEBUG'] = "2" + # import os ; os.environ['EXECNET_DEBUG'] = "2" self.gateway = execnet.makegateway() self.config = config = self.testdir.parseconfigure() putevent = self.use_callback and self.events.put or None @@ -48,14 +51,16 @@ class SlaveSetup: ev = EventCall(data) if name is None or ev.name == name: return ev - print("skipping %s" % (ev,)) + print("skipping %s" % (ev, )) def sendcommand(self, name, **kwargs): self.slp.sendcommand(name, **kwargs) + def pytest_funcarg__slave(request): return SlaveSetup(request) + def test_remoteinitconfig(testdir): from xdist.remote import remote_initconfig config1 = testdir.parseconfig() @@ -63,6 +68,7 @@ def test_remoteinitconfig(testdir): assert config2.option.__dict__ == config1.option.__dict__ assert config2.pluginmanager.getplugin("terminal") in (-1, None) + class TestReportSerialization: def test_itemreport_outcomes(self, testdir): reprec = testdir.inline_runsource(""" @@ -79,7 +85,7 @@ class TestReportSerialization: py.test.xfail("hello") """) reports = reprec.getreports("pytest_runtest_logreport") - assert len(reports) == 17 # with setup/teardown "passed" reports + assert len(reports) == 17 # with setup/teardown "passed" reports for rep in reports: d = serialize_report(rep) check_marshallable(d) @@ -158,7 +164,7 @@ class TestSlaveInteractor: ev = slave.popevent("logstart") assert ev.kwargs["nodeid"].endswith("test_func") assert len(ev.kwargs["location"]) == 3 - ev = slave.popevent("testreport") # setup + ev = slave.popevent("testreport") # setup ev = slave.popevent("testreport") assert ev.name == "testreport" rep = unserialize_report(ev.name, ev.kwargs['data']) @@ -245,4 +251,3 @@ class TestSlaveInteractor: ("pytest_pycollect_makeitem", "name == 'test_func'"), ("pytest_collectreport", "report.collector.fspath == bbb"), ]) - diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index d39a470..2861329 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -7,6 +7,7 @@ from xdist.slavemanage import HostRSync, NodeManager pytest_plugins = "pytester" + def pytest_funcarg__hookrecorder(request, config): hookrecorder = HookRecorder(config.pluginmanager) if hasattr(hookrecorder, "start_recording"): @@ -14,23 +15,32 @@ def pytest_funcarg__hookrecorder(request, config): request.addfinalizer(hookrecorder.finish_recording) return hookrecorder + def pytest_funcarg__config(testdir): return testdir.parseconfig() + def pytest_funcarg__mysetup(tmpdir): class mysetup: source = tmpdir.mkdir("source") dest = tmpdir.mkdir("dest") + return mysetup() + @pytest.fixture def slavecontroller(monkeypatch): class MockController(object): - def __init__(self, *args): pass - def setup(self): pass + def __init__(self, *args): + pass + + def setup(self): + pass + monkeypatch.setattr(slavemanage, 'SlaveController', MockController) return MockController + class TestNodeManagerPopen: def test_popen_no_default_chdir(self, config): gm = NodeManager(config, ["popen"]) @@ -43,7 +53,8 @@ class TestNodeManagerPopen: for spec in NodeManager(config, l, defaultchdir="abc").specs: assert spec.chdir == "abc" - def test_popen_makegateway_events(self, config, hookrecorder, slavecontroller): + def test_popen_makegateway_events(self, config, hookrecorder, + slavecontroller): hm = NodeManager(config, ["popen"] * 2) hm.setup_nodes(None) call = hookrecorder.popcall("pytest_xdist_setupnodes") @@ -64,12 +75,16 @@ class TestNodeManagerPopen: hm.setup_nodes(None) assert len(hm.group) == 2 for gw in hm.group: + class pseudoexec: args = [] + def __init__(self, *args): self.args.extend(args) + def waitclose(self): pass + gw.remote_exec = pseudoexec l = [] for gw in hm.group: @@ -95,8 +110,8 @@ class TestNodeManagerPopen: assert dest.join("dir1", "dir2").check() assert dest.join("dir1", "dir2", 'hello').check() - def test_rsync_same_popen_twice(self, config, mysetup, - hookrecorder, slavecontroller): + def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder, + slavecontroller): source, dest = mysetup.source, mysetup.dest hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2) hm.roots = [] @@ -110,6 +125,7 @@ class TestNodeManagerPopen: assert call.gateways[0] in hm.group call = hookrecorder.popcall("pytest_xdist_rsyncfinish") + class TestHRSync: def test_hrsync_filter(self, mysetup): source, _ = mysetup.source, mysetup.dest # noqa @@ -118,8 +134,7 @@ class TestHRSync: source.ensure(".somedotfile", "moreentries") source.ensure("somedir", "editfile~") syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES) - l = list(source.visit(rec=syncer.filter, - fil=syncer.filter)) + l = list(source.visit(rec=syncer.filter, fil=syncer.filter)) assert len(l) == 3 basenames = [x.basename for x in l] assert 'dir' in basenames @@ -145,7 +160,7 @@ class TestNodeManager: mysetup.source.ensure("dir1", "file1").write("hello") config = testdir.parseconfig(mysetup.source) nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest]) - #assert nodemanager.config.topdir == source == config.topdir + # assert nodemanager.config.topdir == source == config.topdir nodemanager.makegateways() nodemanager.rsync_roots() p, = nodemanager.gwmanager.multi_exec( @@ -164,10 +179,8 @@ class TestNodeManager: for rsyncroot in (dir1, source): dest.remove() nodemanager = NodeManager(testdir.parseconfig( - "--tx", "popen//chdir=%s" % dest, - "--rsyncdir", rsyncroot, - source, - )) + "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, + source, )) nodemanager.setup_nodes(None) # calls .rsync_roots() if rsyncroot == source: dest = dest.join("source") @@ -230,7 +243,8 @@ class TestNodeManager: assert not gwspec.chdir def test_ssh_setup_nodes(self, specssh, testdir): - testdir.makepyfile(__init__="", test_x=""" + testdir.makepyfile(__init__="", + test_x=""" def test_one(): pass """) diff --git a/tox.ini b/tox.ini index b99d2ab..de603e2 100644 --- a/tox.ini +++ b/tox.ini @@ -21,8 +21,8 @@ commands= [testenv:flakes] changedir= -deps = pytest-flakes>=0.2 -commands = py.test --flakes -m flakes testing xdist +deps = flake8 +commands = flake8 setup.py testing xdist [testenv:readme] changedir = diff --git a/xdist/boxed.py b/xdist/boxed.py index 6bd920a..009a112 100644 --- a/xdist/boxed.py +++ b/xdist/boxed.py @@ -4,9 +4,10 @@ import py def pytest_addoption(parser): group = parser.getgroup("xdist", "distributed and subprocess testing") - group.addoption('--boxed', - action="store_true", dest="boxed", default=False, - help="box each test run in a separate process (unix)") + group.addoption( + '--boxed', + action="store_true", dest="boxed", default=False, + help="box each test run in a separate process (unix)") def pytest_runtest_protocol(item): @@ -16,6 +17,7 @@ def pytest_runtest_protocol(item): item.ihook.pytest_runtest_logreport(report=rep) return True + def forked_run_report(item): # for now, we run setup/teardown in the subprocess # XXX optionally allow sharing of setup/teardown @@ -24,6 +26,7 @@ def forked_run_report(item): import marshal from xdist.remote import serialize_report from xdist.slavemanage import unserialize_report + def runforked(): try: reports = runtestprotocol(item, log=False) @@ -38,9 +41,10 @@ def forked_run_report(item): return [unserialize_report("testreport", x) for x in report_dumps] else: if result.exitstatus == EXITSTATUS_TESTEXIT: - py.test.exit("forked test item %s raised Exit" %(item,)) + py.test.exit("forked test item %s raised Exit" % (item,)) return [report_process_crash(item, result)] + def report_process_crash(item, result): path, lineno = item._getfslineno() info = ("%s:%s: running the test CRASHED with signal %d" % diff --git a/xdist/dsession.py b/xdist/dsession.py index e669db6..a0438de 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -257,8 +257,9 @@ class LoadScheduling: assert node in self.node2pending if self.collection_is_completed: # A new node has been added later, perhaps an original one died. - assert self.collection # .init_distribute() should have - # been called by now + # .init_distribute() should have + # been called by now + assert self.collection if collection != self.collection: other_node = next(iter(self.node2collection.keys())) msg = report_collection_diff(self.collection, @@ -398,8 +399,9 @@ class LoadScheduling: same_collection = False self.log(msg) if self.config is not None: - rep = CollectReport(node.gateway.id, 'failed', longrepr=msg, - result=[]) + rep = CollectReport( + node.gateway.id, 'failed', + longrepr=msg, result=[]) self.config.hook.pytest_collectreport(report=rep) return same_collection @@ -641,7 +643,7 @@ class DSession: """ if rep.when == "call" or (rep.when == "setup" and not rep.passed): self.sched.remove_item(node, rep.item_index, rep.duration) - #self.report_line("testreport %s: %s" %(rep.id, rep.status)) + # self.report_line("testreport %s: %s" %(rep.id, rep.status)) rep.node = node self.config.hook.pytest_runtest_logreport(report=rep) self._handlefailures(rep) @@ -719,8 +721,8 @@ class TerminalDistReporter: self.rewrite(self.getstatus()) def getstatus(self): - parts = ["%s %s" %(spec.id, self._status[spec.id]) - for spec in self._specs] + parts = ["%s %s" % (spec.id, self._status[spec.id]) + for spec in self._specs] return " / ".join(parts) def rewrite(self, line, newline=False): @@ -751,7 +753,7 @@ class TerminalDistReporter: def pytest_testnodeready(self, node): if self.config.option.verbose > 0: d = node.slaveinfo - infoline = "[%s] Python %s" %( + infoline = "[%s] Python %s" % ( d['id'], d['version'].replace('\n', ' -- '),) self.rewrite(infoline, newline=True) @@ -760,13 +762,12 @@ class TerminalDistReporter: def pytest_testnodedown(self, node, error): if not error: return - self.write_line("[%s] node down: %s" %(node.gateway.id, error)) + self.write_line("[%s] node down: %s" % (node.gateway.id, error)) - #def pytest_xdist_rsyncstart(self, source, gateways): + # def pytest_xdist_rsyncstart(self, source, gateways): # targets = ",".join([gw.id for gw in gateways]) # msg = "[%s] rsyncing: %s" %(targets, source) # self.write_line(msg) - #def pytest_xdist_rsyncfinish(self, source, gateways): + # def pytest_xdist_rsyncfinish(self, source, gateways): # targets = ", ".join(["[%s]" % gw.id for gw in gateways]) # self.write_line("rsyncfinish: %s -> %s" %(source, targets)) - diff --git a/xdist/looponfail.py b/xdist/looponfail.py index 2c12de8..99604bb 100644 --- a/xdist/looponfail.py +++ b/xdist/looponfail.py @@ -7,23 +7,26 @@ the controlling process which should best never happen. """ -import py, pytest +import py +import pytest import sys import execnet + def pytest_addoption(parser): group = parser.getgroup("xdist", "distributed and subprocess testing") - group._addoption('-f', '--looponfail', - action="store_true", dest="looponfail", default=False, - help="run tests in subprocess, wait for modified files " - "and re-run failing test set until all pass.") + group._addoption( + '-f', '--looponfail', + action="store_true", dest="looponfail", default=False, + help="run tests in subprocess, wait for modified files " + "and re-run failing test set until all pass.") + def pytest_cmdline_main(config): - + if config.getoption("looponfail"): looponfail_main(config) - return 2 # looponfail only can get stop with ctrl-C anyway - + return 2 # looponfail only can get stop with ctrl-C anyway def looponfail_main(config): @@ -34,7 +37,8 @@ def looponfail_main(config): while 1: remotecontrol.loop_once() if not remotecontrol.failures and remotecontrol.wasfailing: - continue # the last failures passed, let's immediately rerun all + # the last failures passed, let's immediately rerun all + continue repr_pytest_looponfailinfo( failreports=remotecontrol.failures, rootdirs=rootdirs) @@ -42,6 +46,7 @@ def looponfail_main(config): except KeyboardInterrupt: print() + class RemoteControl(object): def __init__(self, config): self.config = config @@ -62,11 +67,13 @@ class RemoteControl(object): raise ValueError("already have gateway %r" % self.gateway) self.trace("setting up slave session") self.gateway = self.initgateway() - self.channel = channel = self.gateway.remote_exec(init_slave_session, + self.channel = channel = self.gateway.remote_exec( + init_slave_session, args=self.config.args, option_dict=vars(self.config.option), ) remote_outchannel = channel.receive() + def write(s): out._file.write(s) out._file.flush() @@ -102,7 +109,7 @@ class RemoteControl(object): result = self.runsession() failures, reports, collection_failed = result if collection_failed: - pass # "Collection failed, keeping previous failure set" + pass # "Collection failed, keeping previous failure set" else: uniq_failures = [] for failure in failures: @@ -110,6 +117,7 @@ class RemoteControl(object): uniq_failures.append(failure) self.failures = uniq_failures + def repr_pytest_looponfailinfo(failreports, rootdirs): tr = py.io.TerminalWriter() if failreports: @@ -119,11 +127,12 @@ def repr_pytest_looponfailinfo(failreports, rootdirs): tr.line(report, red=True) tr.sep("#", "waiting for changes", bold=True) for rootdir in rootdirs: - tr.line("### Watching: %s" %(rootdir,), bold=True) + tr.line("### Watching: %s" % (rootdir,), bold=True) def init_slave_session(channel, args, option_dict): - import os, sys + import os + import sys outchannel = channel.gateway.newchannel() sys.stdout = sys.stderr = outchannel.makefile('w') channel.send(outchannel) @@ -136,13 +145,14 @@ def init_slave_session(channel, args, option_dict): newpaths.append(p) sys.path[:] = newpaths - #fullwidth, hasmarkup = channel.receive() + # fullwidth, hasmarkup = channel.receive() from _pytest.config import Config config = Config.fromdictargs(option_dict, list(args)) config.args = args from xdist.looponfail import SlaveFailSession SlaveFailSession(config, channel).main() + class SlaveFailSession: def __init__(self, config, channel): self.config = config @@ -165,7 +175,8 @@ class SlaveFailSession: items = session.perform_collect(self.trails or None) except pytest.UsageError: items = session.perform_collect(None) - hook.pytest_collection_modifyitems(session=session, config=session.config, items=items) + hook.pytest_collection_modifyitems( + session=session, config=session.config, items=items) hook.pytest_collection_finish(session=session) return True @@ -183,7 +194,7 @@ class SlaveFailSession: try: command = self.channel.receive() except KeyboardInterrupt: - return # in the slave we can't do much about this + return # in the slave we can't do much about this self.DEBUG("received", command) self.current_command = command self.config.hook.pytest_cmdline_main(config=self.config) @@ -195,14 +206,16 @@ class SlaveFailSession: failreports.append(loc) self.channel.send((trails, failreports, self.collection_failed)) + class StatRecorder: def __init__(self, rootdirlist): self.rootdirlist = rootdirlist self.statcache = {} - self.check() # snapshot state + self.check() # snapshot state def fil(self, p): return p.check(file=1, dotfile=0) and p.ext != ".pyc" + def rec(self, p): return p.check(dotfile=0) @@ -213,7 +226,7 @@ class StatRecorder: return py.std.time.sleep(checkinterval) - def check(self, removepycfiles=True): + def check(self, removepycfiles=True): # noqa, too complex changed = False statcache = self.statcache newstat = {} @@ -227,8 +240,8 @@ class StatRecorder: changed = True else: if oldstat: - if oldstat.mtime != curstat.mtime or \ - oldstat.size != curstat.size: + if oldstat.mtime != curstat.mtime or \ + oldstat.size != curstat.size: changed = True py.builtin.print_("# MODIFIED", path) if removepycfiles and path.ext == ".py": diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 2034617..0207935 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -2,20 +2,26 @@ def pytest_xdist_setupnodes(config, specs): """ called before any remote node is set up. """ + def pytest_xdist_newgateway(gateway): """ called on new raw gateway creation. """ + def pytest_xdist_rsyncstart(source, gateways): """ called before rsyncing a directory to remote gateways takes place. """ + def pytest_xdist_rsyncfinish(source, gateways): """ called after rsyncing a directory to remote gateways takes place. """ + def pytest_configure_node(node): """ configure node information before it gets instantiated. """ + def pytest_testnodeready(node): """ Test Node is ready to operate. """ + def pytest_testnodedown(node, error): """ Test Node is down. """ diff --git a/xdist/plugin.py b/xdist/plugin.py index 5057109..dc3635e 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -12,45 +12,56 @@ def parse_numprocesses(s): def pytest_addoption(parser): group = parser.getgroup("xdist", "distributed and subprocess testing") - group._addoption('-n', dest="numprocesses", metavar="numprocesses", - action="store", - type=parse_numprocesses, - help="shortcut for '--dist=load --tx=NUM*popen', " - "you can use 'auto' here for auto detection CPUs number on " - "host system") + group._addoption( + '-n', dest="numprocesses", metavar="numprocesses", + action="store", + type=parse_numprocesses, + help="shortcut for '--dist=load --tx=NUM*popen', " + "you can use 'auto' here for auto detection CPUs number on " + "host system") group._addoption('--max-slave-restart', action="store", default=None, help="maximum number of slaves that can be restarted " "when crashed (set to zero to disable this feature)") - group._addoption('--dist', metavar="distmode", - action="store", choices=['load', 'each', 'no'], - type="choice", dest="dist", default="no", - help=("set mode for distributing tests to exec environments.\n\n" - "each: send each test to each available environment.\n\n" - "load: send each test to available environment.\n\n" - "(default) no: run tests inprocess, don't distribute.")) - group._addoption('--tx', dest="tx", action="append", default=[], - metavar="xspec", - help=("add a test execution environment. some examples: " - "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 " - "--tx ssh=user@codespeak.net//chdir=testcache")) - group._addoption('-d', - action="store_true", dest="distload", default=False, - help="load-balance tests. shortcut for '--dist=load'") - group.addoption('--rsyncdir', action="append", default=[], metavar="DIR", - help="add directory for rsyncing to remote tx nodes.") - group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB", - help="add expression for ignores when rsyncing to remote tx nodes.") + group._addoption( + '--dist', metavar="distmode", + action="store", choices=['load', 'each', 'no'], + type="choice", dest="dist", default="no", + help=("set mode for distributing tests to exec environments.\n\n" + "each: send each test to each available environment.\n\n" + "load: send each test to available environment.\n\n" + "(default) no: run tests inprocess, don't distribute.")) + group._addoption( + '--tx', dest="tx", action="append", default=[], + metavar="xspec", + help=("add a test execution environment. some examples: " + "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 " + "--tx ssh=user@codespeak.net//chdir=testcache")) + group._addoption( + '-d', + action="store_true", dest="distload", default=False, + help="load-balance tests. shortcut for '--dist=load'") + group.addoption( + '--rsyncdir', action="append", default=[], metavar="DIR", + help="add directory for rsyncing to remote tx nodes.") + group.addoption( + '--rsyncignore', action="append", default=[], metavar="GLOB", + help="add expression for ignores when rsyncing to remote tx nodes.") - parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' - ' remote distributed testing.', type="pathlist") - parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored ' - 'for rsyncing.', type="pathlist") - parser.addini("looponfailroots", type="pathlist", + parser.addini( + 'rsyncdirs', 'list of (relative) paths to be rsynced for' + ' remote distributed testing.', type="pathlist") + parser.addini( + 'rsyncignore', 'list of (relative) glob-style paths to be ignored ' + 'for rsyncing.', type="pathlist") + parser.addini( + "looponfailroots", type="pathlist", help="directories to check for changes", default=[py.path.local()]) # ------------------------------------------------------------------------- # distributed testing hooks # ------------------------------------------------------------------------- + + def pytest_addhooks(pluginmanager): from xdist import newhooks # avoid warnings with pytest-2.8 @@ -73,6 +84,7 @@ def pytest_configure(config): tr = config.pluginmanager.getplugin("terminalreporter") tr.showfspath = False + @pytest.mark.tryfirst def pytest_cmdline_main(config): if config.option.numprocesses: @@ -85,7 +97,9 @@ def pytest_cmdline_main(config): usepdb = config.option.usepdb # a core option if val("looponfail"): if usepdb: - raise pytest.UsageError("--pdb incompatible with --looponfail.") + raise pytest.UsageError( + "--pdb incompatible with --looponfail.") elif val("dist") != "no": if usepdb: - raise pytest.UsageError("--pdb incompatible with distributing tests.") + raise pytest.UsageError( + "--pdb incompatible with distributing tests.") diff --git a/xdist/remote.py b/xdist/remote.py index a0b2cad..0d6997f 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -6,7 +6,9 @@ needs not to be installed in remote environments. """ -import sys, os +import sys +import os + class SlaveInteractor: def __init__(self, config, channel): @@ -72,7 +74,8 @@ class SlaveInteractor: nextitem=nextitem) def pytest_collection_finish(self, session): - self.sendevent("collectionfinish", + self.sendevent( + "collectionfinish", topdir=str(session.fspath), ids=[item.nodeid for item in session.items]) @@ -89,6 +92,7 @@ class SlaveInteractor: data = serialize_report(report) self.sendevent("collectreport", data=data) + def serialize_report(rep): import py d = rep.__dict__.copy() @@ -100,20 +104,22 @@ def serialize_report(rep): if isinstance(d[name], py.path.local): d[name] = str(d[name]) elif name == "result": - d[name] = None # for now + d[name] = None # for now return d + def getinfodict(): import platform return dict( - version = sys.version, - version_info = tuple(sys.version_info), - sysplatform = sys.platform, - platform = platform.platform(), - executable = sys.executable, - cwd = os.getcwd(), + version=sys.version, + version_info=tuple(sys.version_info), + sysplatform=sys.platform, + platform=platform.platform(), + executable=sys.executable, + cwd=os.getcwd(), ) + def remote_initconfig(option_dict, args): from _pytest.config import Config option_dict['plugins'].append("no:terminal") @@ -131,14 +137,15 @@ if __name__ == '__channelexec__': channel = channel # noqa # python3.2 is not concurrent import safe, so let's play it safe # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32 - if sys.version_info[:2] == (3,2): + if sys.version_info[:2] == (3, 2): os.environ["PYTHONDONTWRITEBYTECODE"] = "1" - slaveinput,args,option_dict = channel.receive() + slaveinput, args, option_dict = channel.receive() importpath = os.getcwd() - sys.path.insert(0, importpath) # XXX only for remote situations - os.environ['PYTHONPATH'] = (importpath + os.pathsep + + sys.path.insert(0, importpath) # XXX only for remote situations + os.environ['PYTHONPATH'] = ( + importpath + os.pathsep + os.environ.get('PYTHONPATH', '')) - #os.environ['PYTHONPATH'] = importpath + # os.environ['PYTHONPATH'] = importpath import py config = remote_initconfig(option_dict, args) config.slaveinput = slaveinput diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 4f1649d..2084e59 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -1,16 +1,19 @@ import fnmatch import os +import re import py import pytest import execnet import xdist.remote -from _pytest import runner # XXX load dynamically +from _pytest import runner # XXX load dynamically + class NodeManager(object): EXIT_TIMEOUT = 10 DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~'] + def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() @@ -79,11 +82,12 @@ class NodeManager(object): break else: return [] - import pytest, _pytest + import pytest + import _pytest pytestpath = pytest.__file__.rstrip("co") pytestdir = py.path.local(_pytest.__file__).dirpath() config = self.config - candidates = [py._pydir,pytestpath,pytestdir] + candidates = [py._pydir, pytestpath, pytestdir] candidates += config.option.rsyncdir rsyncroots = config.getini("rsyncdirs") if rsyncroots: @@ -92,7 +96,7 @@ class NodeManager(object): for root in candidates: root = py.path.local(root).realpath() if not root.check(): - raise pytest.UsageError("rsyncdir doesn't exist: %r" %(root,)) + raise pytest.UsageError("rsyncdir doesn't exist: %r" % (root,)) if root not in roots: roots.append(root) return roots @@ -124,6 +128,7 @@ class NodeManager(object): return if (spec, source) in self._rsynced_specs: return + def finished(): if notify: notify("rsyncrootready", spec, source) @@ -139,19 +144,23 @@ class NodeManager(object): gateways=[gateway], ) + class HostRSync(execnet.RSync): """ RSyncer that filters out common files """ def __init__(self, sourcedir, *args, **kwargs): self._synced = {} - self._ignores = kwargs.pop('ignores', None) or [] + self._ignores = [] + ignores = kwargs.pop('ignores', None) or [] + for x in ignores: + x = getattr(x, 'strpath', x) + self.ignores.append(re.compile(fnmatch.translate(x))) super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs) def filter(self, path): path = py.path.local(path) - for x in self._ignores: - x = getattr(x, 'strpath', x) - if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x): + for check in self._ignores: + if check(path.basename) or check(path.strpath): return False else: return True @@ -187,6 +196,7 @@ def make_reltoroot(roots, args): l.append(splitcode.join(parts)) return l + class SlaveController(object): ENDMARK = -1 @@ -202,7 +212,7 @@ class SlaveController(object): py.log.setconsumer(self.log._keywords, None) def __repr__(self): - return "<%s %s>" %(self.__class__.__name__, self.gateway.id,) + return "<%s %s>" % (self.__class__.__name__, self.gateway.id,) def setup(self): self.log("setting up slave session") @@ -219,7 +229,8 @@ class SlaveController(object): self.channel = self.gateway.remote_exec(xdist.remote) self.channel.send((self.slaveinput, args, option_dict)) if self.putevent: - self.channel.setcallback(self.process_from_remote, + self.channel.setcallback( + self.process_from_remote, endmarker=self.ENDMARK) def ensure_teardown(self): @@ -227,11 +238,11 @@ class SlaveController(object): if not self.channel.isclosed(): self.log("closing", self.channel) self.channel.close() - #del self.channel + # del self.channel if hasattr(self, 'gateway'): self.log("exiting", self.gateway) self.gateway.exit() - #del self.gateway + # del self.gateway def send_runtest_some(self, indices): self.sendcommand("runtests", indices=indices) @@ -255,7 +266,7 @@ class SlaveController(object): self.log("queuing %s(**%s)" % (eventname, kwargs)) self.putevent((eventname, kwargs)) - def process_from_remote(self, eventcall): + def process_from_remote(self, eventcall): # noqa too complex """ this gets called for each object we receive from the other side and if the channel closes. @@ -268,13 +279,13 @@ class SlaveController(object): err = self.channel._getremoteerror() if not self._down: if not err or isinstance(err, EOFError): - err = "Not properly terminated" # lost connection? + err = "Not properly terminated" # lost connection? self.notify_inproc("errordown", node=self, error=err) self._down = True return eventname, kwargs = eventcall if eventname in ("collectionstart"): - self.log("ignoring %s(%s)" %(eventname, kwargs)) + self.log("ignoring %s(%s)" % (eventname, kwargs)) elif eventname == "slaveready": self.notify_inproc(eventname, node=self, **kwargs) elif eventname == "slavefinished": @@ -283,7 +294,8 @@ class SlaveController(object): self.notify_inproc("slavefinished", node=self) elif eventname == "logstart": self.notify_inproc(eventname, node=self, **kwargs) - elif eventname in ("testreport", "collectreport", "teardownreport"): + elif eventname in ( + "testreport", "collectreport", "teardownreport"): item_index = kwargs.pop("item_index", None) rep = unserialize_report(eventname, kwargs['data']) if item_index is not None: @@ -292,7 +304,7 @@ class SlaveController(object): elif eventname == "collectionfinish": self.notify_inproc(eventname, node=self, ids=kwargs['ids']) else: - raise ValueError("unknown event: %s" %(eventname,)) + raise ValueError("unknown event: %s" % (eventname,)) except KeyboardInterrupt: # should not land in receiver-thread raise @@ -301,6 +313,7 @@ class SlaveController(object): py.builtin.print_("!" * 20, excinfo) self.config.pluginmanager.notify_exception(excinfo) + def unserialize_report(name, reportdict): if name == "testreport": return runner.TestReport(**reportdict)