From 4d0422a5480db7dc6dbf9350fed9061bc75011c6 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Tue, 7 Sep 2010 10:46:58 +0200 Subject: [PATCH] remove trailing whitespace from sources --- CHANGELOG | 53 +++++++++--------- ISSUES.txt | 8 +-- README.txt | 12 ++--- setup.py | 8 +-- testing/acceptance_test.py | 24 ++++----- testing/conftest.py | 8 +-- testing/test_deprecated.py | 12 ++--- testing/test_dsession.py | 108 ++++++++++++++++++------------------- testing/test_gwmanage.py | 20 +++---- testing/test_mypickle.py | 38 ++++++------- testing/test_nodemanage.py | 14 ++--- testing/test_pickling.py | 6 +-- testing/test_plugin.py | 12 ++--- testing/test_remote.py | 14 ++--- testing/test_txnode.py | 26 ++++----- testing/test_util.py | 6 +-- tox.ini | 2 +- xdist/__init__.py | 2 +- xdist/dsession.py | 62 ++++++++++----------- xdist/gwmanage.py | 20 +++---- xdist/mypickle.py | 48 ++++++++--------- xdist/newhooks.py | 2 +- xdist/nodemanage.py | 28 +++++----- xdist/plugin.py | 96 ++++++++++++++++----------------- xdist/remote.py | 40 +++++++------- xdist/txnode.py | 52 +++++++++--------- xdist/util.py | 4 +- 27 files changed, 365 insertions(+), 360 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 0f76881..04c92db 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,54 +1,59 @@ +1.5a1 +------------------------- + +- remove all trailing whitespace from source + 1.4 ------------------------- -- perform distributed testing related reporting in the plugin - rather than having dist-related code in the generic py.test +- perform distributed testing related reporting in the plugin + rather than having dist-related code in the generic py.test distribution -- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to - gateway specification strings. +- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to + gateway specification strings. -- show detailed gateway setup and platform information only when - "-v" or "--verbose" is specified. +- show detailed gateway setup and platform information only when + "-v" or "--verbose" is specified. 1.3 ------------------------- - fix --looponfailing - it would not actually run against the fully changed - source tree when initial conftest files load application state. + source tree when initial conftest files load application state. -- adapt for py-1.3.1's new --maxfailure option +- adapt for py-1.3.1's new --maxfailure option 1.2 ------------------------- -- fix issue79: sessionfinish/teardown hooks are now called systematically - on the slave side -- introduce a new data input/output mechanism to allow the master side - to send and receive data from a slave. -- fix race condition in underlying pickling/unpickling handling -- use and require new register hooks facility of py.test>=1.3.0 -- require improved execnet>=1.0.6 because of various race conditions - that can arise in xdist testing modes. -- fix some python3 related pickling related race conditions -- fix PyPI description +- fix issue79: sessionfinish/teardown hooks are now called systematically + on the slave side +- introduce a new data input/output mechanism to allow the master side + to send and receive data from a slave. +- fix race condition in underlying pickling/unpickling handling +- use and require new register hooks facility of py.test>=1.3.0 +- require improved execnet>=1.0.6 because of various race conditions + that can arise in xdist testing modes. +- fix some python3 related pickling related race conditions +- fix PyPI description 1.1 ------------------------- - fix an indefinite hang which would wait for events although no events - are pending - this happened if items arrive very quickly while + are pending - this happened if items arrive very quickly while the "reschedule-event" tried unconditionally avoiding a busy-loop - and not schedule new work. + and not schedule new work. 1.0 ------------------------- - moved code out of py-1.1.1 into its own plugin -- use a new, faster and more sensible model to do load-balancing +- use a new, faster and more sensible model to do load-balancing of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing - works effectively even with very few tests. -- cleaned up termination handling -- make -x cause hard killing of test nodes to decrease wait time + works effectively even with very few tests. +- cleaned up termination handling +- make -x cause hard killing of test nodes to decrease wait time until the traceback shows up on first failure diff --git a/ISSUES.txt b/ISSUES.txt index fafde57..a658427 100644 --- a/ISSUES.txt +++ b/ISSUES.txt @@ -1,10 +1,10 @@ -allow to run xdist tests with xdist +allow to run xdist tests with xdist ----------------------------------------------- -tag: feature +tag: feature allow to run xdist own tests using its own mechanism. currently this doesn't work because the remote side -has no py.test plugin. How to configure/do -register "xdist.plugin" on the remote side? +has no py.test plugin. How to configure/do +register "xdist.plugin" on the remote side? diff --git a/README.txt b/README.txt index ce577f4..5c26090 100644 --- a/README.txt +++ b/README.txt @@ -4,14 +4,14 @@ py.test xdist plugin: distributed testing and loop failure .. _`pytest-xdist respository`: http://bitbucket.org/hpk42/pytest-xdist .. _`pytest`: http://pytest.org -The pytest-xdist plugin extends `py.test`_ to ad-hoc distribute test +The pytest-xdist plugin extends `py.test`_ to ad-hoc distribute test runs to multiple CPUs or remote machines. It requires setuptools -or distribute which help to pull in the neccessary execnet and -pytest-core dependencies. +or distribute which help to pull in the neccessary execnet and +pytest-core dependencies. Install the plugin locally with:: - python setup.py install + python setup.py install or use the package in develope/in-place mode, particularly useful with a checkout of the `pytest-xdist repository`_:: @@ -20,9 +20,9 @@ useful with a checkout of the `pytest-xdist repository`_:: or use one of:: - easy_install pytest-xdist + easy_install pytest-xdist - pip install pytest-xdist + pip install pytest-xdist for downloading and installing it in one go. diff --git a/setup.py b/setup.py index caba1df..6543f84 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ """ -py.test 'xdist' plugin for distributed testing and loop-on-failing modes. +py.test 'xdist' plugin for distributed testing and loop-on-failing modes. See http://pytest.org/plugin/xdist.html for documentation and, after -installation of the ``pytest-xdist`` PyPI package, ``py.test -h`` -for the new options. +installation of the ``pytest-xdist`` PyPI package, ``py.test -h`` +for the new options. """ from setuptools import setup @@ -16,7 +16,7 @@ setup( long_description=__doc__, license='GPLv2 or later', author='holger krekel and contributors', - author_email='py-dev@codespeak.net,holger@merlinux.eu', + author_email='py-dev@codespeak.net,holger@merlinux.eu', url='http://bitbucket.org/hpk42/pytest-xdist', platforms=['linux', 'osx', 'win32'], packages = ['xdist'], diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index be96c9a..9110d59 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -13,7 +13,7 @@ class TestDistribution: pass def test_skip(): py.test.skip("hello") - """, + """, ) result = testdir.runpytest(p1, "-v", '-d', '--tx=popen', '--tx=popen') result.stdout.fnmatch_lines([ @@ -34,7 +34,7 @@ class TestDistribution: pass def test_skip(): py.test.skip("hello") - """, + """, ) testdir.makeconftest(""" option_tx = 'popen popen popen'.split() @@ -52,7 +52,7 @@ class TestDistribution: def test_dist_tests_with_crash(self, testdir): if not hasattr(py.std.os, 'kill'): py.test.skip("no os.kill") - + p1 = testdir.makepyfile(""" import py def test_fail0(): @@ -87,7 +87,7 @@ class TestDistribution: subdir.ensure("__init__.py") p = subdir.join("test_one.py") p.write("def test_5(): assert not __file__.startswith(%r)" % str(p)) - result = testdir.runpytest("-v", "-d", "--rsyncdir=%(subdir)s" % locals(), + result = testdir.runpytest("-v", "-d", "--rsyncdir=%(subdir)s" % locals(), "--tx=popen//chdir=%(dest)s" % locals(), p) assert result.ret == 0 result.stdout.fnmatch_lines([ @@ -143,7 +143,7 @@ class TestDistribution: def pytest_terminal_summary(terminalreporter): if not hasattr(terminalreporter.config, 'slaveinput'): calc_result = terminalreporter.config.calc_result - terminalreporter._tw.sep('-', + terminalreporter._tw.sep('-', 'calculated result is %s' % calc_result) """) p1 = testdir.makepyfile("def test_func(): pass") @@ -171,12 +171,12 @@ class TestDistribution: args = ["-n1"] result = testdir.runpytest(*args) s = result.stdout.str() - assert result.ret + assert result.ret assert 'SIGINT' in s assert 's2call' in s def test_keyboard_interrupt_dist(self, testdir): - # xxx could be refined to check for return code + # xxx could be refined to check for return code p = testdir.makepyfile(""" def test_sleep(): import time @@ -188,7 +188,7 @@ class TestDistribution: child.expect(".*KeyboardInterrupt.*") #child.expect(".*seconds.*") child.close() - #assert ret == 2 + #assert ret == 2 class TestTerminalReporting: def test_pass_skip_fail(self, testdir): @@ -203,9 +203,9 @@ class TestTerminalReporting: """) result = testdir.runpytest("-n1", "-v") expected = [ - "*PASS*test_pass_skip_fail.py:2: *test_ok*", - "*SKIP*test_pass_skip_fail.py:4: *test_skip*", - "*FAIL*test_pass_skip_fail.py:6: *test_func*", + "*PASS*test_pass_skip_fail.py:2: *test_ok*", + "*SKIP*test_pass_skip_fail.py:4: *test_skip*", + "*FAIL*test_pass_skip_fail.py:6: *test_func*", ] for line in expected: result.stdout.fnmatch_lines([line]) @@ -222,7 +222,7 @@ class TestTerminalReporting: """) result = testdir.runpytest("-n1", "-v") result.stdout.fnmatch_lines([ - "*FAIL*test_fail_platinfo.py:1: *test_func*", + "*FAIL*test_fail_platinfo.py:1: *test_func*", "*popen*Python*", " def test_func():", "> assert 0", diff --git a/testing/conftest.py b/testing/conftest.py index ea23fc9..87faba4 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -2,11 +2,11 @@ import py import execnet pytest_plugins = "pytester" - + #rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()] def pytest_addoption(parser): - parser.addoption('--gx', + parser.addoption('--gx', action="append", dest="gspecs", default=None, help=("add a global test environment, XSpec-syntax. ")) @@ -16,9 +16,9 @@ def getgspecs(config): return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")] -# configuration information for tests +# configuration information for tests def getgspecs(config): - return [execnet.XSpec(spec) + return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")] def getspecssh(config): diff --git a/testing/test_deprecated.py b/testing/test_deprecated.py index c369780..a0d647b 100644 --- a/testing/test_deprecated.py +++ b/testing/test_deprecated.py @@ -8,9 +8,9 @@ def test_dist_conftest_options(testdir, recwarn): import py from py.builtin import print_ print_("importing conftest", __file__) - Option = py.test.config.Option - option = py.test.config.addoptions("someopt", - Option('--someopt', action="store_true", + Option = py.test.config.Option + option = py.test.config.addoptions("someopt", + Option('--someopt', action="store_true", dest="someopt", default=False)) dist_rsync_roots = ['../dir'] print_("added options", option) @@ -20,13 +20,13 @@ def test_dist_conftest_options(testdir, recwarn): import py from %s import conftest from py.builtin import print_ - def test_1(): + def test_1(): print_("config from test_1", py.test.config) print_("conftest from test_1", conftest.__file__) print_("test_1: py.test.config.option.someopt", py.test.config.option.someopt) print_("test_1: conftest", conftest) print_("test_1: conftest.option.someopt", conftest.option.someopt) - assert conftest.option.someopt + assert conftest.option.someopt """ % p1.dirpath().purebasename )) result = testdir.runpytest('-d', '--tx=popen', p1, '--someopt') assert result.ret == 0 @@ -34,5 +34,5 @@ def test_dist_conftest_options(testdir, recwarn): "*Deprecation*pytest_addoptions*", ]) result.stdout.fnmatch_lines([ - "*1 passed*", + "*1 passed*", ]) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 5c0c7cb..37caca2 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -7,10 +7,10 @@ XSpec = execnet.XSpec def run(item, node, excinfo=None): runner = item.config.pluginmanager.getplugin("runner") - rep = runner.ItemTestReport(item=item, + rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call") rep.node = node - return rep + return rep class MockNode: def __init__(self): @@ -43,7 +43,7 @@ class TestDSession: assert pending == [item] assert item not in session.item2nodes l = session.removenode(node) - assert not l + assert not l def test_senditems_each_and_receive_with_two_nodes(self, testdir): item = testdir.getitem("def test_func(): pass") @@ -60,7 +60,7 @@ class TestDSession: session.removeitem(item, node1) assert session.item2nodes[item] == [node2] session.removeitem(item, node2) - assert not session.node2pending[node1] + assert not session.node2pending[node1] assert not session.item2nodes def test_senditems_load_and_receive_one_node(self, testdir): @@ -69,11 +69,11 @@ class TestDSession: rep = run(item, node) session = DSession(item.config) session.addnode(node) - session.senditems_load([item]) + session.senditems_load([item]) assert session.node2pending[node] == [item] assert session.item2nodes[item] == [node] session.removeitem(item, node) - assert not session.node2pending[node] + assert not session.node2pending[node] assert not session.item2nodes def test_triggertesting_collect(self, testdir): @@ -110,7 +110,7 @@ class TestDSession: assert name == "pytest_rescheduleitems" assert kwargs['items'] == [item] - + def test_keyboardinterrupt(self, testdir): item = testdir.getitem("def test_func(): pass") session = DSession(item.config) @@ -142,8 +142,8 @@ class TestDSession: session.queueevent("pytest_rescheduleitems", items=[item]) session.loop_once(loopstate) # now we want to not directly trigger work again to avoid busy-wait - assert loopstate.dowork == False - + assert loopstate.dowork == False + session.queueevent(None) session.loop_once(loopstate) session.queueevent(None) @@ -153,8 +153,8 @@ class TestDSession: session.loop_once(loopstate) session.queueevent("pytest_runtest_logreport", report=run(item, node)) session.loop_once(loopstate) - assert loopstate.shuttingdown - assert not loopstate.testsfailed + assert loopstate.shuttingdown + assert not loopstate.testsfailed def test_no_node_remaining_for_tests(self, testdir): item = testdir.getitem("def test_func(): pass") @@ -162,7 +162,7 @@ class TestDSession: session = DSession(item.config) node = MockNode() session.addnode(node) - + # setup a HostDown event session.queueevent("pytest_testnodedown", node=node, error=None) @@ -174,12 +174,12 @@ class TestDSession: def test_removeitem_from_failing_teardown(self, testdir): # teardown reports only come in when they signal a failure - # internal session-management should basically ignore them - # XXX probably it'S best to invent a new error hook for + # internal session-management should basically ignore them + # XXX probably it'S best to invent a new error hook for # teardown/setup related failures modcol = testdir.getmodulecol(""" - def test_one(): - pass + def test_one(): + pass def teardown_function(function): assert 0 """) @@ -190,8 +190,8 @@ class TestDSession: node1, node2 = MockNode(), MockNode() session.addnode(node1) session.addnode(node2) - - # have one test pending for a node that goes down + + # have one test pending for a node that goes down session.senditems_each([item1]) nodes = session.item2nodes[item1] class rep: @@ -233,14 +233,14 @@ class TestDSession: session.loop_once(loopstate) assert node.sent == [item] - ev = run(item, node, excinfo=excinfo) + ev = run(item, node, excinfo=excinfo) session.queueevent("pytest_runtest_logreport", report=ev) session.loop_once(loopstate) - assert loopstate.shuttingdown + assert loopstate.shuttingdown session.queueevent("pytest_testnodedown", node=node, error=None) session.loop_once(loopstate) dumpqueue(session.queue) - return session, loopstate.exitstatus + return session, loopstate.exitstatus def test_exit_completed_tests_ok(self, testdir): item = testdir.getitem("def test_func(): pass") @@ -254,9 +254,9 @@ class TestDSession: def test_exit_on_first_failing(self, testdir): modcol = testdir.getmodulecol(""" - def test_fail(): + def test_fail(): assert 0 - def test_pass(): + def test_pass(): pass """) modcol.config.option.maxfail = 1 @@ -268,10 +268,10 @@ class TestDSession: # trigger testing - this sends tests to the node session.triggertesting(items) - # run tests ourselves and produce reports + # run tests ourselves and produce reports ev1 = run(items[0], node, "fail") ev2 = run(items[1], node, None) - session.queueevent("pytest_runtest_logreport", report=ev1) + session.queueevent("pytest_runtest_logreport", report=ev1) session.queueevent("pytest_runtest_logreport", report=ev2) # now call the loop loopstate = session._initloopstate(items) @@ -281,11 +281,11 @@ class TestDSession: def test_maxfail(self, testdir): modcol = testdir.getmodulecol(""" - def test_fail1(): + def test_fail1(): assert 0 - def test_fail2(): + def test_fail2(): assert 0 - def test_pass(): + def test_pass(): pass """) modcol.config.option.maxfail = 2 @@ -297,7 +297,7 @@ class TestDSession: # trigger testing - this sends tests to the node session.triggertesting(items) - # run tests ourselves and produce reports + # run tests ourselves and produce reports ev1 = run(items[0], node, "fail") ev2 = run(items[1], node, "fail") session.queueevent("pytest_runtest_logreport", report=ev1) # a failing one @@ -329,23 +329,23 @@ class TestDSession: def test_filteritems(self, testdir): modcol = testdir.getmodulecol(""" - def test_fail(): + def test_fail(): assert 0 - def test_pass(): + def test_pass(): pass """) session = DSession(modcol.config) modcol.config.option.keyword = "nothing" dsel = session.filteritems([modcol]) - assert dsel == [modcol] + assert dsel == [modcol] items = modcol.collect() hookrecorder = testdir.getreportrecorder(session).hookrecorder remaining = session.filteritems(items) assert remaining == [] - + event = hookrecorder.getcalls("pytest_deselected")[-1] - assert event.items == items + assert event.items == items modcol.config.option.keyword = "test_fail" remaining = session.filteritems(items) @@ -366,16 +366,16 @@ class TestDSession: session.loop_once(loopstate) assert node._shutdown is True assert loopstate.exitstatus is None, "loop did not wait for testnodedown" - assert loopstate.shuttingdown + assert loopstate.shuttingdown session.queueevent("pytest_testnodedown", node=node, error=None) session.loop_once(loopstate) assert loopstate.exitstatus == 0 def test_nopending_but_collection_remains(self, testdir): modcol = testdir.getmodulecol(""" - def test_fail(): + def test_fail(): assert 0 - def test_pass(): + def test_pass(): pass """) session = DSession(modcol.config) @@ -385,24 +385,24 @@ class TestDSession: colreport = modcol.config.hook.pytest_make_collect_report(collector=modcol) item1, item2 = colreport.result session.senditems_load([item1]) - # node2pending will become empty when the loop sees the report + # node2pending will become empty when the loop sees the report rep = run(item1, node) - session.queueevent("pytest_runtest_logreport", report=run(item1, node)) + session.queueevent("pytest_runtest_logreport", report=run(item1, node)) # but we have a collection pending - session.queueevent("pytest_collectreport", report=colreport) + session.queueevent("pytest_collectreport", report=colreport) loopstate = session._initloopstate([]) session.loop_once(loopstate) assert loopstate.exitstatus is None, "loop did not care for collection report" - assert not loopstate.colitems + assert not loopstate.colitems session.loop_once(loopstate) assert loopstate.colitems == colreport.result assert loopstate.exitstatus is None, "loop did not care for colitems" def test_dist_some_tests(self, testdir): p1 = testdir.makepyfile(test_one=""" - def test_1(): + def test_1(): pass def test_x(): import py @@ -420,7 +420,7 @@ class TestDSession: assert rep.skipped rep = hookrecorder.popcall("pytest_runtest_logreport").report assert rep.failed - # see that the node is really down + # see that the node is really down node = hookrecorder.popcall("pytest_testnodedown").node assert node.gateway.spec.popen #XXX eq.geteventargs("pytest_sessionfinish") @@ -442,7 +442,7 @@ class TestDSession: # executable = "hello" # platform = "xyz" # cwd = "qwe" - + #dsession.pytest_gwmanage_newgateway(gw1, rinfo) #linecomp.assert_contains_lines([ # "*X1*popen*xyz*2.5*" @@ -461,9 +461,9 @@ def test_collected_function_causes_remote_skip(testdir): path.remove() else: py.test.skip("remote skip") - def test_func(): + def test_func(): pass - def test_func2(): + def test_func2(): pass """ % str(sub.ensure("somefile")))) result = testdir.runpytest('-v', '--dist=each', '--tx=popen') @@ -473,14 +473,14 @@ def test_collected_function_causes_remote_skip(testdir): def test_teardownfails_one_function(testdir): p = testdir.makepyfile(""" - def test_func(): + def test_func(): pass def teardown_function(function): assert 0 """) result = testdir.runpytest(p, '--dist=each', '--tx=popen') result.stdout.fnmatch_lines([ - "*def teardown_function(function):*", + "*def teardown_function(function):*", "*1 passed*1 error*" ]) @@ -493,7 +493,7 @@ def test_terminate_on_hangingnode(testdir): time.sleep(3) """) result = testdir.runpytest(p, '--dist=each', '--tx=popen//id=my') - assert result.duration < 2.0 + assert result.duration < 2.0 result.stdout.fnmatch_lines([ "*killed*my*", ]) @@ -510,9 +510,9 @@ def test_session_hooks(testdir): f.write("xy") f.close() # let's fail on the slave - if session.nodeid: + if session.nodeid: raise ValueError(42) - """) + """) p = testdir.makepyfile(""" import sys def test_hello(): @@ -523,7 +523,7 @@ def test_session_hooks(testdir): "*ValueError*", "*1 passed*", ]) - assert result.ret + assert result.ret d = result.parseoutcomes() assert d['passed'] == 1 assert testdir.tmpdir.join("my1").check() @@ -534,7 +534,7 @@ def test_funcarg_teardown_failure(testdir): def pytest_funcarg__myarg(request): def teardown(val): raise ValueError(val) - return request.cached_setup(setup=lambda: 42, teardown=teardown, + return request.cached_setup(setup=lambda: 42, teardown=teardown, scope="module") def test_hello(myarg): pass @@ -562,4 +562,4 @@ def test_crashing_item(testdir): ]) - + diff --git a/testing/test_gwmanage.py b/testing/test_gwmanage.py index 26ae0f8..d7d2208 100644 --- a/testing/test_gwmanage.py +++ b/testing/test_gwmanage.py @@ -25,7 +25,7 @@ class TestGatewayManagerPopen: assert spec.chdir == "pyexecnetcache" for spec in GatewayManager(l, hook, defaultchdir="abc").specs: assert spec.chdir == "abc" - + def test_popen_makegateway_events(self, hook, hookrecorder, _pytest): hm = GatewayManager(["popen"] * 2, hook) hm.makegateways() @@ -34,10 +34,10 @@ class TestGatewayManagerPopen: assert call.gateway.id == "gw0" assert call.platinfo.executable == call.gateway._rinfo().executable call = hookrecorder.popcall("pytest_gwmanage_newgateway") - assert call.gateway.id == "gw1" + assert call.gateway.id == "gw1" assert len(hm.group) == 2 hm.exit() - assert not len(hm.group) + assert not len(hm.group) def test_popens_rsync(self, hook, mysetup): source = mysetup.source @@ -56,11 +56,11 @@ class TestGatewayManagerPopen: hm.rsync(source, notify=lambda *args: l.append(args)) assert not l hm.exit() - assert not len(hm.group) - assert "sys.path.insert" in gw.remote_exec.args[0] + assert not len(hm.group) + assert "sys.path.insert" in gw.remote_exec.args[0] def test_rsync_popen_with_path(self, hook, mysetup): - source, dest = mysetup.source, mysetup.dest + source, dest = mysetup.source, mysetup.dest hm = GatewayManager(["popen//chdir=%s" %dest] * 1, hook) hm.makegateways() source.ensure("dir1", "dir2", "hello") @@ -75,16 +75,16 @@ class TestGatewayManagerPopen: assert dest.join("dir1", "dir2", 'hello').check() def test_rsync_same_popen_twice(self, hook, mysetup, hookrecorder): - source, dest = mysetup.source, mysetup.dest + source, dest = mysetup.source, mysetup.dest hm = GatewayManager(["popen//chdir=%s" %dest] * 2, hook) hm.makegateways() source.ensure("dir1", "dir2", "hello") hm.rsync(source) - call = hookrecorder.popcall("pytest_gwmanage_rsyncstart") - assert call.source == source + call = hookrecorder.popcall("pytest_gwmanage_rsyncstart") + assert call.source == source assert len(call.gateways) == 1 assert call.gateways[0] in hm.group - call = hookrecorder.popcall("pytest_gwmanage_rsyncfinish") + call = hookrecorder.popcall("pytest_gwmanage_rsyncfinish") class pytest_funcarg__mysetup: def __init__(self, request): diff --git a/testing/test_mypickle.py b/testing/test_mypickle.py index 9e5f205..a6c2860 100644 --- a/testing/test_mypickle.py +++ b/testing/test_mypickle.py @@ -7,7 +7,7 @@ Queue = py.builtin._tryimport('queue', 'Queue').Queue from xdist.mypickle import ImmutablePickler, PickleChannel from xdist.mypickle import UnpickleError, makekey -# first let's test some basic functionality +# first let's test some basic functionality def pytest_generate_tests(metafunc): if 'picklemod' in metafunc.funcargnames: @@ -52,17 +52,17 @@ def test_underlying_basic_pickling_mechanisms(picklemod): pickler2.dump(d_other) f2.seek(0) - - unpickler1.memo = dict([(makekey(x), y) + + unpickler1.memo = dict([(makekey(x), y) for x, y in pickler1.memo.values()]) d_back = unpickler1.load() assert d is d_back -class A: +class A: pass - + def test_pickle_and_back_IS_same(obj, proto): p1 = ImmutablePickler(uneven=False, protocol=proto) p2 = ImmutablePickler(uneven=True, protocol=proto) @@ -70,7 +70,7 @@ def test_pickle_and_back_IS_same(obj, proto): d2 = p2.loads(s1) s2 = p2.dumps(d2) obj_back = p1.loads(s2) - assert obj is obj_back + assert obj is obj_back def test_pickling_twice_before_unpickling(): p1 = ImmutablePickler(uneven=False) @@ -78,7 +78,7 @@ def test_pickling_twice_before_unpickling(): a1 = A() a2 = A() - a3 = A() + a3 = A() a3.a1 = a1 a2.a1 = a1 s1 = p1.dumps(a1) @@ -102,7 +102,7 @@ def test_pickling_concurrently(): a1.hasattr = 42 a2 = A() - s1 = p1.dumps(a1) + s1 = p1.dumps(a1) s2 = p2.dumps(a2) other_a1 = p2.loads(s1) other_a2 = p1.loads(s2) @@ -123,7 +123,7 @@ class TestPickleChannelFunctional: "import py ; py.path.local(%r).pyimport()" %(__file__) ) cls.gw.remote_init_threads(5) - # we need the remote test code to import + # we need the remote test code to import # the same test module here def test_popen_send_instance(self): @@ -143,7 +143,7 @@ class TestPickleChannelFunctional: assert a_received.hello == 10 channel.send(a_received) remote_a2_is_a1 = channel.receive() - assert remote_a2_is_a1 + assert remote_a2_is_a1 def test_send_concurrent(self): channel = self.gw.remote_exec(""" @@ -152,7 +152,7 @@ class TestPickleChannelFunctional: from testing.test_mypickle import A l = [A() for i in range(10)] channel.send(l) - other_l = channel.receive() + other_l = channel.receive() channel.send((l, other_l)) channel.send(channel.receive()) channel.receive() @@ -164,17 +164,17 @@ class TestPickleChannelFunctional: channel.send(other_l) ret = channel.receive() assert ret[0] is other_l - assert ret[1] is l + assert ret[1] is l back = channel.receive() - assert other_l is other_l + assert other_l is other_l channel.send(None) - #s1 = p1.dumps(a1) + #s1 = p1.dumps(a1) #s2 = p2.dumps(a2) #other_a1 = p2.loads(s1) #other_a2 = p1.loads(s2) #a1_back = p1.loads(p2.dumps(other_a1)) - + def test_popen_with_callback(self): channel = self.gw.remote_exec(""" from xdist.mypickle import PickleChannel @@ -194,7 +194,7 @@ class TestPickleChannelFunctional: assert a_received.hello == 10 channel.send(a_received) #remote_a2_is_a1 = queue.get(timeout=TESTTIMEOUT) - #assert remote_a2_is_a1 + #assert remote_a2_is_a1 def test_popen_with_callback_with_endmarker(self): channel = self.gw.remote_exec(""" @@ -210,13 +210,13 @@ class TestPickleChannelFunctional: channel = PickleChannel(channel) queue = Queue() channel.setcallback(queue.put, endmarker=-1) - + a_received = queue.get(timeout=TESTTIMEOUT) assert isinstance(a_received, A) assert a_received.hello == 10 channel.send(a_received) remote_a2_is_a1 = queue.get(timeout=TESTTIMEOUT) - assert remote_a2_is_a1 + assert remote_a2_is_a1 endmarker = queue.get(timeout=TESTTIMEOUT) assert endmarker == -1 @@ -235,7 +235,7 @@ class TestPickleChannelFunctional: channel._ipickle._unpicklememo.clear() channel.setcallback(queue.put, endmarker=-1) next = queue.get(timeout=TESTTIMEOUT) - assert next == -1 + assert next == -1 error = channel._getremoteerror() assert isinstance(error, UnpickleError) diff --git a/testing/test_nodemanage.py b/testing/test_nodemanage.py index bd46c9f..4cbe497 100644 --- a/testing/test_nodemanage.py +++ b/testing/test_nodemanage.py @@ -4,7 +4,7 @@ from xdist.nodemanage import NodeManager class pytest_funcarg__mysetup: def __init__(self, request): basetemp = request.config.mktemp( - "mysetup-%s" % request.function.__name__, + "mysetup-%s" % request.function.__name__, numbered=True) self.source = basetemp.mkdir("source") self.dest = basetemp.mkdir("dest") @@ -26,7 +26,7 @@ class TestNodeManager: assert p.join("dir1", "file1").check() def test_popen_rsync_subdir(self, testdir, mysetup): - source, dest = mysetup.source, mysetup.dest + source, dest = mysetup.source, mysetup.dest dir1 = mysetup.source.mkdir("dir1") dir2 = dir1.mkdir("dir2") dir2.ensure("hello") @@ -35,10 +35,10 @@ class TestNodeManager: nodemanager = NodeManager(testdir.parseconfig( "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, - source, + source, )) assert nodemanager.config.topdir == source - nodemanager.rsync_roots() + nodemanager.rsync_roots() if rsyncroot == source: dest = dest.join("source") assert dest.join("dir1").check() @@ -105,7 +105,7 @@ class TestNodeManager: nodemanager.setup_nodes(putevent=[].append) for spec in nodemanager.gwmanager.specs: l = reprec.getcalls("pytest_trace") - assert l + assert l nodemanager.teardown_nodes() def test_ssh_setup_nodes(self, specssh, testdir): @@ -113,8 +113,8 @@ class TestNodeManager: def test_one(): pass """) - reprec = testdir.inline_run("-d", "--rsyncdir=%s" % testdir.tmpdir, + reprec = testdir.inline_run("-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir) rep, = reprec.getreports("pytest_runtest_logreport") - assert rep.passed + assert rep.passed diff --git a/testing/test_pickling.py b/testing/test_pickling.py index bc52fa3..9e6442c 100644 --- a/testing/test_pickling.py +++ b/testing/test_pickling.py @@ -2,7 +2,7 @@ import py import pickle def setglobals(request): - oldconfig = py.test.config + oldconfig = py.test.config print("setting py.test.config to None") py.test.config = None def resetglobals(): @@ -73,10 +73,10 @@ def test_config__setstate__wired_correctly_in_childprocess(testdir): from xdist.mypickle import PickleChannel channel = PickleChannel(channel) config = channel.receive() - assert py.test.config == config + assert py.test.config == config """) channel = PickleChannel(channel) config = testdir.parseconfig() channel.send(config) - channel.waitclose() # this will potentially raise + channel.waitclose() # this will potentially raise gw.exit() diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 877bd55..ccc8cb1 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -27,14 +27,14 @@ class TestDistOptions: xspecs = nodemanager._getxspecs() assert len(xspecs) == 2 print(xspecs) - assert xspecs[0].popen + assert xspecs[0].popen assert xspecs[1].ssh == "xyz" def test_xspecs_multiplied(self, testdir): config = testdir.parseconfigure("--tx=3*popen",) xspecs = NodeManager(config)._getxspecs() assert len(xspecs) == 3 - assert xspecs[1].popen + assert xspecs[1].popen def test_getrsyncdirs(self, testdir): config = testdir.parseconfigure('--rsyncdir=' + str(testdir.tmpdir)) @@ -48,14 +48,14 @@ class TestDistOptions: for bn in 'x y z'.split(): p.mkdir(bn) testdir.makeconftest(""" - rsyncdirs= 'x', + rsyncdirs= 'x', """) config = testdir.parseconfigure( testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z') nm = NodeManager(config, specs=[execnet.XSpec("popen")]) roots = nm._getrsyncdirs() assert len(roots) == 3 + 2 # pylib + xdist - assert py.path.local('y') in roots - assert py.path.local('z') in roots - assert testdir.tmpdir.join('x') in roots + assert py.path.local('y') in roots + assert py.path.local('z') in roots + assert testdir.tmpdir.join('x') in roots diff --git a/testing/test_remote.py b/testing/test_remote.py index 330e221..663977d 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -1,6 +1,6 @@ import py py.test.importorskip("execnet") -from xdist.remote import LooponfailingSession, LoopState, RemoteControl +from xdist.remote import LooponfailingSession, LoopState, RemoteControl class TestRemoteControl: def test_nofailures(self, testdir): @@ -15,7 +15,7 @@ class TestRemoteControl: control = RemoteControl(item.config) control.setup() failures = control.runsession() - assert failures + assert failures control.setup() item.fspath.write("def test_func(): assert 1\n") pyc = item.fspath.new(ext=".pyc") @@ -26,13 +26,13 @@ class TestRemoteControl: def test_failure_change(self, testdir): modcol = testdir.getitem(""" - def test_func(): + def test_func(): assert 0 """) control = RemoteControl(modcol.config) control.setup() failures = control.runsession() - assert failures + assert failures control.setup() modcol.fspath.write(py.code.Source(""" def test_func(): @@ -63,7 +63,7 @@ class TestLooponFailing: loopstate = LoopState() session.loop_once(loopstate) assert len(loopstate.colitems) == 1 - + modcol.fspath.write(py.code.Source(""" def test_one(): x = 15 @@ -73,7 +73,7 @@ class TestLooponFailing: """)) assert session.statrecorder.check() session.loop_once(loopstate) - assert not loopstate.colitems + assert not loopstate.colitems def test_looponfail_from_one_to_two_tests(self, testdir): modcol = testdir.getmodulecol(""" @@ -114,7 +114,7 @@ class TestLooponFailing: modcol.fspath.write(py.code.Source(""" def test_xxx(): # renamed test - assert 0 + assert 0 def test_two(): assert 1 # pass now """)) diff --git a/testing/test_txnode.py b/testing/test_txnode.py index bb8a728..17c92f1 100644 --- a/testing/test_txnode.py +++ b/testing/test_txnode.py @@ -23,7 +23,7 @@ class EventQueue: py.builtin.print_("seen events", events) raise IOError("did not see %r events" % (eventname)) else: - name, args, kwargs = eventcall + name, args, kwargs = eventcall assert isinstance(name, str) if name == eventname: if args: @@ -55,7 +55,7 @@ class MySetup: self.nodemanager = None self.node = TXNode(self.nodemanager, self.gateway, self.config, putevent=self.queue.put) assert not self.node.channel.isclosed() - return self.node + return self.node def xfinalize(self): if hasattr(self, 'node'): @@ -78,9 +78,9 @@ def test_node_hash_equality(mysetup): class TestMasterSlaveConnection: def test_crash_invalid_item(self, mysetup): node = mysetup.makenode() - node.send(123) # invalid item + node.send(123) # invalid item kwargs = mysetup.geteventargs("pytest_testnodedown") - assert kwargs['node'] is node + assert kwargs['node'] is node #assert isinstance(kwargs['error'], execnet.RemoteError) def test_crash_killed(self, testdir, mysetup): @@ -92,19 +92,19 @@ class TestMasterSlaveConnection: os.kill(os.getpid(), 9) """) node = mysetup.makenode(item.config) - node.send(item) + node.send(item) kwargs = mysetup.geteventargs("pytest_testnodedown") - assert kwargs['node'] is node + assert kwargs['node'] is node assert "Not properly terminated" in str(kwargs['error']) def test_node_down(self, mysetup): node = mysetup.makenode() node.shutdown() kwargs = mysetup.geteventargs("pytest_testnodedown") - assert kwargs['node'] is node + assert kwargs['node'] is node assert not kwargs['error'] node.callback(node.ENDMARK) - excinfo = py.test.raises(IOError, + excinfo = py.test.raises(IOError, "mysetup.geteventargs('testnodedown', timeout=0.01)") def test_send_on_closed_channel(self, testdir, mysetup): @@ -120,14 +120,14 @@ class TestMasterSlaveConnection: node = mysetup.makenode(item.config) node.send(item) kwargs = mysetup.geteventargs("pytest_runtest_logreport") - rep = kwargs['report'] - assert rep.passed + rep = kwargs['report'] + assert rep.passed py.builtin.print_(rep) assert rep.item == item def test_send_some(self, testdir, mysetup): items = testdir.getitems(""" - def test_pass(): + def test_pass(): pass def test_fail(): assert 0 @@ -141,12 +141,12 @@ class TestMasterSlaveConnection: for outcome in "passed failed skipped".split(): kwargs = mysetup.geteventargs("pytest_runtest_logreport") report = kwargs['report'] - assert getattr(report, outcome) + assert getattr(report, outcome) node.sendlist(items) for outcome in "passed failed skipped".split(): rep = mysetup.geteventargs("pytest_runtest_logreport")['report'] - assert getattr(rep, outcome) + assert getattr(rep, outcome) def test_send_one_with_env(self, testdir, mysetup, monkeypatch): if execnet.XSpec("popen").env is None: diff --git a/testing/test_util.py b/testing/test_util.py index fd78fa8..7d64fdb 100644 --- a/testing/test_util.py +++ b/testing/test_util.py @@ -15,7 +15,7 @@ def test_filechange(tmpdir): tmp.ensure("new.py") changed = sd.check() assert changed - + tmp.join("new.py").remove() changed = sd.check() assert changed @@ -44,12 +44,12 @@ def test_pycremoval(tmpdir): pycfile = hello + "c" pycfile.ensure() changed = sd.check() - assert not changed + assert not changed hello.write("world") changed = sd.check() assert not pycfile.check() - + def test_waitonchange(tmpdir, monkeypatch): tmp = tmpdir diff --git a/tox.ini b/tox.ini index 7c57fbb..c5eacfc 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ sdistsrc={distshare}/pytest-xdist-* [testenv] changedir=testing -deps= +deps= {distshare}/py-* commands= py.test -rsfxX --tools-on-path \ diff --git a/xdist/__init__.py b/xdist/__init__.py index 47cd1cb..5eb2357 100644 --- a/xdist/__init__.py +++ b/xdist/__init__.py @@ -1,3 +1,3 @@ # -__version__ = "1.4" +__version__ = "1.5a1" diff --git a/xdist/dsession.py b/xdist/dsession.py index aab4994..a0f8cfc 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -1,5 +1,5 @@ import py -from py._test import session +from py._test import session from xdist.nodemanage import NodeManager queue = py.builtin._tryimport('queue', 'Queue') @@ -14,10 +14,10 @@ class LoopState(object): def __init__(self, dsession, colitems): self.dsession = dsession self.colitems = colitems - self.exitstatus = None - # loopstate.dowork is False after reschedule events - # because otherwise we might very busily loop - # waiting for a host to become ready. + self.exitstatus = None + # loopstate.dowork is False after reschedule events + # because otherwise we might very busily loop + # waiting for a host to become ready. self.dowork = True self.shuttingdown = False self.testsfailed = 0 @@ -47,8 +47,8 @@ class LoopState(object): crashitem = pending[0] debug("determined crashitem", crashitem) self.dsession.handle_crashitem(crashitem, node) - # XXX recovery handling for "each"? - # currently pending items are not retried + # XXX recovery handling for "each"? + # currently pending items are not retried if self.dsession.config.option.dist == "load": self.colitems.extend(pending[1:]) @@ -59,10 +59,10 @@ class LoopState(object): self.dowork = False # avoid busywait, nodes still have work class DSession(session.Session): - """ + """ Session drives the collection and running of tests - and generates test events for reporters. - """ + and generates test events for reporters. + """ LOAD_THRESHOLD_NEWITEMS = 5 ITEM_CHUNKSIZE = 10 @@ -96,7 +96,7 @@ class DSession(session.Session): allitems = self.collect_all_items(colitems) exitstatus = self.loop(allitems) self.teardown() - self.sessionfinishes(exitstatus=exitstatus) + self.sessionfinishes(exitstatus=exitstatus) return exitstatus def collect_all_items(self, colitems): @@ -111,19 +111,19 @@ class DSession(session.Session): def loop_once(self, loopstate): if loopstate.shuttingdown: return self.loop_once_shutdown(loopstate) - colitems = loopstate.colitems + colitems = loopstate.colitems if self._nodesready.isSet() and loopstate.dowork and colitems: - self.triggertesting(loopstate.colitems) + self.triggertesting(loopstate.colitems) colitems[:] = [] - # we use a timeout here so that control-C gets through + # we use a timeout here so that control-C gets through while 1: try: eventcall = self.queue.get(timeout=2.0) break except queue.Empty: continue - loopstate.dowork = True - + loopstate.dowork = True + callname, args, kwargs = eventcall if callname is not None: call = getattr(self.config.hook, callname) @@ -132,9 +132,9 @@ class DSession(session.Session): # termination conditions maxfail = self.config.getvalue("maxfail") - if (not self.node2pending or - (loopstate.testsfailed and maxfail and - loopstate.testsfailed >= maxfail) or + if (not self.node2pending or + (loopstate.testsfailed and maxfail and + loopstate.testsfailed >= maxfail) or (not self.item2nodes and not colitems and not self.queue.qsize())): if maxfail and loopstate.testsfailed >= maxfail: raise self.Interrupted("stopping after %d failures" % ( @@ -143,10 +143,10 @@ class DSession(session.Session): loopstate.shuttingdown = True if not self.node2pending: loopstate.exitstatus = session.EXIT_NOHOSTS - + def loop_once_shutdown(self, loopstate): - # once we are in shutdown mode we dont send - # events other than HostDown upstream + # once we are in shutdown mode we dont send + # events other than HostDown upstream eventname, args, kwargs = self.queue.get() if eventname == "pytest_testnodedown": self.config.hook.pytest_testnodedown(**kwargs) @@ -181,7 +181,7 @@ class DSession(session.Session): self.loop_once(loopstate) if loopstate.exitstatus is not None: exitstatus = loopstate.exitstatus - break + break except KeyboardInterrupt: excinfo = py.code.ExceptionInfo() self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo) @@ -201,7 +201,7 @@ class DSession(session.Session): def addnode(self, node): assert node not in self.node2pending self.node2pending[node] = [] - if (not hasattr(self, 'nodemanager') or + if (not hasattr(self, 'nodemanager') or len(self.node2pending) == len(self.nodemanager.gwmanager.group)): self._nodesready.set() @@ -219,7 +219,7 @@ class DSession(session.Session): return pending def triggertesting(self, colitems): - # for now we don't allow sending collectors + # for now we don't allow sending collectors for next in colitems: assert isinstance(next, py.test.collect.Item), next senditems = list(colitems) @@ -230,11 +230,11 @@ class DSession(session.Session): self.senditems_load(senditems) def queueevent(self, eventname, **kwargs): - self.queue.put((eventname, (), kwargs)) + self.queue.put((eventname, (), kwargs)) def senditems_each(self, tosend): if not tosend: - return + return for node, pending in self.node2pending.items(): node.sendlist(tosend) pending.extend(tosend) @@ -247,7 +247,7 @@ class DSession(session.Session): def senditems_load(self, tosend): if not tosend: - return + return available = [] for node, pending in self.node2pending.items(): if len(pending) < self.LOAD_THRESHOLD_NEWITEMS: @@ -281,7 +281,7 @@ class DSession(session.Session): pending.remove(item) def handle_crashitem(self, item, node): - runner = item.config.pluginmanager.getplugin("runner") + runner = item.config.pluginmanager.getplugin("runner") info = "!!! Node %r crashed during running of test %r" %(node, item) rep = runner.ItemTestReport(item=item, excinfo=info, when="???") rep.node = node @@ -290,11 +290,11 @@ class DSession(session.Session): def setup(self): """ setup any neccessary resources ahead of the test run. """ if not self.config.getvalue("verbose"): - self.report_line("instantiating gateways (use -v for details): %s" % + self.report_line("instantiating gateways (use -v for details): %s" % ",".join(self.config.option.tx)) self.nodemanager = NodeManager(self.config) self.nodemanager.setup_nodes(putevent=self.queue.put) def teardown(self): - """ teardown any resources after a test run. """ + """ teardown any resources after a test run. """ self.nodemanager.teardown_nodes() diff --git a/xdist/gwmanage.py b/xdist/gwmanage.py index ec54e70..fdc5480 100644 --- a/xdist/gwmanage.py +++ b/xdist/gwmanage.py @@ -29,8 +29,8 @@ class GatewayManager: gateway=gw, platinfo=gw._rinfo()) def rsync(self, source, notify=None, verbose=False, ignores=None): - """ perform rsync to all remote hosts. - """ + """ perform rsync to all remote hosts. + """ rsync = HostRSync(source, verbose=verbose, ignores=ignores) seen = py.builtin.set() gateways = [] @@ -38,7 +38,7 @@ class GatewayManager: spec = gateway.spec if spec.popen and not spec.chdir: # XXX this assumes that sources are python-packages - # and that adding the basedir does not hurt + # and that adding the basedir does not hurt gateway.remote_exec(""" import sys ; sys.path.insert(0, %r) """ % os.path.dirname(str(source))).waitclose() @@ -52,20 +52,20 @@ class GatewayManager: gateways.append(gateway) if seen: self.hook.pytest_gwmanage_rsyncstart( - source=source, - gateways=gateways, + source=source, + gateways=gateways, ) rsync.send() self.hook.pytest_gwmanage_rsyncfinish( - source=source, - gateways=gateways, + source=source, + gateways=gateways, ) def exit(self): self.group.terminate(self.EXIT_TIMEOUT) class HostRSync(execnet.RSync): - """ RSyncer that filters out common files + """ RSyncer that filters out common files """ def __init__(self, sourcedir, *args, **kwargs): self._synced = {} @@ -78,7 +78,7 @@ class HostRSync(execnet.RSync): def filter(self, path): path = py.path.local(path) if not path.ext in ('.pyc', '.pyo'): - if not path.basename.endswith('~'): + if not path.basename.endswith('~'): if path.check(dotfile=0): for x in self._ignores: if path == x: @@ -88,7 +88,7 @@ class HostRSync(execnet.RSync): def add_target_host(self, gateway, finished=None): remotepath = os.path.basename(self._sourcedir) - super(HostRSync, self).add_target(gateway, remotepath, + super(HostRSync, self).add_target(gateway, remotepath, finishedcallback=finished, delete=True,) diff --git a/xdist/mypickle.py b/xdist/mypickle.py index d30ef7e..4edec91 100644 --- a/xdist/mypickle.py +++ b/xdist/mypickle.py @@ -1,14 +1,14 @@ """ - Pickling support for two processes that want to exchange - *immutable* object instances. Immutable in the sense - that the receiving side of an object can modify its - copy but when it sends it back the original sending + Pickling support for two processes that want to exchange + *immutable* object instances. Immutable in the sense + that the receiving side of an object can modify its + copy but when it sends it back the original sending side will continue to see its unmodified version (and no actual state will go over the wire). - This module also implements an experimental - execnet pickling channel using this idea. + This module also implements an experimental + execnet pickling channel using this idea. """ @@ -18,7 +18,7 @@ import sys, os, struct if sys.version_info >= (3,0): makekey = lambda x: x - fromkey = lambda x: x + fromkey = lambda x: x from pickle import _Pickler as Pickler from pickle import _Unpickler as Unpickler else: @@ -29,7 +29,7 @@ else: class MyPickler(Pickler): """ Pickler with a custom memoize() - to take care of unique ID creation. + to take care of unique ID creation. See the usage in ImmutablePickler """ def __init__(self, immo, file, protocol, uneven): @@ -37,7 +37,7 @@ class MyPickler(Pickler): self.uneven = uneven self._unpicklememo = immo._unpicklememo self.memo = immo._picklememo - + def memoize(self, obj): if self.fast: return @@ -55,23 +55,23 @@ class MyPickler(Pickler): # def save_string(self, obj, pack=struct.pack): # obj = unicode(obj) # self.save_unicode(obj, pack=pack) - # Pickler.dispatch[str] = save_string - + # Pickler.dispatch[str] = save_string + class UnpicklingDict(dict): def __init__(self, picklememo): super(UnpicklingDict, self).__init__() self._picklememo = picklememo - + def __setitem__(self, key, obj): super(UnpicklingDict, self).__setitem__(key, obj) self._picklememo[id(obj)] = (fromkey(key), obj) - + class ImmutablePickler: def __init__(self, uneven, protocol=0): - """ ImmutablePicklers are instantiated in Pairs. + """ ImmutablePicklers are instantiated in Pairs. The two sides need to create unique IDs while pickling their objects. This is - done by using either even or uneven + done by using either even or uneven numbers, depending on the instantiation parameter. """ @@ -82,8 +82,8 @@ class ImmutablePickler: def selfmemoize(self, obj): # this is for feeding objects to ourselfes - # which be the case e.g. if you want to pickle - # from a forked process back to the original + # which be the case e.g. if you want to pickle + # from a forked process back to the original f = py.io.BytesIO() pickler = MyPickler(self, f, self._protocol, uneven=self.uneven) pickler.memoize(obj) @@ -92,7 +92,7 @@ class ImmutablePickler: f = py.io.BytesIO() pickler = MyPickler(self, f, self._protocol, uneven=self.uneven) pickler.dump(obj) - #print >>debug, "dumped", obj + #print >>debug, "dumped", obj #print >>debug, "picklememo", self._picklememo return f.getvalue() @@ -117,20 +117,20 @@ class UnpickleError(Exception): return self.formatted class PickleChannel(object): - """ PickleChannels wrap execnet channels + """ PickleChannels wrap execnet channels and allow to send/receive by using - "immutable pickling". + "immutable pickling". """ _unpicklingerror = None def __init__(self, channel): self._channel = channel - # we use the fact that each side of a + # we use the fact that each side of a # gateway connection counts with uneven - # or even numbers depending on which + # or even numbers depending on which # side it is (for the purpose of creating # unique ids - which is what we need it here for) - uneven = channel.gateway._channelfactory.count % 2 - self._ipickle = ImmutablePickler(uneven=uneven) + uneven = channel.gateway._channelfactory.count % 2 + self._ipickle = ImmutablePickler(uneven=uneven) self.RemoteError = channel.RemoteError def send(self, obj): diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 2d72fd6..f97a0cc 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -1,6 +1,6 @@ def pytest_gwmanage_newgateway(gateway, platinfo): - """ called on new raw gateway creation. """ + """ called on new raw gateway creation. """ def pytest_gwmanage_rsyncstart(source, gateways): """ called before rsyncing a directory to remote gateways takes place. """ diff --git a/xdist/nodemanage.py b/xdist/nodemanage.py index 7cbb394..be0f9b7 100644 --- a/xdist/nodemanage.py +++ b/xdist/nodemanage.py @@ -4,10 +4,10 @@ import xdist from xdist.txnode import TXNode from xdist.gwmanage import GatewayManager import execnet - + class NodeManager(object): def __init__(self, config, specs=None): - self.config = config + self.config = config if specs is None: specs = self._getxspecs() self.roots = self._getrsyncdirs() @@ -23,32 +23,32 @@ class NodeManager(object): def rsync_roots(self): """ make sure that all remote gateways have the same set of roots in their - current directory. + current directory. """ self.makegateways() options = { - 'ignores': self.config_getignores(), + 'ignores': self.config_getignores(), 'verbose': self.config.option.verbose, } if self.roots: # send each rsync root for root in self.roots: self.gwmanager.rsync(root, **options) - else: - XXX # do we want to care for situations without explicit rsyncdirs? + else: + XXX # do we want to care for situations without explicit rsyncdirs? # we transfer our topdir as the root self.gwmanager.rsync(self.config.topdir, **options) - # and cd into it + # and cd into it self.gwmanager.multi_chdir(self.config.topdir.basename, inplacelocal=False) def makegateways(self): - # we change to the topdir sot that - # PopenGateways will have their cwd - # such that unpickling configs will - # pick it up as the right topdir + # we change to the topdir sot that + # PopenGateways will have their cwd + # such that unpickling configs will + # pick it up as the right topdir # (for other gateways this chdir is irrelevant) self.trace("making gateways") - old = self.config.topdir.chdir() + old = self.config.topdir.chdir() try: self.gwmanager.makegateways() finally: @@ -59,7 +59,7 @@ class NodeManager(object): self.trace("setting up nodes") for gateway in self.gwmanager.group: node = TXNode(self, gateway, self.config, putevent) - gateway.node = node # to keep node alive + gateway.node = node # to keep node alive self.trace("started node %r" % node) def teardown_nodes(self): @@ -83,7 +83,7 @@ class NodeManager(object): def _getrsyncdirs(self): config = self.config - candidates = [py._pydir] + candidates = [py._pydir] candidates += [py.path.local(xdist.__file__).dirpath()] candidates += config.option.rsyncdir conftestroots = config.getconftest_pathlist("rsyncdirs") diff --git a/xdist/plugin.py b/xdist/plugin.py index ac150ae..58ecd71 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -1,23 +1,23 @@ """loop on failing tests, distribute test runs to CPUs and hosts. -The `pytest-xdist`_ plugin extends py.test with some unique +The `pytest-xdist`_ plugin extends py.test with some unique test execution modes: * Looponfail: run your tests repeatedly in a subprocess. After each run py.test waits until a file in your project changes and then re-runs the previously failing tests. This is repeated until all tests pass after which again - a full run is performed. + a full run is performed. * Load-balancing: if you have multiple CPUs or hosts you can use - those for a combined test run. This allows to speed up - development or to use special resources of remote machines. + those for a combined test run. This allows to speed up + development or to use special resources of remote machines. * Multi-Platform coverage: you can specify different Python interpreters - or different platforms and run tests in parallel on all of them. + or different platforms and run tests in parallel on all of them. -Before running tests remotely, ``py.test`` efficiently synchronizes your -program source code to the remote place. All test results -are reported back and displayed to your local test session. +Before running tests remotely, ``py.test`` efficiently synchronizes your +program source code to the remote place. All test results +are reported back and displayed to your local test session. You may specify different Python versions and interpreters. .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist @@ -32,11 +32,11 @@ To send tests to multiple CPUs, type:: py.test -n NUM -Especially for longer running tests or tests requiring -a lot of IO this can lead to considerable speed ups. +Especially for longer running tests or tests requiring +a lot of IO this can lead to considerable speed ups. -Running tests in a Python subprocess +Running tests in a Python subprocess +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ To instantiate a python2.4 sub process and send tests to it, you may type:: @@ -44,51 +44,51 @@ To instantiate a python2.4 sub process and send tests to it, you may type:: py.test -d --tx popen//python=python2.4 This will start a subprocess which is run with the "python2.4" -Python interpreter, found in your system binary lookup path. +Python interpreter, found in your system binary lookup path. If you prefix the --tx option value like this:: --tx 3*popen//python=python2.4 then three subprocesses would be created and tests -will be load-balanced across these three processes. +will be load-balanced across these three processes. Sending tests to remote SSH accounts +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Suppose you have a package ``mypkg`` which contains some +Suppose you have a package ``mypkg`` which contains some tests that you can successfully run locally. And you -have a ssh-reachable machine ``myhost``. Then +have a ssh-reachable machine ``myhost``. Then you can ad-hoc distribute your tests by typing:: py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg -This will synchronize your ``mypkg`` package directory -to an remote ssh account and then locally collect tests -and send them to remote places for execution. +This will synchronize your ``mypkg`` package directory +to an remote ssh account and then locally collect tests +and send them to remote places for execution. -You can specify multiple ``--rsyncdir`` directories -to be sent to the remote side. +You can specify multiple ``--rsyncdir`` directories +to be sent to the remote side. **NOTE:** For py.test to collect and send tests correctly you not only need to make sure all code and tests directories are rsynced, but that any test (sub) directory also has an ``__init__.py`` file because internally py.test references tests as a fully qualified python -module path. **You will otherwise get strange errors** +module path. **You will otherwise get strange errors** during setup of the remote side. Sending tests to remote Socket Servers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Download the single-module `socketserver.py`_ Python program +Download the single-module `socketserver.py`_ Python program and run it like this:: python socketserver.py It will tell you that it starts listening on the default -port. You can now on your home machine specify this +port. You can now on your home machine specify this new socket host with something like this:: py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg @@ -96,17 +96,17 @@ new socket host with something like this:: .. _`atonce`: -Running tests on many platforms at once +Running tests on many platforms at once +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ The basic command to run tests on multiple platforms is:: - py.test --dist=each --tx=spec1 --tx=spec2 + py.test --dist=each --tx=spec1 --tx=spec2 If you specify a windows host, an OSX host and a Linux -environment this command will send each tests to all +environment this command will send each tests to all platforms - and report back failures from all platforms -at once. The specifications strings use the `xspec syntax`_. +at once. The specifications strings use the `xspec syntax`_. .. _`xspec syntax`: http://codespeak.net/execnet/trunk/basics.html#xspec @@ -117,14 +117,14 @@ at once. The specifications strings use the `xspec syntax`_. Specifying test exec environments in a conftest.py +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -Instead of specifying command line options, you can +Instead of specifying command line options, you can put options values in a ``conftest.py`` file like this:: option_tx = ['ssh=myhost//python=python2.5', 'popen//python=python2.5'] option_dist = True Any commandline ``--tx`` specifictions will add to the list of -available execution environments. +available execution environments. Specifying "rsync" dirs in a conftest.py +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -144,33 +144,33 @@ import sys import py def pytest_addoption(parser): - group = parser.getgroup("xdist", "distributed and subprocess testing") + group = parser.getgroup("xdist", "distributed and subprocess testing") group._addoption('-f', '--looponfail', action="store_true", dest="looponfail", default=False, help="run tests in subprocess, wait for modified files " "and re-run failing test set until all pass.") - group._addoption('-n', dest="numprocesses", metavar="numprocesses", - action="store", type="int", + group._addoption('-n', dest="numprocesses", metavar="numprocesses", + action="store", type="int", help="shortcut for '--dist=load --tx=NUM*popen'") group.addoption('--boxed', action="store_true", dest="boxed", default=False, - help="box each test run in a separate process (unix)") - group._addoption('--dist', metavar="distmode", - action="store", choices=['load', 'each', 'no'], - type="choice", dest="dist", default="no", + help="box each test run in a separate process (unix)") + group._addoption('--dist', metavar="distmode", + action="store", choices=['load', 'each', 'no'], + type="choice", dest="dist", default="no", help=("set mode for distributing tests to exec environments.\n\n" "each: send each test to each available environment.\n\n" "load: send each test to available environment.\n\n" "(default) no: run tests inprocess, don't distribute.")) - group._addoption('--tx', dest="tx", action="append", default=[], + group._addoption('--tx', dest="tx", action="append", default=[], metavar="xspec", help=("add a test execution environment. some examples: " "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 " "--tx ssh=user@codespeak.net//chdir=testcache")) - group._addoption('-d', + group._addoption('-d', action="store_true", dest="distload", default=False, help="load-balance tests. shortcut for '--dist=load'") - group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", + group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", help="add directory for rsyncing to remote tx nodes.") # ------------------------------------------------------------------------- @@ -223,20 +223,20 @@ def pytest_runtest_protocol(item): return True def forked_run_report(item): - # for now, we run setup/teardown in the subprocess - # XXX optionally allow sharing of setup/teardown + # for now, we run setup/teardown in the subprocess + # XXX optionally allow sharing of setup/teardown from py._plugin.pytest_runner import runtestprotocol EXITSTATUS_TESTEXIT = 4 from xdist.mypickle import ImmutablePickler ipickle = ImmutablePickler(uneven=0) ipickle.selfmemoize(item.config) - # XXX workaround the issue that 2.6 cannot pickle + # XXX workaround the issue that 2.6 cannot pickle # instances of classes defined in global conftest.py files - ipickle.selfmemoize(item) + ipickle.selfmemoize(item) def runforked(): try: reports = runtestprotocol(item, log=False) - except KeyboardInterrupt: + except KeyboardInterrupt: py.std.os._exit(EXITSTATUS_TESTEXIT) return ipickle.dumps(reports) @@ -275,16 +275,16 @@ class TerminalDistReporter: def pytest_runtest_logreport(self, report): if hasattr(report, 'node'): report.headerlines.append(self.gateway2info.get( - report.node.gateway, + report.node.gateway, "node %r (platinfo not found? strange)")) - + def pytest_gwmanage_newgateway(self, gateway, platinfo): #self.write_line("%s instantiated gateway from spec %r" %(gateway.id, gateway.spec._spec)) d = {} d['version'] = self.tplugin.repr_pythonversion(platinfo.version_info) d['id'] = gateway.id - d['spec'] = gateway.spec._spec - d['platform'] = platinfo.platform + d['spec'] = gateway.spec._spec + d['platform'] = platinfo.platform if self.config.option.verbose: d['extra'] = "- " + platinfo.executable else: diff --git a/xdist/remote.py b/xdist/remote.py index 7cb0076..fff3b73 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -1,11 +1,11 @@ """ - LooponfailingSession and Helpers. + LooponfailingSession and Helpers. - NOTE that one really has to avoid loading and depending on - application modules within the controlling process + NOTE that one really has to avoid loading and depending on + application modules within the controlling process (the one that starts repeatedly test processes) - otherwise changes to source code can crash - the controlling process which should never happen. + otherwise changes to source code can crash + the controlling process which should never happen. """ import py import sys @@ -16,8 +16,8 @@ from xdist import util class LooponfailingSession(Session): def __init__(self, config): super(LooponfailingSession, self).__init__(config=config) - self.rootdirs = [self.config.topdir] # xxx dist_rsync_roots? - self.statrecorder = util.StatRecorder(self.rootdirs) + self.rootdirs = [self.config.topdir] # xxx dist_rsync_roots? + self.statrecorder = util.StatRecorder(self.rootdirs) self.remotecontrol = RemoteControl(self.config) self.out = py.io.TerminalWriter() @@ -28,7 +28,7 @@ class LooponfailingSession(Session): self.loop_once(loopstate) if not loopstate.colitems and loopstate.wasfailing: continue # the last failures passed, let's rerun all - self.statrecorder.waitonchange(checkinterval=2.0) + self.statrecorder.waitonchange(checkinterval=2.0) except KeyboardInterrupt: print @@ -74,9 +74,9 @@ class RemoteControl(object): if not os.path.isabs(p): p = os.path.abspath(p) newpaths.append(p) - sys.path[:] = newpaths + sys.path[:] = newpaths os.chdir(chdir) # unpickling config uses cwd as topdir - + config_state = channel.receive() fullwidth, hasmarkup = channel.receive() py.test.config.__setstate__(config_state) @@ -85,7 +85,7 @@ class RemoteControl(object): sys.stdout = sys.stderr = outchannel.makefile('w') from xdist.remote import slave_runsession - slave_runsession(channel, py.test.config, fullwidth, hasmarkup) + slave_runsession(channel, py.test.config, fullwidth, hasmarkup) """) channel.send(str(self.config.topdir)) remote_outchannel = channel.receive() @@ -125,15 +125,15 @@ class RemoteControl(object): def slave_runsession(channel, config, fullwidth, hasmarkup): """ we run this on the other side. """ if config.option.debug: - def DEBUG(*args): + def DEBUG(*args): print(" ".join(map(str, args))) else: def DEBUG(*args): pass DEBUG("SLAVE: received configuration, using topdir:", config.topdir) #config.option.session = None - config.option.looponfail = False - config.option.usepdb = False + config.option.looponfail = False + config.option.usepdb = False try: trails = channel.receive() except KeyboardInterrupt: @@ -142,7 +142,7 @@ def slave_runsession(channel, config, fullwidth, hasmarkup): DEBUG("SLAVE: initsession()") session = config.initsession() # XXX configure the reporter object's terminal writer more directly - # XXX and write a test for this remote-terminal setting logic + # XXX and write a test for this remote-terminal setting logic config.pytest_terminal_hasmarkup = hasmarkup config.pytest_terminal_fullwidth = fullwidth if trails: @@ -152,25 +152,25 @@ def slave_runsession(channel, config, fullwidth, hasmarkup): colitem = config._rootcol.fromtrail(trail) except ValueError: #XXX send info for "test disappeared" or so - continue + continue colitems.append(colitem) else: colitems = config.getinitialnodes() - session.shouldclose = channel.isclosed - + session.shouldclose = channel.isclosed + class Failures(list): def pytest_runtest_logreport(self, report): if report.failed: self.append(report) pytest_collectreport = pytest_runtest_logreport - + failreports = Failures() session.pluginmanager.register(failreports) DEBUG("SLAVE: starting session.main()") session.main(colitems) repr_pytest_looponfailinfo( - failreports=list(failreports), + failreports=list(failreports), rootdirs=[config.topdir]) rootcol = session.config._rootcol channel.send([rootcol.totrail(rep.getnode()) for rep in failreports]) diff --git a/xdist/txnode.py b/xdist/txnode.py index 9aee27f..09a777b 100644 --- a/xdist/txnode.py +++ b/xdist/txnode.py @@ -1,22 +1,22 @@ """ - Manage setup, running and local representation of remote nodes/processes. + Manage setup, running and local representation of remote nodes/processes. """ import py from xdist.mypickle import PickleChannel from py._test.session import Session class TXNode(object): - """ Represents a Test Execution environment in the controlling process. - - sets up a slave node through an execnet gateway + """ Represents a Test Execution environment in the controlling process. + - sets up a slave node through an execnet gateway - manages sending of test-items and receival of results and events - - creates events when the remote side crashes + - creates events when the remote side crashes """ ENDMARK = -1 def __init__(self, nodemanager, gateway, config, putevent): self.nodemanager = nodemanager - self.config = config - self.putevent = putevent + self.config = config + self.putevent = putevent self.gateway = gateway self.slaveinput = {} self.channel = install_slave(self) @@ -31,13 +31,13 @@ class TXNode(object): def notify(self, eventname, *args, **kwargs): assert not args self.putevent((eventname, args, kwargs)) - + def callback(self, eventcall): - """ this gets called for each object we receive from - the other side and if the channel closes. + """ this gets called for each object we receive from + the other side and if the channel closes. Note that channel callbacks run in the receiver - thread of execnet gateways - we need to + thread of execnet gateways - we need to avoid raising exceptions or doing heavy work. """ try: @@ -45,11 +45,11 @@ class TXNode(object): err = self.channel._getremoteerror() if not self._down: if not err or isinstance(err, EOFError): - err = "Not properly terminated" # lost connection? + err = "Not properly terminated" # lost connection? self.notify("pytest_testnodedown", node=self, error=err) self._down = True return - eventname, args, kwargs = eventcall + eventname, args, kwargs = eventcall if eventname == "slaveready": self.notify("pytest_testnodeready", node=self) elif eventname == "slavefinished": @@ -57,15 +57,15 @@ class TXNode(object): self.slaveoutput = kwargs['slaveoutput'] error = kwargs['error'] self.notify("pytest_testnodedown", error=error, node=self) - elif eventname in ("pytest_runtest_logreport", + elif eventname in ("pytest_runtest_logreport", "pytest__teardown_final_logerror"): kwargs['report'].node = self self.notify(eventname, **kwargs) else: self.notify(eventname, **kwargs) - except KeyboardInterrupt: + except KeyboardInterrupt: # should not land in receiver-thread - raise + raise except: excinfo = py.code.ExceptionInfo() py.builtin.print_("!" * 20, excinfo) @@ -84,11 +84,11 @@ class TXNode(object): else: self.channel.send(None) -# configuring and setting up slave node +# configuring and setting up slave node def install_slave(node): channel = node.gateway.remote_exec(source=""" - import os, sys - sys.path.insert(0, os.getcwd()) + import os, sys + sys.path.insert(0, os.getcwd()) from xdist.mypickle import PickleChannel from xdist.txnode import SlaveSession channel.send("basicimport") @@ -107,11 +107,11 @@ def install_slave(node): channel.receive() channel = PickleChannel(channel) basetemp = None - config = node.config + config = node.config config.hook.pytest_configure_node(node=node) if node.gateway.spec.popen: popenbase = config.ensuretemp("popen") - basetemp = py.path.local.make_numbered_dir(prefix="slave-", + basetemp = py.path.local.make_numbered_dir(prefix="slave-", keep=0, rootdir=popenbase) basetemp = str(basetemp) channel.send((config, node.slaveinput, basetemp, node.gateway.id)) @@ -147,13 +147,13 @@ class SlaveSession(Session): self.sendevent("slaveready") self.main(None) error = getattr(self, '_slaveerror', None) - self.sendevent("slavefinished", error=error, + self.sendevent("slavefinished", error=error, slaveoutput=self.config.slaveoutput) - + def _mainloop(self, colitems): while 1: task = self.channel.receive() - if task is None: + if task is None: break if isinstance(task, list): for item in task: @@ -165,10 +165,10 @@ class SlaveSession(Session): call = self.runner.CallInfo(item._reraiseunpicklingproblem, when='setup') if call.excinfo: # likely it is not collectable here because of - # platform/import-dependency induced skips + # platform/import-dependency induced skips # we fake a setup-error report with the obtained exception - # and do not care about capturing or non-runner hooks + # and do not care about capturing or non-runner hooks rep = self.runner.pytest_runtest_makereport(item=item, call=call) self.pytest_runtest_logreport(rep) return - item.config.hook.pytest_runtest_protocol(item=item) + item.config.hook.pytest_runtest_protocol(item=item) diff --git a/xdist/util.py b/xdist/util.py index a12575e..5074a33 100644 --- a/xdist/util.py +++ b/xdist/util.py @@ -6,7 +6,7 @@ class StatRecorder: self.statcache = {} self.check() # snapshot state - def fil(self, p): + def fil(self, p): return p.ext in ('.py', '.txt', '.c', '.h') def rec(self, p): return p.check(dotfile=0) @@ -43,7 +43,7 @@ class StatRecorder: pycfile = path + "c" if pycfile.check(): pycfile.remove() - + else: changed = True if statcache: