diff --git a/.hgignore b/.hgignore index c959df4..6d193f0 100644 --- a/.hgignore +++ b/.hgignore @@ -14,8 +14,14 @@ syntax:glob *.class *.orig +*.sublime-* +.Python + build/ dist/ +include/ +lib/ +bin/ pytest_xdist.egg-info issue/ 3rdparty/ diff --git a/CHANGELOG b/CHANGELOG index 8e5e0e6..dd4ee2e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,11 +1,9 @@ 1.10 unreleased ------------------------- -- ignore pyc files, dot files and directories for changes (editors write tmp/swap - files etc., which also affects mtime of directory) - +- add glob support for rsyncignores, add command line option to pass additional rsyncignores - fix pytest issue382 - produce "pytest_runtest_logstart" event again - in master. Thanks Aron Curzon. + in master. Thanks Aron Curzon. 1.9 ------------------------- @@ -18,14 +16,14 @@ - fix pytest issue41: re-run tests on all file changes, not just randomly select ones like .py/.c. -- fix pytest issue347: slaves running on top of Python3.2 +- fix pytest issue347: slaves running on top of Python3.2 will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency bugs. 1.8 ------------------------- -- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol +- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol interface to perform eager teardowns for test items. 1.7 diff --git a/README.txt b/README.txt index bab21a5..30f5cda 100644 --- a/README.txt +++ b/README.txt @@ -8,10 +8,10 @@ test execution modes: those for a combined test run. This allows to speed up development or to use special resources of `remote machines`_. -* ``--boxed``: (not available on Windows) run each test in a boxed_ +* ``--boxed``: (not available on Windows) run each test in a boxed_ subprocess to survive ``SEGFAULTS`` or otherwise dying processes -* ``--looponfail``: run your tests repeatedly in a subprocess. After each run +* ``--looponfail``: run your tests repeatedly in a subprocess. After each run py.test waits until a file in your project changes and then re-runs the previously failing tests. This is repeated until all tests pass after which again a full run is performed. @@ -33,7 +33,7 @@ Install the plugin with:: easy_install pytest-xdist # or - + pip install pytest-xdist or use the package in develope/in-place mode with @@ -91,7 +91,7 @@ running multiple processes to speed up the test run and use your CPU cores:: py.test -n3 --boxed -this would run 3 testing subprocesses in parallel which each +this would run 3 testing subprocesses in parallel which each create new boxed subprocesses for each test. @@ -122,6 +122,13 @@ py.test references tests as a fully qualified python module path. **You will otherwise get strange errors** during setup of the remote side. +You can specify multiple ``--rsyncignore`` glob-patterns +to be ignored when file are sent to the remote side. +There are also internal ignores: .*, *.pyc, *.pyo, *~ +Those you cannot override using rsyncignore command-line or +ini-file option(s). + + Sending tests to remote Socket Servers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/testing/test_plugin.py b/testing/test_plugin.py index dcff7aa..91eda26 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -46,6 +46,11 @@ class TestDistOptions: assert nm.roots assert testdir.tmpdir in nm.roots + def test_getrsyncignore(self, testdir): + config = testdir.parseconfigure('--rsyncignore=fo*') + nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) + assert 'fo*' in nm.rsyncoptions['ignores'] + def test_getrsyncdirs_with_conftest(self, testdir): p = py.path.local() for bn in 'x y z'.split(): diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index 2ccac26..edf2abb 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -113,7 +113,7 @@ class TestHRSync: source.ensure(".svn", "entries") source.ensure(".somedotfile", "moreentries") source.ensure("somedir", "editfile~") - syncer = HostRSync(source) + syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES) l = list(source.visit(rec=syncer.filter, fil=syncer.filter)) assert len(l) == 3 @@ -197,12 +197,15 @@ class TestNodeManager: dir5 = source.ensure("dir5", "dir6", "bogus") dirf = source.ensure("dir5", "file") dir2.ensure("hello") + dirfoo = source.ensure("foo", "bar") + dirbar = source.ensure("bar", "foo") source.join("tox.ini").write(py.std.textwrap.dedent(""" [pytest] rsyncdirs = dir1 dir5 - rsyncignore = dir1/dir2 dir5/dir6 + rsyncignore = dir1/dir2 dir5/dir6 foo* """)) config = testdir.parseconfig(source) + config.option.rsyncignore = ['bar'] nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager.makegateways() nodemanager.rsync_roots() @@ -210,6 +213,8 @@ class TestNodeManager: assert not dest.join("dir1", "dir2").check() assert dest.join("dir5","file").check() assert not dest.join("dir6").check() + assert not dest.join('foo').check() + assert not dest.join('bar').check() def test_optimise_popen(self, testdir, mysetup): source, dest = mysetup.source, mysetup.dest diff --git a/xdist/plugin.py b/xdist/plugin.py index 43bc72a..378e812 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -1,5 +1,5 @@ -import sys -import py, pytest +import py +import pytest def pytest_addoption(parser): group = parser.getgroup("xdist", "distributed and subprocess testing") @@ -28,12 +28,14 @@ def pytest_addoption(parser): group._addoption('-d', action="store_true", dest="distload", default=False, help="load-balance tests. shortcut for '--dist=load'") - group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", + group.addoption('--rsyncdir', action="append", default=[], metavar="DIR", help="add directory for rsyncing to remote tx nodes.") + group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB", + help="add expression for ignores when rsyncing to remote tx nodes.") parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' ' remote distributed testing.', type="pathlist") - parser.addini('rsyncignore', 'list of (relative) paths to be ignored ' + parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored ' 'for rsyncing.', type="pathlist") parser.addini("looponfailroots", type="pathlist", help="directories to check for changes", default=[py.path.local()]) diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 6a49108..37071d3 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -1,5 +1,8 @@ -import py, pytest -import sys, os +import fnmatch +import os + +import py +import pytest import execnet import xdist.remote @@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically class NodeManager(object): EXIT_TIMEOUT = 10 + DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~'] def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() @@ -23,20 +27,17 @@ class NodeManager(object): self.group.allocate_id(spec) self.specs.append(spec) self.roots = self._getrsyncdirs() + self.rsyncoptions = self._getrsyncoptions() def rsync_roots(self): """ make sure that all remote gateways have the same set of roots in their current directory. """ - options = { - 'ignores': self.config.getini("rsyncignore"), - 'verbose': self.config.option.verbose, - } if self.roots: # send each rsync root for root in self.roots: - self.rsync(root, **options) + self.rsync(root, **self.rsyncoptions) def makegateways(self): assert not list(self.group) @@ -98,6 +99,18 @@ class NodeManager(object): roots.append(root) return roots + def _getrsyncoptions(self): + """Get options to be passed for rsync.""" + ignores = list(self.DEFAULT_IGNORES) + ignores += self.config.option.rsyncignore + ignores += self.config.getini("rsyncignore") + + return { + 'ignores': ignores, + 'verbose': self.config.option.verbose, + } + + def rsync(self, source, notify=None, verbose=False, ignores=None): """ perform rsync to all remote hosts. """ @@ -144,14 +157,12 @@ class HostRSync(execnet.RSync): def filter(self, path): path = py.path.local(path) - if not path.ext in ('.pyc', '.pyo'): - if not path.basename.endswith('~'): - if path.check(dotfile=0): - for x in self._ignores: - if path == x: - break - else: - return True + for x in self._ignores: + x = getattr(x, 'strpath', x) + if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x): + return False + else: + return True def add_target_host(self, gateway, finished=None): remotepath = os.path.basename(self._sourcedir)