From f4b86f0127403ba8acb48997d79acb63936cbdb1 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Thu, 5 Dec 2013 15:19:31 +0100 Subject: [PATCH 1/6] add glob support for rsyncignore. add command line option for rsyncignore --- .hgignore | 6 ++++++ CHANGELOG | 10 ++++----- testing/test_plugin.py | 5 +++++ testing/test_slavemanage.py | 6 ++++-- xdist/plugin.py | 8 +++++--- xdist/slavemanage.py | 41 +++++++++++++++++++++++-------------- 6 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.hgignore b/.hgignore index c959df4..6d193f0 100644 --- a/.hgignore +++ b/.hgignore @@ -14,8 +14,14 @@ syntax:glob *.class *.orig +*.sublime-* +.Python + build/ dist/ +include/ +lib/ +bin/ pytest_xdist.egg-info issue/ 3rdparty/ diff --git a/CHANGELOG b/CHANGELOG index 8e5e0e6..dd4ee2e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,11 +1,9 @@ 1.10 unreleased ------------------------- -- ignore pyc files, dot files and directories for changes (editors write tmp/swap - files etc., which also affects mtime of directory) - +- add glob support for rsyncignores, add command line option to pass additional rsyncignores - fix pytest issue382 - produce "pytest_runtest_logstart" event again - in master. Thanks Aron Curzon. + in master. Thanks Aron Curzon. 1.9 ------------------------- @@ -18,14 +16,14 @@ - fix pytest issue41: re-run tests on all file changes, not just randomly select ones like .py/.c. -- fix pytest issue347: slaves running on top of Python3.2 +- fix pytest issue347: slaves running on top of Python3.2 will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency bugs. 1.8 ------------------------- -- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol +- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol interface to perform eager teardowns for test items. 1.7 diff --git a/testing/test_plugin.py b/testing/test_plugin.py index dcff7aa..91eda26 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -46,6 +46,11 @@ class TestDistOptions: assert nm.roots assert testdir.tmpdir in nm.roots + def test_getrsyncignore(self, testdir): + config = testdir.parseconfigure('--rsyncignore=fo*') + nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) + assert 'fo*' in nm.rsyncoptions['ignores'] + def test_getrsyncdirs_with_conftest(self, testdir): p = py.path.local() for bn in 'x y z'.split(): diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index 2ccac26..525f1a9 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -113,7 +113,7 @@ class TestHRSync: source.ensure(".svn", "entries") source.ensure(".somedotfile", "moreentries") source.ensure("somedir", "editfile~") - syncer = HostRSync(source) + syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES) l = list(source.visit(rec=syncer.filter, fil=syncer.filter)) assert len(l) == 3 @@ -197,10 +197,11 @@ class TestNodeManager: dir5 = source.ensure("dir5", "dir6", "bogus") dirf = source.ensure("dir5", "file") dir2.ensure("hello") + dirfoo = source.ensure("foo", "bar") source.join("tox.ini").write(py.std.textwrap.dedent(""" [pytest] rsyncdirs = dir1 dir5 - rsyncignore = dir1/dir2 dir5/dir6 + rsyncignore = dir1/dir2 dir5/dir6 foo* """)) config = testdir.parseconfig(source) nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) @@ -210,6 +211,7 @@ class TestNodeManager: assert not dest.join("dir1", "dir2").check() assert dest.join("dir5","file").check() assert not dest.join("dir6").check() + assert not dest.join('foo').check() def test_optimise_popen(self, testdir, mysetup): source, dest = mysetup.source, mysetup.dest diff --git a/xdist/plugin.py b/xdist/plugin.py index 43bc72a..d48530e 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -1,5 +1,5 @@ -import sys -import py, pytest +import py +import pytest def pytest_addoption(parser): group = parser.getgroup("xdist", "distributed and subprocess testing") @@ -28,8 +28,10 @@ def pytest_addoption(parser): group._addoption('-d', action="store_true", dest="distload", default=False, help="load-balance tests. shortcut for '--dist=load'") - group.addoption('--rsyncdir', action="append", default=[], metavar="dir1", + group.addoption('--rsyncdir', action="append", default=[], metavar="rsyncdirs", help="add directory for rsyncing to remote tx nodes.") + group.addoption('--rsyncignore', action="append", default=[], metavar="rsyncignores", + help="add expression for ignores when rsyncing to remote tx nodes.") parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' ' remote distributed testing.', type="pathlist") diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 6a49108..ce55c23 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -1,5 +1,8 @@ -import py, pytest -import sys, os +import fnmatch +import os + +import py +import pytest import execnet import xdist.remote @@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically class NodeManager(object): EXIT_TIMEOUT = 10 + DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~'] def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): self.config = config self._nodesready = py.std.threading.Event() @@ -23,20 +27,17 @@ class NodeManager(object): self.group.allocate_id(spec) self.specs.append(spec) self.roots = self._getrsyncdirs() + self.rsyncoptions = self._getrsyncoptions() def rsync_roots(self): """ make sure that all remote gateways have the same set of roots in their current directory. """ - options = { - 'ignores': self.config.getini("rsyncignore"), - 'verbose': self.config.option.verbose, - } if self.roots: # send each rsync root for root in self.roots: - self.rsync(root, **options) + self.rsync(root, **self.rsyncoptions) def makegateways(self): assert not list(self.group) @@ -98,6 +99,18 @@ class NodeManager(object): roots.append(root) return roots + def _getrsyncoptions(self): + """Get options to be passed for rsync.""" + ignores = list(self.DEFAULT_IGNORES) + ignores = self.config.option.rsyncignore + ignores += self.config.getini("rsyncignore") + + return { + 'ignores': ignores, + 'verbose': self.config.option.verbose, + } + + def rsync(self, source, notify=None, verbose=False, ignores=None): """ perform rsync to all remote hosts. """ @@ -144,14 +157,12 @@ class HostRSync(execnet.RSync): def filter(self, path): path = py.path.local(path) - if not path.ext in ('.pyc', '.pyo'): - if not path.basename.endswith('~'): - if path.check(dotfile=0): - for x in self._ignores: - if path == x: - break - else: - return True + for x in self._ignores: + x = getattr(x, 'strpath', x) + if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x): + break + else: + return True def add_target_host(self, gateway, finished=None): remotepath = os.path.basename(self._sourcedir) From 027b3f51ed8706a84d6c7aa94bd173487e2a3cd3 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Thu, 5 Dec 2013 15:31:52 +0100 Subject: [PATCH 2/6] change metavars --- xdist/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xdist/plugin.py b/xdist/plugin.py index d48530e..bcfc906 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -28,9 +28,9 @@ def pytest_addoption(parser): group._addoption('-d', action="store_true", dest="distload", default=False, help="load-balance tests. shortcut for '--dist=load'") - group.addoption('--rsyncdir', action="append", default=[], metavar="rsyncdirs", + group.addoption('--rsyncdir', action="append", default=[], metavar="DIR", help="add directory for rsyncing to remote tx nodes.") - group.addoption('--rsyncignore', action="append", default=[], metavar="rsyncignores", + group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB", help="add expression for ignores when rsyncing to remote tx nodes.") parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' From 152f965a7040b96fd50a3b88fb926b41dd2dedf1 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Thu, 5 Dec 2013 16:35:56 +0100 Subject: [PATCH 3/6] fix ignores mixing --- testing/test_slavemanage.py | 3 +++ xdist/slavemanage.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index 525f1a9..edf2abb 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -198,12 +198,14 @@ class TestNodeManager: dirf = source.ensure("dir5", "file") dir2.ensure("hello") dirfoo = source.ensure("foo", "bar") + dirbar = source.ensure("bar", "foo") source.join("tox.ini").write(py.std.textwrap.dedent(""" [pytest] rsyncdirs = dir1 dir5 rsyncignore = dir1/dir2 dir5/dir6 foo* """)) config = testdir.parseconfig(source) + config.option.rsyncignore = ['bar'] nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager.makegateways() nodemanager.rsync_roots() @@ -212,6 +214,7 @@ class TestNodeManager: assert dest.join("dir5","file").check() assert not dest.join("dir6").check() assert not dest.join('foo').check() + assert not dest.join('bar').check() def test_optimise_popen(self, testdir, mysetup): source, dest = mysetup.source, mysetup.dest diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index ce55c23..23cabd9 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -102,7 +102,7 @@ class NodeManager(object): def _getrsyncoptions(self): """Get options to be passed for rsync.""" ignores = list(self.DEFAULT_IGNORES) - ignores = self.config.option.rsyncignore + ignores += self.config.option.rsyncignore ignores += self.config.getini("rsyncignore") return { From a3fac52e85401c57c05ea387eae72a8206558f41 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Thu, 5 Dec 2013 23:22:49 +0100 Subject: [PATCH 4/6] more readability for return --- xdist/slavemanage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 23cabd9..37071d3 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -160,7 +160,7 @@ class HostRSync(execnet.RSync): for x in self._ignores: x = getattr(x, 'strpath', x) if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x): - break + return False else: return True From bcf1c85f4499f85d796463147b010fbbd9fd07c5 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Fri, 6 Dec 2013 10:19:18 +0100 Subject: [PATCH 5/6] docs fixed --- README.txt | 13 +++++++++---- xdist/plugin.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.txt b/README.txt index bab21a5..d23be93 100644 --- a/README.txt +++ b/README.txt @@ -8,10 +8,10 @@ test execution modes: those for a combined test run. This allows to speed up development or to use special resources of `remote machines`_. -* ``--boxed``: (not available on Windows) run each test in a boxed_ +* ``--boxed``: (not available on Windows) run each test in a boxed_ subprocess to survive ``SEGFAULTS`` or otherwise dying processes -* ``--looponfail``: run your tests repeatedly in a subprocess. After each run +* ``--looponfail``: run your tests repeatedly in a subprocess. After each run py.test waits until a file in your project changes and then re-runs the previously failing tests. This is repeated until all tests pass after which again a full run is performed. @@ -33,7 +33,7 @@ Install the plugin with:: easy_install pytest-xdist # or - + pip install pytest-xdist or use the package in develope/in-place mode with @@ -91,7 +91,7 @@ running multiple processes to speed up the test run and use your CPU cores:: py.test -n3 --boxed -this would run 3 testing subprocesses in parallel which each +this would run 3 testing subprocesses in parallel which each create new boxed subprocesses for each test. @@ -122,6 +122,11 @@ py.test references tests as a fully qualified python module path. **You will otherwise get strange errors** during setup of the remote side. +You can specify multiple ``--rsyncignore`` glob-patterns +to be ignored when file are sent to the remote side. +There are also internal ignores: .*, *.pyc, *.pyo, *~ + + Sending tests to remote Socket Servers +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/xdist/plugin.py b/xdist/plugin.py index bcfc906..378e812 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -35,7 +35,7 @@ def pytest_addoption(parser): parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for' ' remote distributed testing.', type="pathlist") - parser.addini('rsyncignore', 'list of (relative) paths to be ignored ' + parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored ' 'for rsyncing.', type="pathlist") parser.addini("looponfailroots", type="pathlist", help="directories to check for changes", default=[py.path.local()]) From 7e066be35ed56b5172755559fa15fe80ff9f90c3 Mon Sep 17 00:00:00 2001 From: Anatoly Bubenkov Date: Fri, 6 Dec 2013 10:21:32 +0100 Subject: [PATCH 6/6] docs fixed --- README.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.txt b/README.txt index d23be93..30f5cda 100644 --- a/README.txt +++ b/README.txt @@ -125,6 +125,8 @@ during setup of the remote side. You can specify multiple ``--rsyncignore`` glob-patterns to be ignored when file are sent to the remote side. There are also internal ignores: .*, *.pyc, *.pyo, *~ +Those you cannot override using rsyncignore command-line or +ini-file option(s). Sending tests to remote Socket Servers