Merged in paylogic/pytest-xdist/rsyncignore-option (pull request #5)

add glob support for rsyncignore. add command line option for rsyncignore

.hgignore

@@ -14,8 +14,14 @@ syntax:glob
 *.class
 *.orig
 
+*.sublime-*
+.Python
+
 build/
 dist/
+include/
+lib/
+bin/
 pytest_xdist.egg-info
 issue/
 3rdparty/

CHANGELOG

@@ -1,11 +1,9 @@
 1.10 unreleased
 -------------------------
 
-- ignore pyc files, dot files and directories for changes (editors write tmp/swap
+- add glob support for rsyncignores, add command line option to pass additional rsyncignores
-  files etc., which also affects mtime of directory)
-
 - fix pytest issue382 - produce "pytest_runtest_logstart" event again
-  in master. Thanks Aron Curzon. 
+  in master. Thanks Aron Curzon.
 
 1.9
 -------------------------
@@ -18,14 +16,14 @@
 - fix pytest issue41: re-run tests on all file changes, not just
   randomly select ones like .py/.c.
 
-- fix pytest issue347: slaves running on top of Python3.2 
+- fix pytest issue347: slaves running on top of Python3.2
   will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
   bugs.
 
 1.8
 -------------------------
 
-- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol 
+- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
   interface to perform eager teardowns for test items.
 
 1.7

README.txt

@@ -8,10 +8,10 @@ test execution modes:
   those for a combined test run.  This allows to speed up
   development or to use special resources of `remote machines`_.
 
-* ``--boxed``: (not available on Windows) run each test in a boxed_ 
+* ``--boxed``: (not available on Windows) run each test in a boxed_
   subprocess to survive ``SEGFAULTS`` or otherwise dying processes
 
-* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run 
+* ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
   py.test waits until a file in your project changes and then re-runs
   the previously failing tests.  This is repeated until all tests pass
   after which again a full run is performed.
@@ -33,7 +33,7 @@ Install the plugin with::
     easy_install pytest-xdist
 
     # or
-    
+
     pip install pytest-xdist
 
 or use the package in develope/in-place mode with
@@ -91,7 +91,7 @@ running multiple processes to speed up the test run and use your CPU cores::
 
     py.test -n3 --boxed
 
-this would run 3 testing subprocesses in parallel which each 
+this would run 3 testing subprocesses in parallel which each
 create new boxed subprocesses for each test.
 
 
@@ -122,6 +122,13 @@ py.test references tests as a fully qualified python
 module path.  **You will otherwise get strange errors**
 during setup of the remote side.
 
+You can specify multiple ``--rsyncignore`` glob-patterns
+to be ignored when file are sent to the remote side.
+There are also internal ignores: .*, *.pyc, *.pyo, *~
+Those you cannot override using rsyncignore command-line or
+ini-file option(s).
+
+
 Sending tests to remote Socket Servers
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 

testing/test_plugin.py

@@ -46,6 +46,11 @@ class TestDistOptions:
         assert nm.roots
         assert testdir.tmpdir in nm.roots
 
+    def test_getrsyncignore(self, testdir):
+        config = testdir.parseconfigure('--rsyncignore=fo*')
+        nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
+        assert 'fo*' in nm.rsyncoptions['ignores']
+
     def test_getrsyncdirs_with_conftest(self, testdir):
         p = py.path.local()
         for bn in 'x y z'.split():

testing/test_slavemanage.py

@@ -113,7 +113,7 @@ class TestHRSync:
         source.ensure(".svn", "entries")
         source.ensure(".somedotfile", "moreentries")
         source.ensure("somedir", "editfile~")
-        syncer = HostRSync(source)
+        syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
         l = list(source.visit(rec=syncer.filter,
                                    fil=syncer.filter))
         assert len(l) == 3
@@ -197,12 +197,15 @@ class TestNodeManager:
         dir5 = source.ensure("dir5", "dir6", "bogus")
         dirf = source.ensure("dir5", "file")
         dir2.ensure("hello")
+        dirfoo = source.ensure("foo", "bar")
+        dirbar = source.ensure("bar", "foo")
         source.join("tox.ini").write(py.std.textwrap.dedent("""
             [pytest]
             rsyncdirs = dir1 dir5
-            rsyncignore = dir1/dir2 dir5/dir6
+            rsyncignore = dir1/dir2 dir5/dir6 foo*
         """))
         config = testdir.parseconfig(source)
+        config.option.rsyncignore = ['bar']
         nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
         nodemanager.makegateways()
         nodemanager.rsync_roots()
@@ -210,6 +213,8 @@ class TestNodeManager:
         assert not dest.join("dir1", "dir2").check()
         assert dest.join("dir5","file").check()
         assert not dest.join("dir6").check()
+        assert not dest.join('foo').check()
+        assert not dest.join('bar').check()
 
     def test_optimise_popen(self, testdir, mysetup):
         source, dest = mysetup.source, mysetup.dest

xdist/plugin.py

@@ -1,5 +1,5 @@
-import sys
+import py
-import py, pytest
+import pytest
 
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
@@ -28,12 +28,14 @@ def pytest_addoption(parser):
     group._addoption('-d',
            action="store_true", dest="distload", default=False,
            help="load-balance tests.  shortcut for '--dist=load'")
-    group.addoption('--rsyncdir', action="append", default=[], metavar="dir1",
+    group.addoption('--rsyncdir', action="append", default=[], metavar="DIR",
            help="add directory for rsyncing to remote tx nodes.")
+    group.addoption('--rsyncignore', action="append", default=[], metavar="GLOB",
+           help="add expression for ignores when rsyncing to remote tx nodes.")
 
     parser.addini('rsyncdirs', 'list of (relative) paths to be rsynced for'
          ' remote distributed testing.', type="pathlist")
-    parser.addini('rsyncignore', 'list of (relative) paths to be ignored '
+    parser.addini('rsyncignore', 'list of (relative) glob-style paths to be ignored '
          'for rsyncing.', type="pathlist")
     parser.addini("looponfailroots", type="pathlist",
         help="directories to check for changes", default=[py.path.local()])

xdist/slavemanage.py

@@ -1,5 +1,8 @@
-import py, pytest
+import fnmatch
-import sys, os
+import os
+
+import py
+import pytest
 import execnet
 import xdist.remote
 
@@ -7,6 +10,7 @@ from _pytest import runner # XXX load dynamically
 
 class NodeManager(object):
     EXIT_TIMEOUT = 10
+    DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
         self._nodesready = py.std.threading.Event()
@@ -23,20 +27,17 @@ class NodeManager(object):
             self.group.allocate_id(spec)
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
+        self.rsyncoptions = self._getrsyncoptions()
 
     def rsync_roots(self):
         """ make sure that all remote gateways
             have the same set of roots in their
             current directory.
         """
-        options = {
-            'ignores': self.config.getini("rsyncignore"),
-            'verbose': self.config.option.verbose,
-        }
         if self.roots:
             # send each rsync root
             for root in self.roots:
-                self.rsync(root, **options)
+                self.rsync(root, **self.rsyncoptions)
 
     def makegateways(self):
         assert not list(self.group)
@@ -98,6 +99,18 @@ class NodeManager(object):
                 roots.append(root)
         return roots
 
+    def _getrsyncoptions(self):
+        """Get options to be passed for rsync."""
+        ignores = list(self.DEFAULT_IGNORES)
+        ignores += self.config.option.rsyncignore
+        ignores += self.config.getini("rsyncignore")
+
+        return {
+            'ignores': ignores,
+            'verbose': self.config.option.verbose,
+        }
+
+
     def rsync(self, source, notify=None, verbose=False, ignores=None):
         """ perform rsync to all remote hosts.
         """
@@ -144,14 +157,12 @@ class HostRSync(execnet.RSync):
 
     def filter(self, path):
         path = py.path.local(path)
-        if not path.ext in ('.pyc', '.pyo'):
+        for x in self._ignores:
-            if not path.basename.endswith('~'):
+            x = getattr(x, 'strpath', x)
-                if path.check(dotfile=0):
+            if fnmatch.fnmatch(path.basename, x) or fnmatch.fnmatch(path.strpath, x):
-                    for x in self._ignores:
+                return False
-                        if path == x:
+        else:
-                            break
+            return True
-                    else:
-                        return True
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)