diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c2836d..6c17dfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,9 @@ repos: +- repo: https://github.com/PyCQA/autoflake + rev: v1.7.6 + hooks: + - id: autoflake + args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"] - repo: https://github.com/psf/black rev: 22.3.0 hooks: diff --git a/changelog/824.trivial.rst b/changelog/824.trivial.rst new file mode 100644 index 0000000..e6c6c85 --- /dev/null +++ b/changelog/824.trivial.rst @@ -0,0 +1 @@ +Replace internal usages of ``py.path.local`` by ``pathlib.Path``. diff --git a/docs/known-limitations.rst b/docs/known-limitations.rst index a081a0b..ba59ae7 100644 --- a/docs/known-limitations.rst +++ b/docs/known-limitations.rst @@ -17,7 +17,8 @@ Example: import pytest - @pytest.mark.parametrize("param", {"a","b"}) + + @pytest.mark.parametrize("param", {"a", "b"}) def test_pytest_parametrize_unordered(param): pass @@ -37,6 +38,7 @@ Some solutions: import pytest + @pytest.mark.parametrize("param", ["a", "b"]) def test_pytest_parametrize_unordered(param): pass @@ -47,6 +49,7 @@ Some solutions: import pytest + @pytest.mark.parametrize("param", sorted({"a", "b"})) def test_pytest_parametrize_unordered(param): pass @@ -54,7 +57,7 @@ Some solutions: Output (stdout and stderr) from workers --------------------------------------- -The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time. +The ``-s``/``--capture=no`` option is meant to disable pytest capture, so users can then see stdout and stderr output in the terminal from tests and application code in real time. However this option does not work with ``pytest-xdist`` because `execnet `__ the underlying library used for communication between master and workers, does not support transferring stdout/stderr from workers. diff --git a/src/xdist/_path.py b/src/xdist/_path.py new file mode 100644 index 0000000..0af3209 --- /dev/null +++ b/src/xdist/_path.py @@ -0,0 +1,19 @@ +import os +from itertools import chain +from pathlib import Path +from typing import Callable, Iterator + + +def visit_path( + path: Path, *, filter: Callable[[Path], bool], recurse: Callable[[Path], bool] +) -> Iterator[Path]: + """ + Implements the interface of ``py.path.local.visit()`` for Path objects, + to simplify porting the code over from ``py.path.local``. + """ + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [x for x in dirnames if recurse(Path(dirpath, x))] + for name in chain(dirnames, filenames): + p = Path(dirpath, name) + if filter(p): + yield p diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index ef4c34f..4b4f406 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -6,11 +6,17 @@ processes) otherwise changes to source code can crash the controlling process which should best never happen. """ -import py +import os +from pathlib import Path +from typing import Dict, Sequence + import pytest import sys import time import execnet +from _pytest._io import TerminalWriter + +from xdist._path import visit_path @pytest.hookimpl @@ -38,9 +44,9 @@ def pytest_cmdline_main(config): return 2 # looponfail only can get stop with ctrl-C anyway -def looponfail_main(config): +def looponfail_main(config: pytest.Config) -> None: remotecontrol = RemoteControl(config) - rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")] + rootdirs = [Path(root) for root in config.getini("looponfailroots")] statrecorder = StatRecorder(rootdirs) try: while 1: @@ -71,7 +77,7 @@ class RemoteControl: def setup(self, out=None): if out is None: - out = py.io.TerminalWriter() + out = TerminalWriter() if hasattr(self, "gateway"): raise ValueError("already have gateway %r" % self.gateway) self.trace("setting up worker session") @@ -129,7 +135,7 @@ class RemoteControl: def repr_pytest_looponfailinfo(failreports, rootdirs): - tr = py.io.TerminalWriter() + tr = TerminalWriter() if failreports: tr.sep("#", "LOOPONFAILING", bold=True) for report in failreports: @@ -225,16 +231,16 @@ class WorkerFailSession: class StatRecorder: - def __init__(self, rootdirlist): + def __init__(self, rootdirlist: Sequence[Path]) -> None: self.rootdirlist = rootdirlist - self.statcache = {} + self.statcache: Dict[Path, os.stat_result] = {} self.check() # snapshot state - def fil(self, p): - return p.check(file=1, dotfile=0) and p.ext != ".pyc" + def fil(self, p: Path) -> bool: + return p.is_file() and not p.name.startswith(".") and p.suffix != ".pyc" - def rec(self, p): - return p.check(dotfile=0) + def rec(self, p: Path) -> bool: + return not p.name.startswith(".") and p.exists() def waitonchange(self, checkinterval=1.0): while 1: @@ -243,34 +249,34 @@ class StatRecorder: return time.sleep(checkinterval) - def check(self, removepycfiles=True): # noqa, too complex + def check(self, removepycfiles: bool = True) -> bool: # noqa, too complex changed = False - statcache = self.statcache - newstat = {} + newstat: Dict[Path, os.stat_result] = {} for rootdir in self.rootdirlist: - for path in rootdir.visit(self.fil, self.rec): - oldstat = statcache.pop(path, None) + for path in visit_path(rootdir, filter=self.fil, recurse=self.rec): + oldstat = self.statcache.pop(path, None) try: - newstat[path] = curstat = path.stat() - except py.error.ENOENT: + curstat = path.stat() + except OSError: if oldstat: changed = True else: - if oldstat: + newstat[path] = curstat + if oldstat is not None: if ( - oldstat.mtime != curstat.mtime - or oldstat.size != curstat.size + oldstat.st_mtime != curstat.st_mtime + or oldstat.st_size != curstat.st_size ): changed = True print("# MODIFIED", path) - if removepycfiles and path.ext == ".py": - pycfile = path + "c" - if pycfile.check(): - pycfile.remove() + if removepycfiles and path.suffix == ".py": + pycfile = path.with_suffix(".pyc") + if pycfile.is_file(): + os.unlink(pycfile) else: changed = True - if statcache: + if self.statcache: changed = True self.statcache = newstat return changed diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index d0448fa..d73d9eb 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -3,7 +3,6 @@ import uuid import sys from pathlib import Path -import py import pytest @@ -165,7 +164,7 @@ def pytest_addoption(parser): "looponfailroots", type="paths" if PYTEST_GTE_7 else "pathlist", help="directories to check for changes", - default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()], + default=[Path.cwd()], ) diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index a8a7c0b..504a7de 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -3,6 +3,8 @@ import os import re import sys import uuid +from pathlib import Path +from typing import List, Union, Sequence, Optional, Any, Tuple, Set import py import pytest @@ -33,7 +35,7 @@ class NodeManager: EXIT_TIMEOUT = 10 DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"] - def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"): + def __init__(self, config, specs=None, defaultchdir="pyexecnetcache") -> None: self.config = config self.trace = self.config.trace.get("nodemanager") self.testrunuid = self.config.getoption("testrunuid") @@ -52,7 +54,7 @@ class NodeManager: self.specs.append(spec) self.roots = self._getrsyncdirs() self.rsyncoptions = self._getrsyncoptions() - self._rsynced_specs = set() + self._rsynced_specs: Set[Tuple[Any, Any]] = set() def rsync_roots(self, gateway): """Rsync the set of roots to the node's gateway cwd.""" @@ -81,7 +83,7 @@ class NodeManager: def _getxspecs(self): return [execnet.XSpec(x) for x in parse_spec_config(self.config)] - def _getrsyncdirs(self): + def _getrsyncdirs(self) -> List[Path]: for spec in self.specs: if not spec.popen or spec.chdir: break @@ -108,8 +110,8 @@ class NodeManager: candidates.extend(rsyncroots) roots = [] for root in candidates: - root = py.path.local(root).realpath() - if not root.check(): + root = Path(root).resolve() + if not root.exists(): raise pytest.UsageError("rsyncdir doesn't exist: {!r}".format(root)) if root not in roots: roots.append(root) @@ -160,18 +162,24 @@ class NodeManager: class HostRSync(execnet.RSync): """RSyncer that filters out common files""" - def __init__(self, sourcedir, *args, **kwargs): - self._synced = {} - ignores = kwargs.pop("ignores", None) or [] - self._ignores = [ - re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores - ] - super().__init__(sourcedir=sourcedir, **kwargs) + PathLike = Union[str, "os.PathLike[str]"] - def filter(self, path): - path = py.path.local(path) + def __init__( + self, + sourcedir: PathLike, + *, + ignores: Optional[Sequence[PathLike]] = None, + **kwargs: object + ) -> None: + if ignores is None: + ignores = [] + self._ignores = [re.compile(fnmatch.translate(os.fspath(x))) for x in ignores] + super().__init__(sourcedir=Path(sourcedir), **kwargs) + + def filter(self, path: PathLike) -> bool: + path = Path(path) for cre in self._ignores: - if cre.match(path.basename) or cre.match(path.strpath): + if cre.match(path.name) or cre.match(str(path)): return False else: return True @@ -187,20 +195,28 @@ class HostRSync(execnet.RSync): print("{}:{} <= {}".format(gateway.spec, remotepath, path)) -def make_reltoroot(roots, args): +def make_reltoroot(roots: Sequence[Path], args: List[str]) -> List[str]: # XXX introduce/use public API for splitting pytest args splitcode = "::" result = [] for arg in args: parts = arg.split(splitcode) - fspath = py.path.local(parts[0]) - if not fspath.exists(): + fspath = Path(parts[0]) + try: + exists = fspath.exists() + except OSError: + exists = False + if not exists: result.append(arg) continue for root in roots: - x = fspath.relto(root) + x: Optional[Path] + try: + x = fspath.relative_to(root) + except ValueError: + x = None if x or fspath == root: - parts[0] = root.basename + "/" + x + parts[0] = root.name + "/" + str(x) break else: raise ValueError("arg {} not relative to an rsync root".format(arg)) diff --git a/testing/test_looponfail.py b/testing/test_looponfail.py index c70b00f..92d4384 100644 --- a/testing/test_looponfail.py +++ b/testing/test_looponfail.py @@ -1,4 +1,6 @@ -import py +import unittest.mock +from typing import List + import pytest import shutil import textwrap @@ -16,7 +18,7 @@ class TestStatRecorder: tmp = tmp_path hello = tmp / "hello.py" hello.touch() - sd = StatRecorder([py.path.local(tmp)]) + sd = StatRecorder([tmp]) changed = sd.check() assert not changed @@ -56,15 +58,12 @@ class TestStatRecorder: tmp = tmp_path tmp.joinpath("dir").mkdir() tmp.joinpath("dir", "hello.py").touch() - sd = StatRecorder([py.path.local(tmp)]) - assert not sd.fil(py.path.local(tmp / "dir")) + sd = StatRecorder([tmp]) + assert not sd.fil(tmp / "dir") - def test_filechange_deletion_race( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_filechange_deletion_race(self, tmp_path: Path) -> None: tmp = tmp_path - pytmp = py.path.local(tmp) - sd = StatRecorder([pytmp]) + sd = StatRecorder([tmp]) changed = sd.check() assert not changed @@ -76,16 +75,20 @@ class TestStatRecorder: p.unlink() # make check()'s visit() call return our just removed # path as if we were in a race condition - monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)]) - - changed = sd.check() + dirname = str(tmp) + dirnames: List[str] = [] + filenames = [str(p)] + with unittest.mock.patch( + "os.walk", return_value=[(dirname, dirnames, filenames)], autospec=True + ): + changed = sd.check() assert changed def test_pycremoval(self, tmp_path: Path) -> None: tmp = tmp_path hello = tmp / "hello.py" hello.touch() - sd = StatRecorder([py.path.local(tmp)]) + sd = StatRecorder([tmp]) changed = sd.check() assert not changed @@ -100,7 +103,7 @@ class TestStatRecorder: self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: tmp = tmp_path - sd = StatRecorder([py.path.local(tmp)]) + sd = StatRecorder([tmp]) ret_values = [True, False] monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop()) diff --git a/testing/test_remote.py b/testing/test_remote.py index 348febc..8a44b73 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -1,5 +1,4 @@ import pprint -import py import pytest import sys import uuid @@ -108,7 +107,7 @@ class TestWorkerInteractor: assert ev.name == "collectionstart" assert not ev.kwargs ev = worker.popevent("collectionfinish") - assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path) + assert ev.kwargs["topdir"] == str(worker.pytester.path) ids = ev.kwargs["ids"] assert len(ids) == 1 worker.sendcommand("runtests", indices=list(range(len(ids)))) diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index b19c524..e83b574 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -1,5 +1,4 @@ import execnet -import py import pytest import shutil import textwrap @@ -7,6 +6,7 @@ import warnings from pathlib import Path from util import generate_warning from xdist import workermanage +from xdist._path import visit_path from xdist.remote import serialize_warning_message from xdist.workermanage import HostRSync, NodeManager, unserialize_warning_message @@ -157,12 +157,9 @@ class TestHRSync: source.joinpath("somedir").mkdir() source.joinpath("somedir", "editfile~").touch() syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES) - files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter)) - assert len(files) == 3 - basenames = [x.basename for x in files] - assert "dir" in basenames - assert "file.txt" in basenames - assert "somedir" in basenames + files = list(visit_path(source, recurse=syncer.filter, filter=syncer.filter)) + names = {x.name for x in files} + assert names == {"dir", "file.txt", "somedir"} def test_hrsync_one_host(self, source: Path, dest: Path) -> None: gw = execnet.makegateway("popen//chdir=%s" % dest)