IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #723 from bluetech/modernize

Browse Source 
Modernize packaging, require pytest 6.2, prepare for pytest 7, other changes
This commit is contained in:
Ran Benita
2021-11-01 09:19:04 +02:00
committed by GitHub
parent b2f3d08106 ae58690f3e
commit 827909b4e1
21 changed files with 895 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.pre-commit-config.yaml
View File

@@ -4,7 +4,6 @@ repos:
hooks: hooks:
- id: black - id: black
args: [--safe, --quiet, --target-version, py35] args: [--safe, --quiet, --target-version, py35]
language_version: python3.7
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1 rev: v4.0.1
hooks: hooks:
@@ -29,4 +28,12 @@ repos:
files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$ files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
language: python language: python
additional_dependencies: [pygments, restructuredtext_lint] additional_dependencies: [pygments, restructuredtext_lint]
language_version: python3.7 - repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.910-1
hooks:
- id: mypy
files: ^(src/|testing/)
args: []
additional_dependencies:
- pytest>=6.2.0
- py>=1.10.0

1
changelog/719.trivial.rst Normal file
View File

@@ -0,0 +1 @@
Use up-to-date ``setup.cfg``/``pyproject.toml`` packaging setup.

1
changelog/720.trivial.rst Normal file
View File

@@ -0,0 +1 @@
Require pytest>=6.2.0.

1
changelog/721.trivial.rst Normal file
View File

@@ -0,0 +1 @@
Started using type annotations and mypy checking internally. The types are incomplete and not published.

1
changelog/722.feature.rst Normal file
View File

@@ -0,0 +1 @@
Full compatibility with pytest 7 - no deprecation warnings or use of legacy features.

12
pyproject.toml
View File

@@ -1,3 +1,15 @@
[build-system]
requires = [
# sync with setup.py until we discard non-pep-517/518
"setuptools>=45.0",
"setuptools-scm[toml]>=6.2.3",
"wheel",
]
build-backend = "setuptools.build_meta"
[tool.setuptools_scm]
write_to = "src/xdist/_version.py"
[tool.towncrier] [tool.towncrier]
package = "xdist" package = "xdist"
filename = "CHANGELOG.rst" filename = "CHANGELOG.rst"

71
setup.cfg
View File

@@ -1,5 +1,76 @@
[metadata] [metadata]
name = pytest-xdist
description = pytest xdist plugin for distributed testing and loop-on-failing modes
long_description = file: README.rst
license = MIT
author = holger krekel and contributors
author_email = pytest-dev@python.org,holger@merlinux.eu
url = https://github.com/pytest-dev/pytest-xdist
platforms =
linux
osx
win32
classifiers =
Development Status :: 5 - Production/Stable
Framework :: Pytest
Intended Audience :: Developers
License :: OSI Approved :: MIT License
Operating System :: POSIX
Operating System :: Microsoft :: Windows
Operating System :: MacOS :: MacOS X
Topic :: Software Development :: Testing
Topic :: Software Development :: Quality Assurance
Topic :: Utilities
Programming Language :: Python
Programming Language :: Python :: 3
Programming Language :: Python :: 3 :: Only
Programming Language :: Python :: 3.6
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
Programming Language :: Python :: 3.10
license_file = LICENSE license_file = LICENSE
[options]
packages = find:
package_dir = =src
zip_safe = False
python_requires = >=3.6
install_requires =
execnet>=1.1
pytest>=6.2.0
pytest-forked
setup_requires = setuptools_scm>=6.0
[options.packages.find]
where = src
[options.entry_points]
pytest11 =
xdist = xdist.plugin
xdist.looponfail = xdist.looponfail
[options.extras_require]
testing =
filelock
psutil = psutil>=3.0
setproctitle = setproctitle
[flake8] [flake8]
max-line-length = 100 max-line-length = 100
[mypy]
mypy_path = src
# TODO: Enable this & fix errors.
# check_untyped_defs = True
disallow_any_generics = True
ignore_missing_imports = True
no_implicit_optional = True
show_error_codes = True
strict_equality = True
warn_redundant_casts = True
warn_return_any = True
warn_unreachable = True
warn_unused_configs = True
# TODO: Enable this & fix errors.
# no_implicit_reexport = True

55
setup.py
View File

@@ -1,53 +1,4 @@
from setuptools import setup, find_packages from setuptools import setup
install_requires = ["execnet>=1.1", "pytest>=6.0.0", "pytest-forked"] if __name__ == "__main__":
setup()
with open("README.rst") as f:
long_description = f.read()
setup(
name="pytest-xdist",
use_scm_version={"write_to": "src/xdist/_version.py"},
description="pytest xdist plugin for distributed testing and loop-on-failing modes",
long_description=long_description,
license="MIT",
author="holger krekel and contributors",
author_email="pytest-dev@python.org,holger@merlinux.eu",
url="https://github.com/pytest-dev/pytest-xdist",
platforms=["linux", "osx", "win32"],
packages=find_packages(where="src"),
package_dir={"": "src"},
extras_require={
"testing": ["filelock"],
"psutil": ["psutil>=3.0"],
"setproctitle": ["setproctitle"],
},
entry_points={
"pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
},
zip_safe=False,
python_requires=">=3.6",
install_requires=install_requires,
setup_requires=["setuptools_scm"],
classifiers=[
"Development Status :: 5 - Production/Stable",
"Framework :: Pytest",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Operating System :: POSIX",
"Operating System :: Microsoft :: Windows",
"Operating System :: MacOS :: MacOS X",
"Topic :: Software Development :: Testing",
"Topic :: Software Development :: Quality Assurance",
"Topic :: Utilities",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
],
)

2
src/xdist/looponfail.py
View File

@@ -38,7 +38,7 @@ def pytest_cmdline_main(config):
def looponfail_main(config): def looponfail_main(config):
remotecontrol = RemoteControl(config) remotecontrol = RemoteControl(config)
rootdirs = config.getini("looponfailroots") rootdirs = [py.path.local(root) for root in config.getini("looponfailroots")]
statrecorder = StatRecorder(rootdirs) statrecorder = StatRecorder(rootdirs)
try: try:
while 1: while 1:

14
src/xdist/plugin.py
View File

@@ -1,10 +1,14 @@
import os import os
import uuid import uuid
import sys import sys
from pathlib import Path
import py import py
import pytest import pytest
PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0) # type: ignore[attr-defined]
_sys_path = list(sys.path) # freeze a copy of sys.path at interpreter startup _sys_path = list(sys.path) # freeze a copy of sys.path at interpreter startup
@@ -147,18 +151,18 @@ def pytest_addoption(parser):
parser.addini( parser.addini(
"rsyncdirs", "rsyncdirs",
"list of (relative) paths to be rsynced for remote distributed testing.", "list of (relative) paths to be rsynced for remote distributed testing.",
type="pathlist", type="paths" if PYTEST_GTE_7 else "pathlist",
) )
parser.addini( parser.addini(
"rsyncignore", "rsyncignore",
"list of (relative) glob-style paths to be ignored for rsyncing.", "list of (relative) glob-style paths to be ignored for rsyncing.",
type="pathlist", type="paths" if PYTEST_GTE_7 else "pathlist",
) )
parser.addini( parser.addini(
"looponfailroots", "looponfailroots",
type="pathlist", type="paths" if PYTEST_GTE_7 else "pathlist",
help="directories to check for changes", help="directories to check for changes",
default=[py.path.local()], default=[Path.cwd() if PYTEST_GTE_7 else py.path.local()],
) )
@@ -250,7 +254,7 @@ def is_xdist_controller(request_or_session) -> bool:
is_xdist_master = is_xdist_controller is_xdist_master = is_xdist_controller
def get_xdist_worker_id(request_or_session) -> str: def get_xdist_worker_id(request_or_session):
"""Return the id of the current worker ('gw0', 'gw1', etc) or 'master' """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
if running on the controller node. if running on the controller node.

10
src/xdist/remote.py
View File

@@ -236,8 +236,8 @@ def setup_config(config, basetemp):
if __name__ == "__channelexec__": if __name__ == "__channelexec__":
channel = channel # noqa channel = channel # type: ignore[name-defined] # noqa: F821
workerinput, args, option_dict, change_sys_path = channel.receive() workerinput, args, option_dict, change_sys_path = channel.receive() # type: ignore[name-defined]
if change_sys_path is None: if change_sys_path is None:
importpath = os.getcwd() importpath = os.getcwd()
@@ -260,7 +260,7 @@ if __name__ == "__channelexec__":
setup_config(config, option_dict.get("basetemp")) setup_config(config, option_dict.get("basetemp"))
config._parser.prog = os.path.basename(workerinput["mainargv"][0]) config._parser.prog = os.path.basename(workerinput["mainargv"][0])
config.workerinput = workerinput config.workerinput = workerinput # type: ignore[attr-defined]
config.workeroutput = {} config.workeroutput = {} # type: ignore[attr-defined]
interactor = WorkerInteractor(config, channel) interactor = WorkerInteractor(config, channel) # type: ignore[name-defined]
config.hook.pytest_cmdline_main(config=config) config.hook.pytest_cmdline_main(config=config)

14
src/xdist/workermanage.py
View File

@@ -118,8 +118,8 @@ class NodeManager:
def _getrsyncoptions(self): def _getrsyncoptions(self):
"""Get options to be passed for rsync.""" """Get options to be passed for rsync."""
ignores = list(self.DEFAULT_IGNORES) ignores = list(self.DEFAULT_IGNORES)
ignores += self.config.option.rsyncignore ignores += [str(path) for path in self.config.option.rsyncignore]
ignores += self.config.getini("rsyncignore") ignores += [str(path) for path in self.config.getini("rsyncignore")]
return { return {
"ignores": ignores, "ignores": ignores,
@@ -254,9 +254,9 @@ class WorkerController:
args = make_reltoroot(self.nodemanager.roots, args) args = make_reltoroot(self.nodemanager.roots, args)
if spec.popen: if spec.popen:
name = "popen-%s" % self.gateway.id name = "popen-%s" % self.gateway.id
if hasattr(self.config, "_tmpdirhandler"): if hasattr(self.config, "_tmp_path_factory"):
basetemp = self.config._tmpdirhandler.getbasetemp() basetemp = self.config._tmp_path_factory.getbasetemp()
option_dict["basetemp"] = str(basetemp.join(name)) option_dict["basetemp"] = str(basetemp / name)
self.config.hook.pytest_configure_node(node=self) self.config.hook.pytest_configure_node(node=self)
remote_module = self.config.hook.pytest_xdist_getremotemodule() remote_module = self.config.hook.pytest_xdist_getremotemodule()
@@ -423,9 +423,9 @@ def unserialize_warning_message(data):
kwargs = {"message": message, "category": category} kwargs = {"message": message, "category": category}
# access private _WARNING_DETAILS because the attributes vary between Python versions # access private _WARNING_DETAILS because the attributes vary between Python versions
for attr_name in warnings.WarningMessage._WARNING_DETAILS: for attr_name in warnings.WarningMessage._WARNING_DETAILS: # type: ignore[attr-defined]
if attr_name in ("message", "category"): if attr_name in ("message", "category"):
continue continue
kwargs[attr_name] = data[attr_name] kwargs[attr_name] = data[attr_name]
return warnings.WarningMessage(**kwargs) return warnings.WarningMessage(**kwargs) # type: ignore[arg-type]

489
testing/acceptance_test.py
View File

File diff suppressed because it is too large Load Diff

25
testing/conftest.py
View File

@@ -1,12 +1,13 @@
import py
import pytest
import execnet import execnet
import pytest
import shutil
from typing import List
pytest_plugins = "pytester" pytest_plugins = "pytester"
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def _divert_atexit(request, monkeypatch): def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
import atexit import atexit
finalizers = [] finalizers = []
@@ -23,7 +24,7 @@ def _divert_atexit(request, monkeypatch):
func(*args, **kwargs) func(*args, **kwargs)
def pytest_addoption(parser): def pytest_addoption(parser) -> None:
parser.addoption( parser.addoption(
"--gx", "--gx",
action="append", action="append",
@@ -33,28 +34,28 @@ def pytest_addoption(parser):
@pytest.fixture @pytest.fixture
def specssh(request): def specssh(request) -> str:
return getspecssh(request.config) return getspecssh(request.config)
# configuration information for tests # configuration information for tests
def getgspecs(config): def getgspecs(config) -> List[execnet.XSpec]:
return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")] return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
def getspecssh(config): def getspecssh(config) -> str: # type: ignore[return]
xspecs = getgspecs(config) xspecs = getgspecs(config)
for spec in xspecs: for spec in xspecs:
if spec.ssh: if spec.ssh:
if not py.path.local.sysfind("ssh"): if not shutil.which("ssh"):
py.test.skip("command not found: ssh") pytest.skip("command not found: ssh")
return str(spec) return str(spec)
py.test.skip("need '--gx ssh=...'") pytest.skip("need '--gx ssh=...'")
def getsocketspec(config): def getsocketspec(config) -> execnet.XSpec:
xspecs = getgspecs(config) xspecs = getgspecs(config)
for spec in xspecs: for spec in xspecs:
if spec.socket: if spec.socket:
return spec return spec
py.test.skip("need '--gx socket=...'") pytest.skip("need '--gx socket=...'")

91
testing/test_dsession.py
View File

@@ -1,59 +1,44 @@
from xdist.dsession import DSession, get_default_max_worker_restart from xdist.dsession import DSession, get_default_max_worker_restart
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.scheduler import EachScheduling, LoadScheduling from xdist.scheduler import EachScheduling, LoadScheduling
from typing import Optional
import py
import pytest import pytest
import execnet import execnet
XSpec = execnet.XSpec
def run(item, node, excinfo=None):
runner = item.config.pluginmanager.getplugin("runner")
rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call")
rep.node = node
return rep
class MockGateway: class MockGateway:
_count = 0 def __init__(self) -> None:
self._count = 0
def __init__(self):
self.id = str(self._count) self.id = str(self._count)
self._count += 1 self._count += 1
class MockNode: class MockNode:
def __init__(self): def __init__(self) -> None:
self.sent = [] self.sent = [] # type: ignore[var-annotated]
self.gateway = MockGateway() self.gateway = MockGateway()
self._shutdown = False self._shutdown = False
def send_runtest_some(self, indices): def send_runtest_some(self, indices) -> None:
self.sent.extend(indices) self.sent.extend(indices)
def send_runtest_all(self): def send_runtest_all(self) -> None:
self.sent.append("ALL") self.sent.append("ALL")
def shutdown(self): def shutdown(self) -> None:
self._shutdown = True self._shutdown = True
@property @property
def shutting_down(self): def shutting_down(self) -> bool:
return self._shutdown return self._shutdown
def dumpqueue(queue):
while queue.qsize():
print(queue.get())
class TestEachScheduling: class TestEachScheduling:
def test_schedule_load_simple(self, testdir): def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
node1 = MockNode() node1 = MockNode()
node2 = MockNode() node2 = MockNode()
config = testdir.parseconfig("--tx=2*popen") config = pytester.parseconfig("--tx=2*popen")
sched = EachScheduling(config) sched = EachScheduling(config)
sched.add_node(node1) sched.add_node(node1)
sched.add_node(node2) sched.add_node(node2)
@@ -74,9 +59,9 @@ class TestEachScheduling:
sched.mark_test_complete(node2, 0) sched.mark_test_complete(node2, 0)
assert sched.tests_finished assert sched.tests_finished
def test_schedule_remove_node(self, testdir): def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
node1 = MockNode() node1 = MockNode()
config = testdir.parseconfig("--tx=popen") config = pytester.parseconfig("--tx=popen")
sched = EachScheduling(config) sched = EachScheduling(config)
sched.add_node(node1) sched.add_node(node1)
collection = ["a.py::test_1"] collection = ["a.py::test_1"]
@@ -93,8 +78,8 @@ class TestEachScheduling:
class TestLoadScheduling: class TestLoadScheduling:
def test_schedule_load_simple(self, testdir): def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfig("--tx=2*popen") config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.add_node(MockNode()) sched.add_node(MockNode())
sched.add_node(MockNode()) sched.add_node(MockNode())
@@ -117,8 +102,8 @@ class TestLoadScheduling:
sched.mark_test_complete(node1, node1.sent[0]) sched.mark_test_complete(node1, node1.sent[0])
assert sched.tests_finished assert sched.tests_finished
def test_schedule_batch_size(self, testdir): def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfig("--tx=2*popen") config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.add_node(MockNode()) sched.add_node(MockNode())
sched.add_node(MockNode()) sched.add_node(MockNode())
@@ -144,8 +129,8 @@ class TestLoadScheduling:
assert node1.sent == [0, 2, 4, 5] assert node1.sent == [0, 2, 4, 5]
assert not sched.pending assert not sched.pending
def test_schedule_fewer_tests_than_nodes(self, testdir): def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfig("--tx=2*popen") config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.add_node(MockNode()) sched.add_node(MockNode())
sched.add_node(MockNode()) sched.add_node(MockNode())
@@ -164,8 +149,10 @@ class TestLoadScheduling:
assert sent3 == [] assert sent3 == []
assert not sched.pending assert not sched.pending
def test_schedule_fewer_than_two_tests_per_node(self, testdir): def test_schedule_fewer_than_two_tests_per_node(
config = testdir.parseconfig("--tx=2*popen") self, pytester: pytest.Pytester
) -> None:
config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.add_node(MockNode()) sched.add_node(MockNode())
sched.add_node(MockNode()) sched.add_node(MockNode())
@@ -184,9 +171,9 @@ class TestLoadScheduling:
assert sent3 == [2] assert sent3 == [2]
assert not sched.pending assert not sched.pending
def test_add_remove_node(self, testdir): def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
node = MockNode() node = MockNode()
config = testdir.parseconfig("--tx=popen") config = pytester.parseconfig("--tx=popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.add_node(node) sched.add_node(node)
collection = ["test_file.py::test_func"] collection = ["test_file.py::test_func"]
@@ -197,7 +184,7 @@ class TestLoadScheduling:
crashitem = sched.remove_node(node) crashitem = sched.remove_node(node)
assert crashitem == collection[0] assert crashitem == collection[0]
def test_different_tests_collected(self, testdir): def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
""" """
Test that LoadScheduling is reporting collection errors when Test that LoadScheduling is reporting collection errors when
different test ids are collected by workers. different test ids are collected by workers.
@@ -215,7 +202,7 @@ class TestLoadScheduling:
self.reports.append(report) self.reports.append(report)
collect_hook = CollectHook() collect_hook = CollectHook()
config = testdir.parseconfig("--tx=2*popen") config = pytester.parseconfig("--tx=2*popen")
config.pluginmanager.register(collect_hook, "collect_hook") config.pluginmanager.register(collect_hook, "collect_hook")
node1 = MockNode() node1 = MockNode()
node2 = MockNode() node2 = MockNode()
@@ -231,9 +218,9 @@ class TestLoadScheduling:
class TestDistReporter: class TestDistReporter:
@py.test.mark.xfail @pytest.mark.xfail
def test_rsync_printing(self, testdir, linecomp): def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
config = testdir.parseconfig() config = pytester.parseconfig()
from _pytest.pytest_terminal import TerminalReporter from _pytest.pytest_terminal import TerminalReporter
rep = TerminalReporter(config, file=linecomp.stringio) rep = TerminalReporter(config, file=linecomp.stringio)
@@ -258,21 +245,21 @@ class TestDistReporter:
# linecomp.assert_contains_lines([ # linecomp.assert_contains_lines([
# "*X1*popen*xyz*2.5*" # "*X1*popen*xyz*2.5*"
# ]) # ])
dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2]) dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2]) # type: ignore[attr-defined]
linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"]) linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
def test_report_collection_diff_equal(): def test_report_collection_diff_equal() -> None:
"""Test reporting of equal collections.""" """Test reporting of equal collections."""
from_collection = to_collection = ["aaa", "bbb", "ccc"] from_collection = to_collection = ["aaa", "bbb", "ccc"]
assert report_collection_diff(from_collection, to_collection, 1, 2) is None assert report_collection_diff(from_collection, to_collection, 1, 2) is None
def test_default_max_worker_restart(): def test_default_max_worker_restart() -> None:
class config: class config:
class option: class option:
maxworkerrestart = None maxworkerrestart: Optional[str] = None
numprocesses = 0 numprocesses: int = 0
assert get_default_max_worker_restart(config) is None assert get_default_max_worker_restart(config) is None
@@ -286,7 +273,7 @@ def test_default_max_worker_restart():
assert get_default_max_worker_restart(config) == 0 assert get_default_max_worker_restart(config) == 0
def test_report_collection_diff_different(): def test_report_collection_diff_different() -> None:
"""Test reporting of different collections.""" """Test reporting of different collections."""
from_collection = ["aaa", "bbb", "ccc", "YYY"] from_collection = ["aaa", "bbb", "ccc", "YYY"]
to_collection = ["aZa", "bbb", "XXX", "ccc"] to_collection = ["aZa", "bbb", "XXX", "ccc"]
@@ -311,8 +298,8 @@ def test_report_collection_diff_different():
@pytest.mark.xfail(reason="duplicate test ids not supported yet") @pytest.mark.xfail(reason="duplicate test ids not supported yet")
def test_pytest_issue419(testdir): def test_pytest_issue419(pytester: pytest.Pytester) -> None:
testdir.makepyfile( pytester.makepyfile(
""" """
import pytest import pytest
@@ -321,6 +308,6 @@ def test_pytest_issue419(testdir):
pass pass
""" """
) )
reprec = testdir.inline_run("-n1") reprec = pytester.inline_run("-n1")
reprec.assertoutcome(passed=2) reprec.assertoutcome(passed=2)
assert 0 assert 0

345
testing/test_looponfail.py
View File

@@ -1,90 +1,106 @@
import py import py
import pytest import pytest
from pkg_resources import parse_version import shutil
import textwrap
from pathlib import Path
from xdist.looponfail import RemoteControl from xdist.looponfail import RemoteControl
from xdist.looponfail import StatRecorder from xdist.looponfail import StatRecorder
PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0) # type: ignore[attr-defined]
class TestStatRecorder: class TestStatRecorder:
def test_filechange(self, tmpdir): def test_filechange(self, tmp_path: Path) -> None:
tmp = tmpdir tmp = tmp_path
hello = tmp.ensure("hello.py") hello = tmp / "hello.py"
sd = StatRecorder([tmp]) hello.touch()
sd = StatRecorder([py.path.local(tmp)])
changed = sd.check() changed = sd.check()
assert not changed assert not changed
hello.write("world") hello.write_text("world")
changed = sd.check() changed = sd.check()
assert changed assert changed
(hello + "c").write("hello") hello.with_suffix(".pyc").write_text("hello")
changed = sd.check() changed = sd.check()
assert not changed assert not changed
p = tmp.ensure("new.py") p = tmp / "new.py"
p.touch()
changed = sd.check() changed = sd.check()
assert changed assert changed
p.remove() p.unlink()
changed = sd.check() changed = sd.check()
assert changed assert changed
tmp.join("a", "b", "c.py").ensure() tmp.joinpath("a", "b").mkdir(parents=True)
tmp.joinpath("a", "b", "c.py").touch()
changed = sd.check() changed = sd.check()
assert changed assert changed
tmp.join("a", "c.txt").ensure() tmp.joinpath("a", "c.txt").touch()
changed = sd.check() changed = sd.check()
assert changed assert changed
changed = sd.check() changed = sd.check()
assert not changed assert not changed
tmp.join("a").remove() shutil.rmtree(str(tmp.joinpath("a")))
changed = sd.check() changed = sd.check()
assert changed assert changed
def test_dirchange(self, tmpdir): def test_dirchange(self, tmp_path: Path) -> None:
tmp = tmpdir tmp = tmp_path
tmp.ensure("dir", "hello.py") tmp.joinpath("dir").mkdir()
sd = StatRecorder([tmp]) tmp.joinpath("dir", "hello.py").touch()
assert not sd.fil(tmp.join("dir")) sd = StatRecorder([py.path.local(tmp)])
assert not sd.fil(py.path.local(tmp / "dir"))
def test_filechange_deletion_race(self, tmpdir, monkeypatch): def test_filechange_deletion_race(
tmp = tmpdir self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
sd = StatRecorder([tmp]) ) -> None:
tmp = tmp_path
pytmp = py.path.local(tmp)
sd = StatRecorder([pytmp])
changed = sd.check() changed = sd.check()
assert not changed assert not changed
p = tmp.ensure("new.py") p = tmp.joinpath("new.py")
p.touch()
changed = sd.check() changed = sd.check()
assert changed assert changed
p.remove() p.unlink()
# make check()'s visit() call return our just removed # make check()'s visit() call return our just removed
# path as if we were in a race condition # path as if we were in a race condition
monkeypatch.setattr(tmp, "visit", lambda *args: [p]) monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
changed = sd.check() changed = sd.check()
assert changed assert changed
def test_pycremoval(self, tmpdir): def test_pycremoval(self, tmp_path: Path) -> None:
tmp = tmpdir tmp = tmp_path
hello = tmp.ensure("hello.py") hello = tmp / "hello.py"
sd = StatRecorder([tmp]) hello.touch()
sd = StatRecorder([py.path.local(tmp)])
changed = sd.check() changed = sd.check()
assert not changed assert not changed
pycfile = hello + "c" pycfile = hello.with_suffix(".pyc")
pycfile.ensure() pycfile.touch()
hello.write("world") hello.write_text("world")
changed = sd.check() changed = sd.check()
assert changed assert changed
assert not pycfile.check() assert not pycfile.exists()
def test_waitonchange(self, tmpdir, monkeypatch): def test_waitonchange(
tmp = tmpdir self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
sd = StatRecorder([tmp]) ) -> None:
tmp = tmp_path
sd = StatRecorder([py.path.local(tmp)])
ret_values = [True, False] ret_values = [True, False]
monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop()) monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -93,63 +109,74 @@ class TestStatRecorder:
class TestRemoteControl: class TestRemoteControl:
def test_nofailures(self, testdir): def test_nofailures(self, pytester: pytest.Pytester) -> None:
item = testdir.getitem("def test_func(): pass\n") item = pytester.getitem("def test_func(): pass\n")
control = RemoteControl(item.config) control = RemoteControl(item.config)
control.setup() control.setup()
topdir, failures = control.runsession()[:2] topdir, failures = control.runsession()[:2]
assert not failures assert not failures
def test_failures_somewhere(self, testdir): def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
item = testdir.getitem("def test_func():\n assert 0\n") item = pytester.getitem("def test_func():\n assert 0\n")
control = RemoteControl(item.config) control = RemoteControl(item.config)
control.setup() control.setup()
failures = control.runsession() failures = control.runsession()
assert failures assert failures
control.setup() control.setup()
item.fspath.write("def test_func():\n assert 1\n") item_path = item.path if PYTEST_GTE_7 else Path(item.fspath) # type: ignore[attr-defined]
removepyc(item.fspath) item_path.write_text("def test_func():\n assert 1\n")
removepyc(item_path)
topdir, failures = control.runsession()[:2] topdir, failures = control.runsession()[:2]
assert not failures assert not failures
def test_failure_change(self, testdir): def test_failure_change(self, pytester: pytest.Pytester) -> None:
modcol = testdir.getitem( modcol = pytester.getitem(
""" textwrap.dedent(
def test_func(): """
assert 0 def test_func():
""" assert 0
"""
)
) )
control = RemoteControl(modcol.config) control = RemoteControl(modcol.config)
control.loop_once() control.loop_once()
assert control.failures assert control.failures
modcol.fspath.write( modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath) # type: ignore[attr-defined]
py.code.Source( modcol_path.write_text(
textwrap.dedent(
"""
def test_func():
assert 1
def test_new():
assert 0
""" """
def test_func():
assert 1
def test_new():
assert 0
"""
) )
) )
removepyc(modcol.fspath) removepyc(modcol_path)
control.loop_once() control.loop_once()
assert not control.failures assert not control.failures
control.loop_once() control.loop_once()
assert control.failures assert control.failures
assert str(control.failures).find("test_new") != -1 assert str(control.failures).find("test_new") != -1
def test_failure_subdir_no_init(self, testdir): def test_failure_subdir_no_init(
modcol = testdir.getitem( self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
""" ) -> None:
def test_func(): modcol = pytester.getitem(
assert 0 textwrap.dedent(
""" """
def test_func():
assert 0
"""
)
) )
parent = modcol.fspath.dirpath().dirpath() if PYTEST_GTE_7:
parent.chdir() parent = modcol.path.parent.parent # type: ignore[attr-defined]
else:
parent = Path(modcol.fspath.dirpath().dirpath())
monkeypatch.chdir(parent)
modcol.config.args = [ modcol.config.args = [
py.path.local(x).relto(parent) for x in modcol.config.args str(Path(x).relative_to(parent)) for x in modcol.config.args
] ]
control = RemoteControl(modcol.config) control = RemoteControl(modcol.config)
control.loop_once() control.loop_once()
@@ -159,57 +186,63 @@ class TestRemoteControl:
class TestLooponFailing: class TestLooponFailing:
def test_looponfail_from_fail_to_ok(self, testdir): def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
modcol = testdir.getmodulecol( modcol = pytester.getmodulecol(
""" textwrap.dedent(
def test_one(): """
x = 0 def test_one():
assert x == 1 x = 0
def test_two(): assert x == 1
assert 1 def test_two():
""" assert 1
"""
)
) )
remotecontrol = RemoteControl(modcol.config) remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1 assert len(remotecontrol.failures) == 1
modcol.fspath.write( modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
py.code.Source( modcol_path.write_text(
textwrap.dedent(
"""
def test_one():
assert 1
def test_two():
assert 1
""" """
def test_one():
assert 1
def test_two():
assert 1
"""
) )
) )
removepyc(modcol.fspath) removepyc(modcol_path)
remotecontrol.loop_once() remotecontrol.loop_once()
assert not remotecontrol.failures assert not remotecontrol.failures
def test_looponfail_from_one_to_two_tests(self, testdir): def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
modcol = testdir.getmodulecol( modcol = pytester.getmodulecol(
""" textwrap.dedent(
def test_one(): """
assert 0 def test_one():
""" assert 0
"""
)
) )
remotecontrol = RemoteControl(modcol.config) remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1 assert len(remotecontrol.failures) == 1
assert "test_one" in remotecontrol.failures[0] assert "test_one" in remotecontrol.failures[0]
modcol.fspath.write( modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
py.code.Source( modcol_path.write_text(
textwrap.dedent(
"""
def test_one():
assert 1 # passes now
def test_two():
assert 0 # new and fails
""" """
def test_one():
assert 1 # passes now
def test_two():
assert 0 # new and fails
"""
) )
) )
removepyc(modcol.fspath) removepyc(modcol_path)
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 0 assert len(remotecontrol.failures) == 0
remotecontrol.loop_once() remotecontrol.loop_once()
@@ -217,47 +250,49 @@ class TestLooponFailing:
assert "test_one" not in remotecontrol.failures[0] assert "test_one" not in remotecontrol.failures[0]
assert "test_two" in remotecontrol.failures[0] assert "test_two" in remotecontrol.failures[0]
@pytest.mark.xfail( @pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
parse_version(pytest.__version__) >= parse_version("3.1"), def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
reason="broken by pytest 3.1+", modcol = pytester.getmodulecol(
strict=True, textwrap.dedent(
) """
def test_looponfail_removed_test(self, testdir): def test_one():
modcol = testdir.getmodulecol( assert 0
""" def test_two():
def test_one(): assert 0
assert 0 """
def test_two(): )
assert 0
"""
) )
remotecontrol = RemoteControl(modcol.config) remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 2 assert len(remotecontrol.failures) == 2
modcol.fspath.write( modcol.path.write_text(
py.code.Source( textwrap.dedent(
"""
def test_xxx(): # renamed test
assert 0
def test_two():
assert 1 # pass now
""" """
def test_xxx(): # renamed test
assert 0
def test_two():
assert 1 # pass now
"""
) )
) )
removepyc(modcol.fspath) removepyc(modcol.path)
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 0 assert len(remotecontrol.failures) == 0
remotecontrol.loop_once() remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1 assert len(remotecontrol.failures) == 1
def test_looponfail_multiple_errors(self, testdir, monkeypatch): def test_looponfail_multiple_errors(
modcol = testdir.getmodulecol( self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
""" ) -> None:
def test_one(): modcol = pytester.getmodulecol(
assert 0 textwrap.dedent(
""" """
def test_one():
assert 0
"""
)
) )
remotecontrol = RemoteControl(modcol.config) remotecontrol = RemoteControl(modcol.config)
orig_runsession = remotecontrol.runsession orig_runsession = remotecontrol.runsession
@@ -274,55 +309,59 @@ class TestLooponFailing:
class TestFunctional: class TestFunctional:
def test_fail_to_ok(self, testdir): def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
p = testdir.makepyfile( p = pytester.makepyfile(
""" textwrap.dedent(
def test_one(): """
x = 0 def test_one():
assert x == 1 x = 0
""" assert x == 1
"""
)
) )
# p = testdir.mkdir("sub").join(p1.basename) # p = pytester.mkdir("sub").join(p1.basename)
# p1.move(p) # p1.move(p)
child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0) child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
child.expect("def test_one") child.expect("def test_one")
child.expect("x == 1") child.expect("x == 1")
child.expect("1 failed") child.expect("1 failed")
child.expect("### LOOPONFAILING ####") child.expect("### LOOPONFAILING ####")
child.expect("waiting for changes") child.expect("waiting for changes")
p.write( p.write_text(
py.code.Source( textwrap.dedent(
""" """
def test_one(): def test_one():
x = 1 x = 1
assert x == 1 assert x == 1
""" """
) ),
) )
child.expect(".*1 passed.*") child.expect(".*1 passed.*")
child.kill(15) child.kill(15)
def test_xfail_passes(self, testdir): def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
p = testdir.makepyfile( p = pytester.makepyfile(
""" textwrap.dedent(
import py """
@py.test.mark.xfail import pytest
def test_one(): @pytest.mark.xfail
pass def test_one():
""" pass
"""
)
) )
child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0) child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
child.expect("1 xpass") child.expect("1 xpass")
# child.expect("### LOOPONFAILING ####") # child.expect("### LOOPONFAILING ####")
child.expect("waiting for changes") child.expect("waiting for changes")
child.kill(15) child.kill(15)
def removepyc(path): def removepyc(path: Path) -> None:
# XXX damn those pyc files # XXX damn those pyc files
pyc = path + "c" pyc = path.with_suffix(".pyc")
if pyc.check(): if pyc.exists():
pyc.remove() pyc.unlink()
c = path.dirpath("__pycache__") c = path.parent / "__pycache__"
if c.check(): if c.exists():
c.remove() shutil.rmtree(c)

26
testing/test_newhooks.py
View File

@@ -3,8 +3,8 @@ import pytest
class TestHooks: class TestHooks:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def create_test_file(self, testdir): def create_test_file(self, pytester: pytest.Pytester) -> None:
testdir.makepyfile( pytester.makepyfile(
""" """
import os import os
def test_a(): pass def test_a(): pass
@@ -13,11 +13,11 @@ class TestHooks:
""" """
) )
def test_runtest_logreport(self, testdir): def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
"""Test that log reports from pytest_runtest_logreport when running """Test that log reports from pytest_runtest_logreport when running
with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8) with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
""" """
testdir.makeconftest( pytester.makeconftest(
""" """
def pytest_runtest_logreport(report): def pytest_runtest_logreport(report):
if hasattr(report, 'node'): if hasattr(report, 'node'):
@@ -35,7 +35,7 @@ class TestHooks:
% (report.nodeid, report.worker_id, report.testrun_uid)) % (report.nodeid, report.worker_id, report.testrun_uid))
""" """
) )
res = testdir.runpytest("-n1", "-s") res = pytester.runpytest("-n1", "-s")
res.stdout.fnmatch_lines( res.stdout.fnmatch_lines(
[ [
"*HOOK: test_runtest_logreport.py::test_a gw0 *", "*HOOK: test_runtest_logreport.py::test_a gw0 *",
@@ -45,9 +45,9 @@ class TestHooks:
] ]
) )
def test_node_collection_finished(self, testdir): def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
"""Test pytest_xdist_node_collection_finished hook (#8).""" """Test pytest_xdist_node_collection_finished hook (#8)."""
testdir.makeconftest( pytester.makeconftest(
""" """
def pytest_xdist_node_collection_finished(node, ids): def pytest_xdist_node_collection_finished(node, ids):
workerid = node.workerinput['workerid'] workerid = node.workerinput['workerid']
@@ -55,7 +55,7 @@ class TestHooks:
print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids))) print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
""" """
) )
res = testdir.runpytest("-n2", "-s") res = pytester.runpytest("-n2", "-s")
res.stdout.fnmatch_lines_random( res.stdout.fnmatch_lines_random(
["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"] ["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
) )
@@ -64,8 +64,8 @@ class TestHooks:
class TestCrashItem: class TestCrashItem:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def create_test_file(self, testdir): def create_test_file(self, pytester: pytest.Pytester) -> None:
testdir.makepyfile( pytester.makepyfile(
""" """
import os import os
def test_a(): pass def test_a(): pass
@@ -75,9 +75,9 @@ class TestCrashItem:
""" """
) )
def test_handlecrashitem(self, testdir): def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
"""Test pytest_handlecrashitem hook.""" """Test pytest_handlecrashitem hook."""
testdir.makeconftest( pytester.makeconftest(
""" """
test_runs = 0 test_runs = 0
@@ -91,6 +91,6 @@ class TestCrashItem:
print("HOOK: pytest_handlecrashitem") print("HOOK: pytest_handlecrashitem")
""" """
) )
res = testdir.runpytest("-n2", "-s") res = pytester.runpytest("-n2", "-s")
res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"]) res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
res.stdout.fnmatch_lines(["*3 passed*"]) res.stdout.fnmatch_lines(["*3 passed*"])

110
testing/test_plugin.py
View File

@@ -1,44 +1,46 @@
from contextlib import suppress from contextlib import suppress
from pathlib import Path
import py
import execnet import execnet
from xdist.workermanage import NodeManager from xdist.workermanage import NodeManager
import pytest import pytest
def test_dist_incompatibility_messages(testdir): def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
result = testdir.runpytest("--pdb", "--looponfail") result = pytester.runpytest("--pdb", "--looponfail")
assert result.ret != 0 assert result.ret != 0
result = testdir.runpytest("--pdb", "-n", "3") result = pytester.runpytest("--pdb", "-n", "3")
assert result.ret != 0 assert result.ret != 0
assert "incompatible" in result.stderr.str() assert "incompatible" in result.stderr.str()
result = testdir.runpytest("--pdb", "-d", "--tx", "popen") result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
assert result.ret != 0 assert result.ret != 0
assert "incompatible" in result.stderr.str() assert "incompatible" in result.stderr.str()
def test_dist_options(testdir): def test_dist_options(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
config = testdir.parseconfigure("-n 2") config = pytester.parseconfigure("-n 2")
check_options(config) check_options(config)
assert config.option.dist == "load" assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2 assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("--numprocesses", "2") config = pytester.parseconfigure("--numprocesses", "2")
check_options(config) check_options(config)
assert config.option.dist == "load" assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2 assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2") config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
check_options(config) check_options(config)
assert config.option.dist == "load" assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2 assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("-d") config = pytester.parseconfigure("-d")
check_options(config) check_options(config)
assert config.option.dist == "load" assert config.option.dist == "load"
def test_auto_detect_cpus(testdir, monkeypatch): def test_auto_detect_cpus(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
import os import os
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
@@ -56,20 +58,20 @@ def test_auto_detect_cpus(testdir, monkeypatch):
monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99) monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
config = testdir.parseconfigure("-n2") config = pytester.parseconfigure("-n2")
assert config.getoption("numprocesses") == 2 assert config.getoption("numprocesses") == 2
config = testdir.parseconfigure("-nauto") config = pytester.parseconfigure("-nauto")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 99 assert config.getoption("numprocesses") == 99
config = testdir.parseconfigure("-nauto", "--pdb") config = pytester.parseconfigure("-nauto", "--pdb")
check_options(config) check_options(config)
assert config.getoption("usepdb") assert config.getoption("usepdb")
assert config.getoption("numprocesses") == 0 assert config.getoption("numprocesses") == 0
assert config.getoption("dist") == "no" assert config.getoption("dist") == "no"
config = testdir.parseconfigure("-nlogical", "--pdb") config = pytester.parseconfigure("-nlogical", "--pdb")
check_options(config) check_options(config)
assert config.getoption("usepdb") assert config.getoption("usepdb")
assert config.getoption("numprocesses") == 0 assert config.getoption("numprocesses") == 0
@@ -77,91 +79,95 @@ def test_auto_detect_cpus(testdir, monkeypatch):
monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.delattr(os, "sched_getaffinity", raising=False)
monkeypatch.setenv("TRAVIS", "true") monkeypatch.setenv("TRAVIS", "true")
config = testdir.parseconfigure("-nauto") config = pytester.parseconfigure("-nauto")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 2 assert config.getoption("numprocesses") == 2
def test_auto_detect_cpus_psutil(testdir, monkeypatch): def test_auto_detect_cpus_psutil(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
psutil = pytest.importorskip("psutil") psutil = pytest.importorskip("psutil")
monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42) monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
config = testdir.parseconfigure("-nauto") config = pytester.parseconfigure("-nauto")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 42 assert config.getoption("numprocesses") == 42
config = testdir.parseconfigure("-nlogical") config = pytester.parseconfigure("-nlogical")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 84 assert config.getoption("numprocesses") == 84
def test_hook_auto_num_workers(testdir, monkeypatch): def test_hook_auto_num_workers(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
testdir.makeconftest( pytester.makeconftest(
""" """
def pytest_xdist_auto_num_workers(): def pytest_xdist_auto_num_workers():
return 42 return 42
""" """
) )
config = testdir.parseconfigure("-nauto") config = pytester.parseconfigure("-nauto")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 42 assert config.getoption("numprocesses") == 42
config = testdir.parseconfigure("-nlogical") config = pytester.parseconfigure("-nlogical")
check_options(config) check_options(config)
assert config.getoption("numprocesses") == 42 assert config.getoption("numprocesses") == 42
def test_boxed_with_collect_only(testdir): def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
config = testdir.parseconfigure("-n1", "--boxed") config = pytester.parseconfigure("-n1", "--boxed")
check_options(config) check_options(config)
assert config.option.forked assert config.option.forked
config = testdir.parseconfigure("-n1", "--collect-only") config = pytester.parseconfigure("-n1", "--collect-only")
check_options(config) check_options(config)
assert not config.option.forked assert not config.option.forked
config = testdir.parseconfigure("-n1", "--boxed", "--collect-only") config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
check_options(config) check_options(config)
assert config.option.forked assert config.option.forked
def test_dsession_with_collect_only(testdir): def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options from xdist.plugin import pytest_cmdline_main as check_options
from xdist.plugin import pytest_configure as configure from xdist.plugin import pytest_configure as configure
config = testdir.parseconfigure("-n1") config = pytester.parseconfigure("-n1")
check_options(config) check_options(config)
configure(config) configure(config)
assert config.pluginmanager.hasplugin("dsession") assert config.pluginmanager.hasplugin("dsession")
config = testdir.parseconfigure("-n1", "--collect-only") config = pytester.parseconfigure("-n1", "--collect-only")
check_options(config) check_options(config)
configure(config) configure(config)
assert not config.pluginmanager.hasplugin("dsession") assert not config.pluginmanager.hasplugin("dsession")
def test_testrunuid_provided(testdir): def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--testrunuid", "test123", "--tx=popen") config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
nm = NodeManager(config) nm = NodeManager(config)
assert nm.testrunuid == "test123" assert nm.testrunuid == "test123"
def test_testrunuid_generated(testdir): def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--tx=popen") config = pytester.parseconfigure("--tx=popen")
nm = NodeManager(config) nm = NodeManager(config)
assert len(nm.testrunuid) == 32 assert len(nm.testrunuid) == 32
class TestDistOptions: class TestDistOptions:
def test_getxspecs(self, testdir): def test_getxspecs(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz") config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
nodemanager = NodeManager(config) nodemanager = NodeManager(config)
xspecs = nodemanager._getxspecs() xspecs = nodemanager._getxspecs()
assert len(xspecs) == 2 assert len(xspecs) == 2
@@ -169,39 +175,39 @@ class TestDistOptions:
assert xspecs[0].popen assert xspecs[0].popen
assert xspecs[1].ssh == "xyz" assert xspecs[1].ssh == "xyz"
def test_xspecs_multiplied(self, testdir): def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--tx=3*popen") config = pytester.parseconfigure("--tx=3*popen")
xspecs = NodeManager(config)._getxspecs() xspecs = NodeManager(config)._getxspecs()
assert len(xspecs) == 3 assert len(xspecs) == 3
assert xspecs[1].popen assert xspecs[1].popen
def test_getrsyncdirs(self, testdir): def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--rsyncdir=" + str(testdir.tmpdir)) config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
nm = NodeManager(config, specs=[execnet.XSpec("popen")]) nm = NodeManager(config, specs=[execnet.XSpec("popen")])
assert not nm._getrsyncdirs() assert not nm._getrsyncdirs()
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert nm.roots assert nm.roots
assert testdir.tmpdir in nm.roots assert pytester.path in nm.roots
def test_getrsyncignore(self, testdir): def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
config = testdir.parseconfigure("--rsyncignore=fo*") config = pytester.parseconfigure("--rsyncignore=fo*")
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert "fo*" in nm.rsyncoptions["ignores"] assert "fo*" in nm.rsyncoptions["ignores"]
def test_getrsyncdirs_with_conftest(self, testdir): def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
p = py.path.local() p = Path.cwd()
for bn in "x y z".split(): for bn in ("x", "y", "z"):
p.mkdir(bn) p.joinpath(bn).mkdir()
testdir.makeini( pytester.makeini(
""" """
[pytest] [pytest]
rsyncdirs= x rsyncdirs= x
""" """
) )
config = testdir.parseconfigure(testdir.tmpdir, "--rsyncdir=y", "--rsyncdir=z") config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")]) nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
roots = nm._getrsyncdirs() roots = nm._getrsyncdirs()
# assert len(roots) == 3 + 1 # pylib # assert len(roots) == 3 + 1 # pylib
assert py.path.local("y") in roots assert Path("y").resolve() in roots
assert py.path.local("z") in roots assert Path("z").resolve() in roots
assert testdir.tmpdir.join("x") in roots assert pytester.path.joinpath("x") in roots

92
testing/test_remote.py
View File

@@ -1,5 +1,5 @@
import py
import pprint import pprint
import py
import pytest import pytest
import sys import sys
import uuid import uuid
@@ -32,18 +32,16 @@ class EventCall:
class WorkerSetup: class WorkerSetup:
use_callback = False use_callback = False
def __init__(self, request, testdir): def __init__(self, request, pytester: pytest.Pytester) -> None:
self.request = request self.request = request
self.testdir = testdir self.pytester = pytester
self.events = Queue() self.events = Queue() # type: ignore[var-annotated]
def setup( def setup(self) -> None:
self, self.pytester.chdir()
):
self.testdir.chdir()
# import os ; os.environ['EXECNET_DEBUG'] = "2" # import os ; os.environ['EXECNET_DEBUG'] = "2"
self.gateway = execnet.makegateway() self.gateway = execnet.makegateway()
self.config = config = self.testdir.parseconfigure() self.config = config = self.pytester.parseconfigure()
putevent = self.use_callback and self.events.put or None putevent = self.use_callback and self.events.put or None
class DummyMananger: class DummyMananger:
@@ -70,15 +68,15 @@ class WorkerSetup:
@pytest.fixture @pytest.fixture
def worker(request, testdir): def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
return WorkerSetup(request, testdir) return WorkerSetup(request, pytester)
@pytest.mark.xfail(reason="#59") @pytest.mark.xfail(reason="#59")
def test_remoteinitconfig(testdir): def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
from xdist.remote import remote_initconfig from xdist.remote import remote_initconfig
config1 = testdir.parseconfig() config1 = pytester.parseconfig()
config2 = remote_initconfig(config1.option.__dict__, config1.args) config2 = remote_initconfig(config1.option.__dict__, config1.args)
assert config2.option.__dict__ == config1.option.__dict__ assert config2.option.__dict__ == config1.option.__dict__
assert config2.pluginmanager.getplugin("terminal") in (-1, None) assert config2.pluginmanager.getplugin("terminal") in (-1, None)
@@ -94,8 +92,10 @@ class TestWorkerInteractor:
return unserialize return unserialize
def test_basic_collect_and_runtests(self, worker, unserialize_report): def test_basic_collect_and_runtests(
worker.testdir.makepyfile( self, worker: WorkerSetup, unserialize_report
) -> None:
worker.pytester.makepyfile(
""" """
def test_func(): def test_func():
pass pass
@@ -108,7 +108,7 @@ class TestWorkerInteractor:
assert ev.name == "collectionstart" assert ev.name == "collectionstart"
assert not ev.kwargs assert not ev.kwargs
ev = worker.popevent("collectionfinish") ev = worker.popevent("collectionfinish")
assert ev.kwargs["topdir"] == worker.testdir.tmpdir assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
ids = ev.kwargs["ids"] ids = ev.kwargs["ids"]
assert len(ids) == 1 assert len(ids) == 1
worker.sendcommand("runtests", indices=list(range(len(ids)))) worker.sendcommand("runtests", indices=list(range(len(ids))))
@@ -126,8 +126,8 @@ class TestWorkerInteractor:
ev = worker.popevent("workerfinished") ev = worker.popevent("workerfinished")
assert "workeroutput" in ev.kwargs assert "workeroutput" in ev.kwargs
def test_remote_collect_skip(self, worker, unserialize_report): def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
worker.testdir.makepyfile( worker.pytester.makepyfile(
""" """
import pytest import pytest
pytest.skip("hello", allow_module_level=True) pytest.skip("hello", allow_module_level=True)
@@ -144,8 +144,8 @@ class TestWorkerInteractor:
ev = worker.popevent("collectionfinish") ev = worker.popevent("collectionfinish")
assert not ev.kwargs["ids"] assert not ev.kwargs["ids"]
def test_remote_collect_fail(self, worker, unserialize_report): def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
worker.testdir.makepyfile("""aasd qwe""") worker.pytester.makepyfile("""aasd qwe""")
worker.setup() worker.setup()
ev = worker.popevent("collectionstart") ev = worker.popevent("collectionstart")
assert not ev.kwargs assert not ev.kwargs
@@ -156,8 +156,8 @@ class TestWorkerInteractor:
ev = worker.popevent("collectionfinish") ev = worker.popevent("collectionfinish")
assert not ev.kwargs["ids"] assert not ev.kwargs["ids"]
def test_runtests_all(self, worker, unserialize_report): def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
worker.testdir.makepyfile( worker.pytester.makepyfile(
""" """
def test_func(): pass def test_func(): pass
def test_func2(): pass def test_func2(): pass
@@ -183,17 +183,19 @@ class TestWorkerInteractor:
ev = worker.popevent("workerfinished") ev = worker.popevent("workerfinished")
assert "workeroutput" in ev.kwargs assert "workeroutput" in ev.kwargs
def test_happy_run_events_converted(self, testdir, worker): def test_happy_run_events_converted(
py.test.xfail("implement a simple test for event production") self, pytester: pytest.Pytester, worker: WorkerSetup
assert not worker.use_callback ) -> None:
worker.testdir.makepyfile( pytest.xfail("implement a simple test for event production")
assert not worker.use_callback # type: ignore[unreachable]
worker.pytester.makepyfile(
""" """
def test_func(): def test_func():
pass pass
""" """
) )
worker.setup() worker.setup()
hookrec = testdir.getreportrecorder(worker.config) hookrec = pytester.getreportrecorder(worker.config)
for data in worker.slp.channel: for data in worker.slp.channel:
worker.slp.process_from_remote(data) worker.slp.process_from_remote(data)
worker.slp.process_from_remote(worker.slp.ENDMARK) worker.slp.process_from_remote(worker.slp.ENDMARK)
@@ -209,7 +211,9 @@ class TestWorkerInteractor:
] ]
) )
def test_process_from_remote_error_handling(self, worker, capsys): def test_process_from_remote_error_handling(
self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
) -> None:
worker.use_callback = True worker.use_callback = True
worker.setup() worker.setup()
worker.slp.process_from_remote(("<nonono>", ())) worker.slp.process_from_remote(("<nonono>", ()))
@@ -219,8 +223,8 @@ class TestWorkerInteractor:
assert ev.name == "errordown" assert ev.name == "errordown"
def test_remote_env_vars(testdir): def test_remote_env_vars(pytester: pytest.Pytester) -> None:
testdir.makepyfile( pytester.makepyfile(
""" """
import os import os
def test(): def test():
@@ -229,13 +233,13 @@ def test_remote_env_vars(testdir):
assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2' assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
""" """
) )
result = testdir.runpytest("-n2", "--max-worker-restart=0") result = pytester.runpytest("-n2", "--max-worker-restart=0")
assert result.ret == 0 assert result.ret == 0
def test_remote_inner_argv(testdir): def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
"""Test/document the behavior due to execnet using `python -c`.""" """Test/document the behavior due to execnet using `python -c`."""
testdir.makepyfile( pytester.makepyfile(
""" """
import sys import sys
@@ -243,14 +247,14 @@ def test_remote_inner_argv(testdir):
assert sys.argv == ["-c"] assert sys.argv == ["-c"]
""" """
) )
result = testdir.runpytest("-n1") result = pytester.runpytest("-n1")
assert result.ret == 0 assert result.ret == 0
def test_remote_mainargv(testdir): def test_remote_mainargv(pytester: pytest.Pytester) -> None:
outer_argv = sys.argv outer_argv = sys.argv
testdir.makepyfile( pytester.makepyfile(
""" """
def test_mainargv(request): def test_mainargv(request):
assert request.config.workerinput["mainargv"] == {!r} assert request.config.workerinput["mainargv"] == {!r}
@@ -258,14 +262,14 @@ def test_remote_mainargv(testdir):
outer_argv outer_argv
) )
) )
result = testdir.runpytest("-n1") result = pytester.runpytest("-n1")
assert result.ret == 0 assert result.ret == 0
def test_remote_usage_prog(testdir, request): def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
if not hasattr(request.config._parser, "prog"): if not hasattr(request.config._parser, "prog"):
pytest.skip("prog not available in config parser") pytest.skip("prog not available in config parser")
testdir.makeconftest( pytester.makeconftest(
""" """
import pytest import pytest
@@ -280,7 +284,7 @@ def test_remote_usage_prog(testdir, request):
config_parser = config._parser config_parser = config._parser
""" """
) )
testdir.makepyfile( pytester.makepyfile(
""" """
import sys import sys
@@ -289,14 +293,14 @@ def test_remote_usage_prog(testdir, request):
""" """
) )
result = testdir.runpytest_subprocess("-n1") result = pytester.runpytest_subprocess("-n1")
assert result.ret == 1 assert result.ret == 1
result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"]) result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
def test_remote_sys_path(testdir): def test_remote_sys_path(pytester: pytest.Pytester) -> None:
"""Work around sys.path differences due to execnet using `python -c`.""" """Work around sys.path differences due to execnet using `python -c`."""
testdir.makepyfile( pytester.makepyfile(
""" """
import sys import sys
@@ -304,5 +308,5 @@ def test_remote_sys_path(testdir):
assert "" not in sys.path assert "" not in sys.path
""" """
) )
result = testdir.runpytest("-n1") result = pytester.runpytest("-n1")
assert result.ret == 0 assert result.ret == 0

273
testing/test_workermanage.py
View File

@@ -1,39 +1,42 @@
import execnet
import py import py
import pytest import pytest
import shutil
import textwrap import textwrap
import execnet from pathlib import Path
from _pytest.pytester import HookRecorder from xdist import workermanage
from xdist import workermanage, newhooks
from xdist.workermanage import HostRSync, NodeManager from xdist.workermanage import HostRSync, NodeManager
pytest_plugins = "pytester" pytest_plugins = "pytester"
@pytest.fixture @pytest.fixture
def hookrecorder(request, config): def hookrecorder(request, config, pytester: pytest.Pytester):
hookrecorder = HookRecorder(config.pluginmanager) hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
if hasattr(hookrecorder, "start_recording"):
hookrecorder.start_recording(newhooks)
request.addfinalizer(hookrecorder.finish_recording)
return hookrecorder return hookrecorder
@pytest.fixture @pytest.fixture
def config(testdir): def config(pytester: pytest.Pytester):
return testdir.parseconfig() return pytester.parseconfig()
@pytest.fixture @pytest.fixture
def mysetup(tmpdir): def source(tmp_path: Path) -> Path:
class mysetup: source = tmp_path / "source"
source = tmpdir.mkdir("source") source.mkdir()
dest = tmpdir.mkdir("dest") return source
return mysetup()
@pytest.fixture @pytest.fixture
def workercontroller(monkeypatch): def dest(tmp_path: Path) -> Path:
dest = tmp_path / "dest"
dest.mkdir()
return dest
@pytest.fixture
def workercontroller(monkeypatch: pytest.MonkeyPatch):
class MockController: class MockController:
def __init__(self, *args): def __init__(self, *args):
pass pass
@@ -46,18 +49,20 @@ def workercontroller(monkeypatch):
class TestNodeManagerPopen: class TestNodeManagerPopen:
def test_popen_no_default_chdir(self, config): def test_popen_no_default_chdir(self, config) -> None:
gm = NodeManager(config, ["popen"]) gm = NodeManager(config, ["popen"])
assert gm.specs[0].chdir is None assert gm.specs[0].chdir is None
def test_default_chdir(self, config): def test_default_chdir(self, config) -> None:
specs = ["ssh=noco", "socket=xyz"] specs = ["ssh=noco", "socket=xyz"]
for spec in NodeManager(config, specs).specs: for spec in NodeManager(config, specs).specs:
assert spec.chdir == "pyexecnetcache" assert spec.chdir == "pyexecnetcache"
for spec in NodeManager(config, specs, defaultchdir="abc").specs: for spec in NodeManager(config, specs, defaultchdir="abc").specs:
assert spec.chdir == "abc" assert spec.chdir == "abc"
def test_popen_makegateway_events(self, config, hookrecorder, workercontroller): def test_popen_makegateway_events(
self, config, hookrecorder, workercontroller
) -> None:
hm = NodeManager(config, ["popen"] * 2) hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None) hm.setup_nodes(None)
call = hookrecorder.popcall("pytest_xdist_setupnodes") call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -72,15 +77,16 @@ class TestNodeManagerPopen:
hm.teardown_nodes() hm.teardown_nodes()
assert not len(hm.group) assert not len(hm.group)
def test_popens_rsync(self, config, mysetup, workercontroller): def test_popens_rsync(
source = mysetup.source self, config, source: Path, dest: Path, workercontroller
) -> None:
hm = NodeManager(config, ["popen"] * 2) hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None) hm.setup_nodes(None)
assert len(hm.group) == 2 assert len(hm.group) == 2
for gw in hm.group: for gw in hm.group:
class pseudoexec: class pseudoexec:
args = [] args = [] # type: ignore[var-annotated]
def __init__(self, *args): def __init__(self, *args):
self.args.extend(args) self.args.extend(args)
@@ -97,30 +103,37 @@ class TestNodeManagerPopen:
assert not len(hm.group) assert not len(hm.group)
assert "sys.path.insert" in gw.remote_exec.args[0] assert "sys.path.insert" in gw.remote_exec.args[0]
def test_rsync_popen_with_path(self, config, mysetup, workercontroller): def test_rsync_popen_with_path(
source, dest = mysetup.source, mysetup.dest self, config, source: Path, dest: Path, workercontroller
) -> None:
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1) hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
hm.setup_nodes(None) hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello") source.joinpath("dir1", "dir2").mkdir(parents=True)
source.joinpath("dir1", "dir2", "hello").touch()
notifications = [] notifications = []
for gw in hm.group: for gw in hm.group:
hm.rsync(gw, source, notify=lambda *args: notifications.append(args)) hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
assert len(notifications) == 1 assert len(notifications) == 1
assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source) assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
hm.teardown_nodes() hm.teardown_nodes()
dest = dest.join(source.basename) dest = dest.joinpath(source.name)
assert dest.join("dir1").check() assert dest.joinpath("dir1").exists()
assert dest.join("dir1", "dir2").check() assert dest.joinpath("dir1", "dir2").exists()
assert dest.join("dir1", "dir2", "hello").check() assert dest.joinpath("dir1", "dir2", "hello").exists()
def test_rsync_same_popen_twice( def test_rsync_same_popen_twice(
self, config, mysetup, hookrecorder, workercontroller self,
): config,
source, dest = mysetup.source, mysetup.dest source: Path,
dest: Path,
hookrecorder,
workercontroller,
) -> None:
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2) hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
hm.roots = [] hm.roots = []
hm.setup_nodes(None) hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello") source.joinpath("dir1", "dir2").mkdir(parents=True)
source.joinpath("dir1", "dir2", "hello").touch()
gw = hm.group[0] gw = hm.group[0]
hm.rsync(gw, source) hm.rsync(gw, source)
call = hookrecorder.popcall("pytest_xdist_rsyncstart") call = hookrecorder.popcall("pytest_xdist_rsyncstart")
@@ -131,83 +144,98 @@ class TestNodeManagerPopen:
class TestHRSync: class TestHRSync:
def test_hrsync_filter(self, mysetup): def test_hrsync_filter(self, source: Path, dest: Path) -> None:
source, _ = mysetup.source, mysetup.dest # noqa source.joinpath("dir").mkdir()
source.ensure("dir", "file.txt") source.joinpath("dir", "file.txt").touch()
source.ensure(".svn", "entries") source.joinpath(".svn").mkdir()
source.ensure(".somedotfile", "moreentries") source.joinpath(".svn", "entries").touch()
source.ensure("somedir", "editfile~") source.joinpath(".somedotfile").mkdir()
source.joinpath(".somedotfile", "moreentries").touch()
source.joinpath("somedir").mkdir()
source.joinpath("somedir", "editfile~").touch()
syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES) syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
files = list(source.visit(rec=syncer.filter, fil=syncer.filter)) files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
assert len(files) == 3 assert len(files) == 3
basenames = [x.basename for x in files] basenames = [x.basename for x in files]
assert "dir" in basenames assert "dir" in basenames
assert "file.txt" in basenames assert "file.txt" in basenames
assert "somedir" in basenames assert "somedir" in basenames
def test_hrsync_one_host(self, mysetup): def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
source, dest = mysetup.source, mysetup.dest
gw = execnet.makegateway("popen//chdir=%s" % dest) gw = execnet.makegateway("popen//chdir=%s" % dest)
finished = [] finished = []
rsync = HostRSync(source) rsync = HostRSync(source)
rsync.add_target_host(gw, finished=lambda: finished.append(1)) rsync.add_target_host(gw, finished=lambda: finished.append(1))
source.join("hello.py").write("world") source.joinpath("hello.py").write_text("world")
rsync.send() rsync.send()
gw.exit() gw.exit()
assert dest.join(source.basename, "hello.py").check() assert dest.joinpath(source.name, "hello.py").exists()
assert len(finished) == 1 assert len(finished) == 1
class TestNodeManager: class TestNodeManager:
@py.test.mark.xfail(run=False) @pytest.mark.xfail(run=False)
def test_rsync_roots_no_roots(self, testdir, mysetup): def test_rsync_roots_no_roots(
mysetup.source.ensure("dir1", "file1").write("hello") self, pytester: pytest.Pytester, source: Path, dest: Path
config = testdir.parseconfig(mysetup.source) ) -> None:
nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest]) source.joinpath("dir1").mkdir()
source.joinpath("dir1", "file1").write_text("hello")
config = pytester.parseconfig(source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
# assert nodemanager.config.topdir == source == config.topdir # assert nodemanager.config.topdir == source == config.topdir
nodemanager.makegateways() nodemanager.makegateways() # type: ignore[attr-defined]
nodemanager.rsync_roots() nodemanager.rsync_roots() # type: ignore[call-arg]
(p,) = nodemanager.gwmanager.multi_exec( (p,) = nodemanager.gwmanager.multi_exec( # type: ignore[attr-defined]
"import os ; channel.send(os.getcwd())" "import os ; channel.send(os.getcwd())"
).receive_each() ).receive_each()
p = py.path.local(p) p = Path(p)
print("remote curdir", p) print("remote curdir", p)
assert p == mysetup.dest.join(config.topdir.basename) assert p == dest.joinpath(config.rootpath.name)
assert p.join("dir1").check() assert p.joinpath("dir1").check()
assert p.join("dir1", "file1").check() assert p.joinpath("dir1", "file1").check()
def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller): def test_popen_rsync_subdir(
source, dest = mysetup.source, mysetup.dest self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
dir1 = mysetup.source.mkdir("dir1") ) -> None:
dir2 = dir1.mkdir("dir2") dir1 = source / "dir1"
dir2.ensure("hello") dir1.mkdir()
dir2 = dir1 / "dir2"
dir2.mkdir()
dir2.joinpath("hello").touch()
for rsyncroot in (dir1, source): for rsyncroot in (dir1, source):
dest.remove() shutil.rmtree(str(dest), ignore_errors=True)
nodemanager = NodeManager( nodemanager = NodeManager(
testdir.parseconfig( pytester.parseconfig(
"--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
) )
) )
nodemanager.setup_nodes(None) # calls .rsync_roots() nodemanager.setup_nodes(None) # calls .rsync_roots()
if rsyncroot == source: if rsyncroot == source:
dest = dest.join("source") dest = dest.joinpath("source")
assert dest.join("dir1").check() assert dest.joinpath("dir1").exists()
assert dest.join("dir1", "dir2").check() assert dest.joinpath("dir1", "dir2").exists()
assert dest.join("dir1", "dir2", "hello").check() assert dest.joinpath("dir1", "dir2", "hello").exists()
nodemanager.teardown_nodes() nodemanager.teardown_nodes()
@pytest.mark.parametrize( @pytest.mark.parametrize(
"flag, expects_report", [("-q", False), ("", False), ("-v", True)] "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
) )
def test_rsync_report( def test_rsync_report(
self, testdir, mysetup, workercontroller, capsys, flag, expects_report self,
): pytester: pytest.Pytester,
source, dest = mysetup.source, mysetup.dest source: Path,
dir1 = mysetup.source.mkdir("dir1") dest: Path,
args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source workercontroller,
capsys: pytest.CaptureFixture[str],
flag: str,
expects_report: bool,
) -> None:
dir1 = source / "dir1"
dir1.mkdir()
args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
if flag: if flag:
args += (flag,) args.append(flag)
nodemanager = NodeManager(testdir.parseconfig(*args)) nodemanager = NodeManager(pytester.parseconfig(*args))
nodemanager.setup_nodes(None) # calls .rsync_roots() nodemanager.setup_nodes(None) # calls .rsync_roots()
out, _ = capsys.readouterr() out, _ = capsys.readouterr()
if expects_report: if expects_report:
@@ -215,77 +243,86 @@ class TestNodeManager:
else: else:
assert "<= pytest/__init__.py" not in out assert "<= pytest/__init__.py" not in out
def test_init_rsync_roots(self, testdir, mysetup, workercontroller): def test_init_rsync_roots(
source, dest = mysetup.source, mysetup.dest self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
dir2 = source.ensure("dir1", "dir2", dir=1) ) -> None:
source.ensure("dir1", "somefile", dir=1) dir2 = source.joinpath("dir1", "dir2")
dir2.ensure("hello") dir2.mkdir(parents=True)
source.ensure("bogusdir", "file") source.joinpath("dir1", "somefile").mkdir()
source.join("tox.ini").write( dir2.joinpath("hello").touch()
source.joinpath("bogusdir").mkdir()
source.joinpath("bogusdir", "file").touch()
source.joinpath("tox.ini").write_text(
textwrap.dedent( textwrap.dedent(
""" """
[pytest] [pytest]
rsyncdirs=dir1/dir2 rsyncdirs=dir1/dir2
""" """
) )
) )
config = testdir.parseconfig(source) config = pytester.parseconfig(source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots() nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir2").check() assert dest.joinpath("dir2").exists()
assert not dest.join("dir1").check() assert not dest.joinpath("dir1").exists()
assert not dest.join("bogus").check() assert not dest.joinpath("bogus").exists()
def test_rsyncignore(self, testdir, mysetup, workercontroller): def test_rsyncignore(
source, dest = mysetup.source, mysetup.dest self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
dir2 = source.ensure("dir1", "dir2", dir=1) ) -> None:
source.ensure("dir5", "dir6", "bogus") dir2 = source.joinpath("dir1", "dir2")
source.ensure("dir5", "file") dir2.mkdir(parents=True)
dir2.ensure("hello") source.joinpath("dir5", "dir6").mkdir(parents=True)
source.ensure("foo", "bar") source.joinpath("dir5", "dir6", "bogus").touch()
source.ensure("bar", "foo") source.joinpath("dir5", "file").touch()
source.join("tox.ini").write( dir2.joinpath("hello").touch()
source.joinpath("foo").mkdir()
source.joinpath("foo", "bar").touch()
source.joinpath("bar").mkdir()
source.joinpath("bar", "foo").touch()
source.joinpath("tox.ini").write_text(
textwrap.dedent( textwrap.dedent(
""" """
[pytest] [pytest]
rsyncdirs = dir1 dir5 rsyncdirs = dir1 dir5
rsyncignore = dir1/dir2 dir5/dir6 foo* rsyncignore = dir1/dir2 dir5/dir6 foo*
""" """
) )
) )
config = testdir.parseconfig(source) config = pytester.parseconfig(source)
config.option.rsyncignore = ["bar"] config.option.rsyncignore = ["bar"]
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots() nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir1").check() assert dest.joinpath("dir1").exists()
assert not dest.join("dir1", "dir2").check() assert not dest.joinpath("dir1", "dir2").exists()
assert dest.join("dir5", "file").check() assert dest.joinpath("dir5", "file").exists()
assert not dest.join("dir6").check() assert not dest.joinpath("dir6").exists()
assert not dest.join("foo").check() assert not dest.joinpath("foo").exists()
assert not dest.join("bar").check() assert not dest.joinpath("bar").exists()
def test_optimise_popen(self, testdir, mysetup, workercontroller): def test_optimise_popen(
source = mysetup.source self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
) -> None:
specs = ["popen"] * 3 specs = ["popen"] * 3
source.join("conftest.py").write("rsyncdirs = ['a']") source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
source.ensure("a", dir=1) source.joinpath("a").mkdir()
config = testdir.parseconfig(source) config = pytester.parseconfig(source)
nodemanager = NodeManager(config, specs) nodemanager = NodeManager(config, specs)
nodemanager.setup_nodes(None) # calls .rysnc_roots() nodemanager.setup_nodes(None) # calls .rysnc_roots()
for gwspec in nodemanager.specs: for gwspec in nodemanager.specs:
assert gwspec._samefilesystem() assert gwspec._samefilesystem()
assert not gwspec.chdir assert not gwspec.chdir
def test_ssh_setup_nodes(self, specssh, testdir): def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
testdir.makepyfile( pytester.makepyfile(
__init__="", __init__="",
test_x=""" test_x="""
def test_one(): def test_one():
pass pass
""", """,
) )
reprec = testdir.inline_run( reprec = pytester.inline_run(
"-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir "-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
) )
(rep,) = reprec.getreports("pytest_runtest_logreport") (rep,) = reprec.getreports("pytest_runtest_logreport")
assert rep.passed assert rep.passed

9
tox.ini
View File

@@ -18,7 +18,6 @@ commands=
extras = extras =
testing testing
psutil psutil
deps = pytest
commands = commands =
pytest {posargs:-k psutil} pytest {posargs:-k psutil}
@@ -30,6 +29,14 @@ deps = pytest
commands = commands =
pytest {posargs} pytest {posargs}
[testenv:linting]
skip_install = True
usedevelop = True
passenv = PRE_COMMIT_HOME
deps =
pre-commit
commands = pre-commit run --all-files --show-diff-on-failure
[testenv:release] [testenv:release]
changedir= changedir=
decription = do a release, required posarg of the version number decription = do a release, required posarg of the version number