IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update test suite to modern pytest

Browse Source 
- Use pytest>=6.2 features (same as install_requires).
  When we can require >=7, can fix some more typing omissions and
  version checks.
- Replace testdir with pytester
- Replace py.test with pytest
- Replace tmpdir with tmp_path
- Remove (almost) all other uses of py
- Add some type annotations (not checked yet)

Ref #722.
This commit is contained in:
Ran Benita
2021-10-30 11:02:23 +03:00
parent 5672d85809
commit 9ddb274f23
11 changed files with 768 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
setup.cfg
View File

@@ -53,7 +53,6 @@ pytest11 =
[options.extras_require]
testing =
filelock
pytest
psutil = psutil>=3.0
setproctitle = setproctitle

4
src/xdist/workermanage.py
View File

@@ -423,9 +423,9 @@ def unserialize_warning_message(data):
kwargs = {"message": message, "category": category}
# access private _WARNING_DETAILS because the attributes vary between Python versions
for attr_name in warnings.WarningMessage._WARNING_DETAILS:
for attr_name in warnings.WarningMessage._WARNING_DETAILS: # type: ignore[attr-defined]
if attr_name in ("message", "category"):
continue
kwargs[attr_name] = data[attr_name]
return warnings.WarningMessage(**kwargs)
return warnings.WarningMessage(**kwargs) # type: ignore[arg-type]

489
testing/acceptance_test.py
View File

File diff suppressed because it is too large Load Diff

25
testing/conftest.py
View File

@@ -1,12 +1,13 @@
import py
import pytest
import execnet
import pytest
import shutil
from typing import List
pytest_plugins = "pytester"
@pytest.fixture(autouse=True)
def _divert_atexit(request, monkeypatch):
def _divert_atexit(request, monkeypatch: pytest.MonkeyPatch):
import atexit
finalizers = []
@@ -23,7 +24,7 @@ def _divert_atexit(request, monkeypatch):
func(*args, **kwargs)
def pytest_addoption(parser):
def pytest_addoption(parser) -> None:
parser.addoption(
"--gx",
action="append",
@@ -33,28 +34,28 @@ def pytest_addoption(parser):
@pytest.fixture
def specssh(request):
def specssh(request) -> str:
return getspecssh(request.config)
# configuration information for tests
def getgspecs(config):
def getgspecs(config) -> List[execnet.XSpec]:
return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]
def getspecssh(config):
def getspecssh(config) -> str: # type: ignore[return]
xspecs = getgspecs(config)
for spec in xspecs:
if spec.ssh:
if not py.path.local.sysfind("ssh"):
py.test.skip("command not found: ssh")
if not shutil.which("ssh"):
pytest.skip("command not found: ssh")
return str(spec)
py.test.skip("need '--gx ssh=...'")
pytest.skip("need '--gx ssh=...'")
def getsocketspec(config):
def getsocketspec(config) -> execnet.XSpec:
xspecs = getgspecs(config)
for spec in xspecs:
if spec.socket:
return spec
py.test.skip("need '--gx socket=...'")
pytest.skip("need '--gx socket=...'")

91
testing/test_dsession.py
View File

@@ -1,59 +1,44 @@
from xdist.dsession import DSession, get_default_max_worker_restart
from xdist.report import report_collection_diff
from xdist.scheduler import EachScheduling, LoadScheduling
from typing import Optional
import py
import pytest
import execnet
XSpec = execnet.XSpec
def run(item, node, excinfo=None):
runner = item.config.pluginmanager.getplugin("runner")
rep = runner.ItemTestReport(item=item, excinfo=excinfo, when="call")
rep.node = node
return rep
class MockGateway:
_count = 0
def __init__(self):
def __init__(self) -> None:
self._count = 0
self.id = str(self._count)
self._count += 1
class MockNode:
def __init__(self):
self.sent = []
def __init__(self) -> None:
self.sent = [] # type: ignore[var-annotated]
self.gateway = MockGateway()
self._shutdown = False
def send_runtest_some(self, indices):
def send_runtest_some(self, indices) -> None:
self.sent.extend(indices)
def send_runtest_all(self):
def send_runtest_all(self) -> None:
self.sent.append("ALL")
def shutdown(self):
def shutdown(self) -> None:
self._shutdown = True
@property
def shutting_down(self):
def shutting_down(self) -> bool:
return self._shutdown
def dumpqueue(queue):
while queue.qsize():
print(queue.get())
class TestEachScheduling:
def test_schedule_load_simple(self, testdir):
def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
node1 = MockNode()
node2 = MockNode()
config = testdir.parseconfig("--tx=2*popen")
config = pytester.parseconfig("--tx=2*popen")
sched = EachScheduling(config)
sched.add_node(node1)
sched.add_node(node2)
@@ -74,9 +59,9 @@ class TestEachScheduling:
sched.mark_test_complete(node2, 0)
assert sched.tests_finished
def test_schedule_remove_node(self, testdir):
def test_schedule_remove_node(self, pytester: pytest.Pytester) -> None:
node1 = MockNode()
config = testdir.parseconfig("--tx=popen")
config = pytester.parseconfig("--tx=popen")
sched = EachScheduling(config)
sched.add_node(node1)
collection = ["a.py::test_1"]
@@ -93,8 +78,8 @@ class TestEachScheduling:
class TestLoadScheduling:
def test_schedule_load_simple(self, testdir):
config = testdir.parseconfig("--tx=2*popen")
def test_schedule_load_simple(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config)
sched.add_node(MockNode())
sched.add_node(MockNode())
@@ -117,8 +102,8 @@ class TestLoadScheduling:
sched.mark_test_complete(node1, node1.sent[0])
assert sched.tests_finished
def test_schedule_batch_size(self, testdir):
config = testdir.parseconfig("--tx=2*popen")
def test_schedule_batch_size(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config)
sched.add_node(MockNode())
sched.add_node(MockNode())
@@ -144,8 +129,8 @@ class TestLoadScheduling:
assert node1.sent == [0, 2, 4, 5]
assert not sched.pending
def test_schedule_fewer_tests_than_nodes(self, testdir):
config = testdir.parseconfig("--tx=2*popen")
def test_schedule_fewer_tests_than_nodes(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config)
sched.add_node(MockNode())
sched.add_node(MockNode())
@@ -164,8 +149,10 @@ class TestLoadScheduling:
assert sent3 == []
assert not sched.pending
def test_schedule_fewer_than_two_tests_per_node(self, testdir):
config = testdir.parseconfig("--tx=2*popen")
def test_schedule_fewer_than_two_tests_per_node(
self, pytester: pytest.Pytester
) -> None:
config = pytester.parseconfig("--tx=2*popen")
sched = LoadScheduling(config)
sched.add_node(MockNode())
sched.add_node(MockNode())
@@ -184,9 +171,9 @@ class TestLoadScheduling:
assert sent3 == [2]
assert not sched.pending
def test_add_remove_node(self, testdir):
def test_add_remove_node(self, pytester: pytest.Pytester) -> None:
node = MockNode()
config = testdir.parseconfig("--tx=popen")
config = pytester.parseconfig("--tx=popen")
sched = LoadScheduling(config)
sched.add_node(node)
collection = ["test_file.py::test_func"]
@@ -197,7 +184,7 @@ class TestLoadScheduling:
crashitem = sched.remove_node(node)
assert crashitem == collection[0]
def test_different_tests_collected(self, testdir):
def test_different_tests_collected(self, pytester: pytest.Pytester) -> None:
"""
Test that LoadScheduling is reporting collection errors when
different test ids are collected by workers.
@@ -215,7 +202,7 @@ class TestLoadScheduling:
self.reports.append(report)
collect_hook = CollectHook()
config = testdir.parseconfig("--tx=2*popen")
config = pytester.parseconfig("--tx=2*popen")
config.pluginmanager.register(collect_hook, "collect_hook")
node1 = MockNode()
node2 = MockNode()
@@ -231,9 +218,9 @@ class TestLoadScheduling:
class TestDistReporter:
@py.test.mark.xfail
def test_rsync_printing(self, testdir, linecomp):
config = testdir.parseconfig()
@pytest.mark.xfail
def test_rsync_printing(self, pytester: pytest.Pytester, linecomp) -> None:
config = pytester.parseconfig()
from _pytest.pytest_terminal import TerminalReporter
rep = TerminalReporter(config, file=linecomp.stringio)
@@ -258,21 +245,21 @@ class TestDistReporter:
# linecomp.assert_contains_lines([
# "*X1*popen*xyz*2.5*"
# ])
dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2])
dsession.pytest_xdist_rsyncstart(source="hello", gateways=[gw1, gw2]) # type: ignore[attr-defined]
linecomp.assert_contains_lines(["[X1,X2] rsyncing: hello"])
def test_report_collection_diff_equal():
def test_report_collection_diff_equal() -> None:
"""Test reporting of equal collections."""
from_collection = to_collection = ["aaa", "bbb", "ccc"]
assert report_collection_diff(from_collection, to_collection, 1, 2) is None
def test_default_max_worker_restart():
def test_default_max_worker_restart() -> None:
class config:
class option:
maxworkerrestart = None
numprocesses = 0
maxworkerrestart: Optional[str] = None
numprocesses: int = 0
assert get_default_max_worker_restart(config) is None
@@ -286,7 +273,7 @@ def test_default_max_worker_restart():
assert get_default_max_worker_restart(config) == 0
def test_report_collection_diff_different():
def test_report_collection_diff_different() -> None:
"""Test reporting of different collections."""
from_collection = ["aaa", "bbb", "ccc", "YYY"]
to_collection = ["aZa", "bbb", "XXX", "ccc"]
@@ -311,8 +298,8 @@ def test_report_collection_diff_different():
@pytest.mark.xfail(reason="duplicate test ids not supported yet")
def test_pytest_issue419(testdir):
testdir.makepyfile(
def test_pytest_issue419(pytester: pytest.Pytester) -> None:
pytester.makepyfile(
"""
import pytest
@@ -321,6 +308,6 @@ def test_pytest_issue419(testdir):
pass
"""
)
reprec = testdir.inline_run("-n1")
reprec = pytester.inline_run("-n1")
reprec.assertoutcome(passed=2)
assert 0

221
testing/test_looponfail.py
View File

@@ -1,90 +1,106 @@
import py
import pytest
from pkg_resources import parse_version
import shutil
import textwrap
from pathlib import Path
from xdist.looponfail import RemoteControl
from xdist.looponfail import StatRecorder
PYTEST_GTE_7 = hasattr(pytest, "version_tuple") and pytest.version_tuple >= (7, 0) # type: ignore[attr-defined]
class TestStatRecorder:
def test_filechange(self, tmpdir):
tmp = tmpdir
hello = tmp.ensure("hello.py")
sd = StatRecorder([tmp])
def test_filechange(self, tmp_path: Path) -> None:
tmp = tmp_path
hello = tmp / "hello.py"
hello.touch()
sd = StatRecorder([py.path.local(tmp)])
changed = sd.check()
assert not changed
hello.write("world")
hello.write_text("world")
changed = sd.check()
assert changed
(hello + "c").write("hello")
hello.with_suffix(".pyc").write_text("hello")
changed = sd.check()
assert not changed
p = tmp.ensure("new.py")
p = tmp / "new.py"
p.touch()
changed = sd.check()
assert changed
p.remove()
p.unlink()
changed = sd.check()
assert changed
tmp.join("a", "b", "c.py").ensure()
tmp.joinpath("a", "b").mkdir(parents=True)
tmp.joinpath("a", "b", "c.py").touch()
changed = sd.check()
assert changed
tmp.join("a", "c.txt").ensure()
tmp.joinpath("a", "c.txt").touch()
changed = sd.check()
assert changed
changed = sd.check()
assert not changed
tmp.join("a").remove()
shutil.rmtree(str(tmp.joinpath("a")))
changed = sd.check()
assert changed
def test_dirchange(self, tmpdir):
tmp = tmpdir
tmp.ensure("dir", "hello.py")
sd = StatRecorder([tmp])
assert not sd.fil(tmp.join("dir"))
def test_dirchange(self, tmp_path: Path) -> None:
tmp = tmp_path
tmp.joinpath("dir").mkdir()
tmp.joinpath("dir", "hello.py").touch()
sd = StatRecorder([py.path.local(tmp)])
assert not sd.fil(py.path.local(tmp / "dir"))
def test_filechange_deletion_race(self, tmpdir, monkeypatch):
tmp = tmpdir
sd = StatRecorder([tmp])
def test_filechange_deletion_race(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
tmp = tmp_path
pytmp = py.path.local(tmp)
sd = StatRecorder([pytmp])
changed = sd.check()
assert not changed
p = tmp.ensure("new.py")
p = tmp.joinpath("new.py")
p.touch()
changed = sd.check()
assert changed
p.remove()
p.unlink()
# make check()'s visit() call return our just removed
# path as if we were in a race condition
monkeypatch.setattr(tmp, "visit", lambda *args: [p])
monkeypatch.setattr(pytmp, "visit", lambda *args: [py.path.local(p)])
changed = sd.check()
assert changed
def test_pycremoval(self, tmpdir):
tmp = tmpdir
hello = tmp.ensure("hello.py")
sd = StatRecorder([tmp])
def test_pycremoval(self, tmp_path: Path) -> None:
tmp = tmp_path
hello = tmp / "hello.py"
hello.touch()
sd = StatRecorder([py.path.local(tmp)])
changed = sd.check()
assert not changed
pycfile = hello + "c"
pycfile.ensure()
hello.write("world")
pycfile = hello.with_suffix(".pyc")
pycfile.touch()
hello.write_text("world")
changed = sd.check()
assert changed
assert not pycfile.check()
assert not pycfile.exists()
def test_waitonchange(self, tmpdir, monkeypatch):
tmp = tmpdir
sd = StatRecorder([tmp])
def test_waitonchange(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
tmp = tmp_path
sd = StatRecorder([py.path.local(tmp)])
ret_values = [True, False]
monkeypatch.setattr(StatRecorder, "check", lambda self: ret_values.pop())
@@ -93,37 +109,41 @@ class TestStatRecorder:
class TestRemoteControl:
def test_nofailures(self, testdir):
item = testdir.getitem("def test_func(): pass\n")
def test_nofailures(self, pytester: pytest.Pytester) -> None:
item = pytester.getitem("def test_func(): pass\n")
control = RemoteControl(item.config)
control.setup()
topdir, failures = control.runsession()[:2]
assert not failures
def test_failures_somewhere(self, testdir):
item = testdir.getitem("def test_func():\n assert 0\n")
def test_failures_somewhere(self, pytester: pytest.Pytester) -> None:
item = pytester.getitem("def test_func():\n assert 0\n")
control = RemoteControl(item.config)
control.setup()
failures = control.runsession()
assert failures
control.setup()
item.fspath.write("def test_func():\n assert 1\n")
removepyc(item.fspath)
item_path = item.path if PYTEST_GTE_7 else Path(item.fspath) # type: ignore[attr-defined]
item_path.write_text("def test_func():\n assert 1\n")
removepyc(item_path)
topdir, failures = control.runsession()[:2]
assert not failures
def test_failure_change(self, testdir):
modcol = testdir.getitem(
def test_failure_change(self, pytester: pytest.Pytester) -> None:
modcol = pytester.getitem(
textwrap.dedent(
"""
def test_func():
assert 0
"""
)
)
control = RemoteControl(modcol.config)
control.loop_once()
assert control.failures
modcol.fspath.write(
py.code.Source(
modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath) # type: ignore[attr-defined]
modcol_path.write_text(
textwrap.dedent(
"""
def test_func():
assert 1
@@ -132,24 +152,31 @@ class TestRemoteControl:
"""
)
)
removepyc(modcol.fspath)
removepyc(modcol_path)
control.loop_once()
assert not control.failures
control.loop_once()
assert control.failures
assert str(control.failures).find("test_new") != -1
def test_failure_subdir_no_init(self, testdir):
modcol = testdir.getitem(
def test_failure_subdir_no_init(
self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
modcol = pytester.getitem(
textwrap.dedent(
"""
def test_func():
assert 0
"""
)
parent = modcol.fspath.dirpath().dirpath()
parent.chdir()
)
if PYTEST_GTE_7:
parent = modcol.path.parent.parent # type: ignore[attr-defined]
else:
parent = Path(modcol.fspath.dirpath().dirpath())
monkeypatch.chdir(parent)
modcol.config.args = [
py.path.local(x).relto(parent) for x in modcol.config.args
str(Path(x).relative_to(parent)) for x in modcol.config.args
]
control = RemoteControl(modcol.config)
control.loop_once()
@@ -159,8 +186,9 @@ class TestRemoteControl:
class TestLooponFailing:
def test_looponfail_from_fail_to_ok(self, testdir):
modcol = testdir.getmodulecol(
def test_looponfail_from_fail_to_ok(self, pytester: pytest.Pytester) -> None:
modcol = pytester.getmodulecol(
textwrap.dedent(
"""
def test_one():
x = 0
@@ -169,12 +197,14 @@ class TestLooponFailing:
assert 1
"""
)
)
remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1
modcol.fspath.write(
py.code.Source(
modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
modcol_path.write_text(
textwrap.dedent(
"""
def test_one():
assert 1
@@ -183,24 +213,27 @@ class TestLooponFailing:
"""
)
)
removepyc(modcol.fspath)
removepyc(modcol_path)
remotecontrol.loop_once()
assert not remotecontrol.failures
def test_looponfail_from_one_to_two_tests(self, testdir):
modcol = testdir.getmodulecol(
def test_looponfail_from_one_to_two_tests(self, pytester: pytest.Pytester) -> None:
modcol = pytester.getmodulecol(
textwrap.dedent(
"""
def test_one():
assert 0
"""
)
)
remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1
assert "test_one" in remotecontrol.failures[0]
modcol.fspath.write(
py.code.Source(
modcol_path = modcol.path if PYTEST_GTE_7 else Path(modcol.fspath)
modcol_path.write_text(
textwrap.dedent(
"""
def test_one():
assert 1 # passes now
@@ -209,7 +242,7 @@ class TestLooponFailing:
"""
)
)
removepyc(modcol.fspath)
removepyc(modcol_path)
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 0
remotecontrol.loop_once()
@@ -217,13 +250,10 @@ class TestLooponFailing:
assert "test_one" not in remotecontrol.failures[0]
assert "test_two" in remotecontrol.failures[0]
@pytest.mark.xfail(
parse_version(pytest.__version__) >= parse_version("3.1"),
reason="broken by pytest 3.1+",
strict=True,
)
def test_looponfail_removed_test(self, testdir):
modcol = testdir.getmodulecol(
@pytest.mark.xfail(reason="broken by pytest 3.1+", strict=True)
def test_looponfail_removed_test(self, pytester: pytest.Pytester) -> None:
modcol = pytester.getmodulecol(
textwrap.dedent(
"""
def test_one():
assert 0
@@ -231,12 +261,13 @@ class TestLooponFailing:
assert 0
"""
)
)
remotecontrol = RemoteControl(modcol.config)
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 2
modcol.fspath.write(
py.code.Source(
modcol.path.write_text(
textwrap.dedent(
"""
def test_xxx(): # renamed test
assert 0
@@ -245,20 +276,24 @@ class TestLooponFailing:
"""
)
)
removepyc(modcol.fspath)
removepyc(modcol.path)
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 0
remotecontrol.loop_once()
assert len(remotecontrol.failures) == 1
def test_looponfail_multiple_errors(self, testdir, monkeypatch):
modcol = testdir.getmodulecol(
def test_looponfail_multiple_errors(
self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
modcol = pytester.getmodulecol(
textwrap.dedent(
"""
def test_one():
assert 0
"""
)
)
remotecontrol = RemoteControl(modcol.config)
orig_runsession = remotecontrol.runsession
@@ -274,55 +309,59 @@ class TestLooponFailing:
class TestFunctional:
def test_fail_to_ok(self, testdir):
p = testdir.makepyfile(
def test_fail_to_ok(self, pytester: pytest.Pytester) -> None:
p = pytester.makepyfile(
textwrap.dedent(
"""
def test_one():
x = 0
assert x == 1
"""
)
# p = testdir.mkdir("sub").join(p1.basename)
)
# p = pytester.mkdir("sub").join(p1.basename)
# p1.move(p)
child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
child = pytester.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
child.expect("def test_one")
child.expect("x == 1")
child.expect("1 failed")
child.expect("### LOOPONFAILING ####")
child.expect("waiting for changes")
p.write(
py.code.Source(
p.write_text(
textwrap.dedent(
"""
def test_one():
x = 1
assert x == 1
"""
)
),
)
child.expect(".*1 passed.*")
child.kill(15)
def test_xfail_passes(self, testdir):
p = testdir.makepyfile(
def test_xfail_passes(self, pytester: pytest.Pytester) -> None:
p = pytester.makepyfile(
textwrap.dedent(
"""
import py
@py.test.mark.xfail
import pytest
@pytest.mark.xfail
def test_one():
pass
"""
)
child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
)
child = pytester.spawn_pytest("-f %s" % p, expect_timeout=30.0)
child.expect("1 xpass")
# child.expect("### LOOPONFAILING ####")
child.expect("waiting for changes")
child.kill(15)
def removepyc(path):
def removepyc(path: Path) -> None:
# XXX damn those pyc files
pyc = path + "c"
if pyc.check():
pyc.remove()
c = path.dirpath("__pycache__")
if c.check():
c.remove()
pyc = path.with_suffix(".pyc")
if pyc.exists():
pyc.unlink()
c = path.parent / "__pycache__"
if c.exists():
shutil.rmtree(c)

26
testing/test_newhooks.py
View File

@@ -3,8 +3,8 @@ import pytest
class TestHooks:
@pytest.fixture(autouse=True)
def create_test_file(self, testdir):
testdir.makepyfile(
def create_test_file(self, pytester: pytest.Pytester) -> None:
pytester.makepyfile(
"""
import os
def test_a(): pass
@@ -13,11 +13,11 @@ class TestHooks:
"""
)
def test_runtest_logreport(self, testdir):
def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
"""Test that log reports from pytest_runtest_logreport when running
with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8)
"""
testdir.makeconftest(
pytester.makeconftest(
"""
def pytest_runtest_logreport(report):
if hasattr(report, 'node'):
@@ -35,7 +35,7 @@ class TestHooks:
% (report.nodeid, report.worker_id, report.testrun_uid))
"""
)
res = testdir.runpytest("-n1", "-s")
res = pytester.runpytest("-n1", "-s")
res.stdout.fnmatch_lines(
[
"*HOOK: test_runtest_logreport.py::test_a gw0 *",
@@ -45,9 +45,9 @@ class TestHooks:
]
)
def test_node_collection_finished(self, testdir):
def test_node_collection_finished(self, pytester: pytest.Pytester) -> None:
"""Test pytest_xdist_node_collection_finished hook (#8)."""
testdir.makeconftest(
pytester.makeconftest(
"""
def pytest_xdist_node_collection_finished(node, ids):
workerid = node.workerinput['workerid']
@@ -55,7 +55,7 @@ class TestHooks:
print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
"""
)
res = testdir.runpytest("-n2", "-s")
res = pytester.runpytest("-n2", "-s")
res.stdout.fnmatch_lines_random(
["*HOOK: gw0 test_a, test_b, test_c", "*HOOK: gw1 test_a, test_b, test_c"]
)
@@ -64,8 +64,8 @@ class TestHooks:
class TestCrashItem:
@pytest.fixture(autouse=True)
def create_test_file(self, testdir):
testdir.makepyfile(
def create_test_file(self, pytester: pytest.Pytester) -> None:
pytester.makepyfile(
"""
import os
def test_a(): pass
@@ -75,9 +75,9 @@ class TestCrashItem:
"""
)
def test_handlecrashitem(self, testdir):
def test_handlecrashitem(self, pytester: pytest.Pytester) -> None:
"""Test pytest_handlecrashitem hook."""
testdir.makeconftest(
pytester.makeconftest(
"""
test_runs = 0
@@ -91,6 +91,6 @@ class TestCrashItem:
print("HOOK: pytest_handlecrashitem")
"""
)
res = testdir.runpytest("-n2", "-s")
res = pytester.runpytest("-n2", "-s")
res.stdout.fnmatch_lines_random(["*HOOK: pytest_handlecrashitem"])
res.stdout.fnmatch_lines(["*3 passed*"])

110
testing/test_plugin.py
View File

@@ -1,44 +1,46 @@
from contextlib import suppress
from pathlib import Path
import py
import execnet
from xdist.workermanage import NodeManager
import pytest
def test_dist_incompatibility_messages(testdir):
result = testdir.runpytest("--pdb", "--looponfail")
def test_dist_incompatibility_messages(pytester: pytest.Pytester) -> None:
result = pytester.runpytest("--pdb", "--looponfail")
assert result.ret != 0
result = testdir.runpytest("--pdb", "-n", "3")
result = pytester.runpytest("--pdb", "-n", "3")
assert result.ret != 0
assert "incompatible" in result.stderr.str()
result = testdir.runpytest("--pdb", "-d", "--tx", "popen")
result = pytester.runpytest("--pdb", "-d", "--tx", "popen")
assert result.ret != 0
assert "incompatible" in result.stderr.str()
def test_dist_options(testdir):
def test_dist_options(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options
config = testdir.parseconfigure("-n 2")
config = pytester.parseconfigure("-n 2")
check_options(config)
assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("--numprocesses", "2")
config = pytester.parseconfigure("--numprocesses", "2")
check_options(config)
assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
config = pytester.parseconfigure("--numprocesses", "3", "--maxprocesses", "2")
check_options(config)
assert config.option.dist == "load"
assert config.option.tx == ["popen"] * 2
config = testdir.parseconfigure("-d")
config = pytester.parseconfigure("-d")
check_options(config)
assert config.option.dist == "load"
def test_auto_detect_cpus(testdir, monkeypatch):
def test_auto_detect_cpus(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
import os
from xdist.plugin import pytest_cmdline_main as check_options
@@ -56,20 +58,20 @@ def test_auto_detect_cpus(testdir, monkeypatch):
monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
config = testdir.parseconfigure("-n2")
config = pytester.parseconfigure("-n2")
assert config.getoption("numprocesses") == 2
config = testdir.parseconfigure("-nauto")
config = pytester.parseconfigure("-nauto")
check_options(config)
assert config.getoption("numprocesses") == 99
config = testdir.parseconfigure("-nauto", "--pdb")
config = pytester.parseconfigure("-nauto", "--pdb")
check_options(config)
assert config.getoption("usepdb")
assert config.getoption("numprocesses") == 0
assert config.getoption("dist") == "no"
config = testdir.parseconfigure("-nlogical", "--pdb")
config = pytester.parseconfigure("-nlogical", "--pdb")
check_options(config)
assert config.getoption("usepdb")
assert config.getoption("numprocesses") == 0
@@ -77,91 +79,95 @@ def test_auto_detect_cpus(testdir, monkeypatch):
monkeypatch.delattr(os, "sched_getaffinity", raising=False)
monkeypatch.setenv("TRAVIS", "true")
config = testdir.parseconfigure("-nauto")
config = pytester.parseconfigure("-nauto")
check_options(config)
assert config.getoption("numprocesses") == 2
def test_auto_detect_cpus_psutil(testdir, monkeypatch):
def test_auto_detect_cpus_psutil(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
from xdist.plugin import pytest_cmdline_main as check_options
psutil = pytest.importorskip("psutil")
monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 84 if logical else 42)
config = testdir.parseconfigure("-nauto")
config = pytester.parseconfigure("-nauto")
check_options(config)
assert config.getoption("numprocesses") == 42
config = testdir.parseconfigure("-nlogical")
config = pytester.parseconfigure("-nlogical")
check_options(config)
assert config.getoption("numprocesses") == 84
def test_hook_auto_num_workers(testdir, monkeypatch):
def test_hook_auto_num_workers(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
) -> None:
from xdist.plugin import pytest_cmdline_main as check_options
testdir.makeconftest(
pytester.makeconftest(
"""
def pytest_xdist_auto_num_workers():
return 42
"""
)
config = testdir.parseconfigure("-nauto")
config = pytester.parseconfigure("-nauto")
check_options(config)
assert config.getoption("numprocesses") == 42
config = testdir.parseconfigure("-nlogical")
config = pytester.parseconfigure("-nlogical")
check_options(config)
assert config.getoption("numprocesses") == 42
def test_boxed_with_collect_only(testdir):
def test_boxed_with_collect_only(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options
config = testdir.parseconfigure("-n1", "--boxed")
config = pytester.parseconfigure("-n1", "--boxed")
check_options(config)
assert config.option.forked
config = testdir.parseconfigure("-n1", "--collect-only")
config = pytester.parseconfigure("-n1", "--collect-only")
check_options(config)
assert not config.option.forked
config = testdir.parseconfigure("-n1", "--boxed", "--collect-only")
config = pytester.parseconfigure("-n1", "--boxed", "--collect-only")
check_options(config)
assert config.option.forked
def test_dsession_with_collect_only(testdir):
def test_dsession_with_collect_only(pytester: pytest.Pytester) -> None:
from xdist.plugin import pytest_cmdline_main as check_options
from xdist.plugin import pytest_configure as configure
config = testdir.parseconfigure("-n1")
config = pytester.parseconfigure("-n1")
check_options(config)
configure(config)
assert config.pluginmanager.hasplugin("dsession")
config = testdir.parseconfigure("-n1", "--collect-only")
config = pytester.parseconfigure("-n1", "--collect-only")
check_options(config)
configure(config)
assert not config.pluginmanager.hasplugin("dsession")
def test_testrunuid_provided(testdir):
config = testdir.parseconfigure("--testrunuid", "test123", "--tx=popen")
def test_testrunuid_provided(pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--testrunuid", "test123", "--tx=popen")
nm = NodeManager(config)
assert nm.testrunuid == "test123"
def test_testrunuid_generated(testdir):
config = testdir.parseconfigure("--tx=popen")
def test_testrunuid_generated(pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--tx=popen")
nm = NodeManager(config)
assert len(nm.testrunuid) == 32
class TestDistOptions:
def test_getxspecs(self, testdir):
config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
def test_getxspecs(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
nodemanager = NodeManager(config)
xspecs = nodemanager._getxspecs()
assert len(xspecs) == 2
@@ -169,39 +175,39 @@ class TestDistOptions:
assert xspecs[0].popen
assert xspecs[1].ssh == "xyz"
def test_xspecs_multiplied(self, testdir):
config = testdir.parseconfigure("--tx=3*popen")
def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--tx=3*popen")
xspecs = NodeManager(config)._getxspecs()
assert len(xspecs) == 3
assert xspecs[1].popen
def test_getrsyncdirs(self, testdir):
config = testdir.parseconfigure("--rsyncdir=" + str(testdir.tmpdir))
def test_getrsyncdirs(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--rsyncdir=" + str(pytester.path))
nm = NodeManager(config, specs=[execnet.XSpec("popen")])
assert not nm._getrsyncdirs()
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert nm.roots
assert testdir.tmpdir in nm.roots
assert pytester.path in nm.roots
def test_getrsyncignore(self, testdir):
config = testdir.parseconfigure("--rsyncignore=fo*")
def test_getrsyncignore(self, pytester: pytest.Pytester) -> None:
config = pytester.parseconfigure("--rsyncignore=fo*")
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert "fo*" in nm.rsyncoptions["ignores"]
def test_getrsyncdirs_with_conftest(self, testdir):
p = py.path.local()
for bn in "x y z".split():
p.mkdir(bn)
testdir.makeini(
def test_getrsyncdirs_with_conftest(self, pytester: pytest.Pytester) -> None:
p = Path.cwd()
for bn in ("x", "y", "z"):
p.joinpath(bn).mkdir()
pytester.makeini(
"""
[pytest]
rsyncdirs= x
"""
)
config = testdir.parseconfigure(testdir.tmpdir, "--rsyncdir=y", "--rsyncdir=z")
config = pytester.parseconfigure(pytester.path, "--rsyncdir=y", "--rsyncdir=z")
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
roots = nm._getrsyncdirs()
# assert len(roots) == 3 + 1 # pylib
assert py.path.local("y") in roots
assert py.path.local("z") in roots
assert testdir.tmpdir.join("x") in roots
assert Path("y").resolve() in roots
assert Path("z").resolve() in roots
assert pytester.path.joinpath("x") in roots

92
testing/test_remote.py
View File

@@ -1,5 +1,5 @@
import py
import pprint
import py
import pytest
import sys
import uuid
@@ -32,18 +32,16 @@ class EventCall:
class WorkerSetup:
use_callback = False
def __init__(self, request, testdir):
def __init__(self, request, pytester: pytest.Pytester) -> None:
self.request = request
self.testdir = testdir
self.events = Queue()
self.pytester = pytester
self.events = Queue() # type: ignore[var-annotated]
def setup(
self,
):
self.testdir.chdir()
def setup(self) -> None:
self.pytester.chdir()
# import os ; os.environ['EXECNET_DEBUG'] = "2"
self.gateway = execnet.makegateway()
self.config = config = self.testdir.parseconfigure()
self.config = config = self.pytester.parseconfigure()
putevent = self.use_callback and self.events.put or None
class DummyMananger:
@@ -70,15 +68,15 @@ class WorkerSetup:
@pytest.fixture
def worker(request, testdir):
return WorkerSetup(request, testdir)
def worker(request, pytester: pytest.Pytester) -> WorkerSetup:
return WorkerSetup(request, pytester)
@pytest.mark.xfail(reason="#59")
def test_remoteinitconfig(testdir):
def test_remoteinitconfig(pytester: pytest.Pytester) -> None:
from xdist.remote import remote_initconfig
config1 = testdir.parseconfig()
config1 = pytester.parseconfig()
config2 = remote_initconfig(config1.option.__dict__, config1.args)
assert config2.option.__dict__ == config1.option.__dict__
assert config2.pluginmanager.getplugin("terminal") in (-1, None)
@@ -94,8 +92,10 @@ class TestWorkerInteractor:
return unserialize
def test_basic_collect_and_runtests(self, worker, unserialize_report):
worker.testdir.makepyfile(
def test_basic_collect_and_runtests(
self, worker: WorkerSetup, unserialize_report
) -> None:
worker.pytester.makepyfile(
"""
def test_func():
pass
@@ -108,7 +108,7 @@ class TestWorkerInteractor:
assert ev.name == "collectionstart"
assert not ev.kwargs
ev = worker.popevent("collectionfinish")
assert ev.kwargs["topdir"] == worker.testdir.tmpdir
assert ev.kwargs["topdir"] == py.path.local(worker.pytester.path)
ids = ev.kwargs["ids"]
assert len(ids) == 1
worker.sendcommand("runtests", indices=list(range(len(ids))))
@@ -126,8 +126,8 @@ class TestWorkerInteractor:
ev = worker.popevent("workerfinished")
assert "workeroutput" in ev.kwargs
def test_remote_collect_skip(self, worker, unserialize_report):
worker.testdir.makepyfile(
def test_remote_collect_skip(self, worker: WorkerSetup, unserialize_report) -> None:
worker.pytester.makepyfile(
"""
import pytest
pytest.skip("hello", allow_module_level=True)
@@ -144,8 +144,8 @@ class TestWorkerInteractor:
ev = worker.popevent("collectionfinish")
assert not ev.kwargs["ids"]
def test_remote_collect_fail(self, worker, unserialize_report):
worker.testdir.makepyfile("""aasd qwe""")
def test_remote_collect_fail(self, worker: WorkerSetup, unserialize_report) -> None:
worker.pytester.makepyfile("""aasd qwe""")
worker.setup()
ev = worker.popevent("collectionstart")
assert not ev.kwargs
@@ -156,8 +156,8 @@ class TestWorkerInteractor:
ev = worker.popevent("collectionfinish")
assert not ev.kwargs["ids"]
def test_runtests_all(self, worker, unserialize_report):
worker.testdir.makepyfile(
def test_runtests_all(self, worker: WorkerSetup, unserialize_report) -> None:
worker.pytester.makepyfile(
"""
def test_func(): pass
def test_func2(): pass
@@ -183,17 +183,19 @@ class TestWorkerInteractor:
ev = worker.popevent("workerfinished")
assert "workeroutput" in ev.kwargs
def test_happy_run_events_converted(self, testdir, worker):
py.test.xfail("implement a simple test for event production")
assert not worker.use_callback
worker.testdir.makepyfile(
def test_happy_run_events_converted(
self, pytester: pytest.Pytester, worker: WorkerSetup
) -> None:
pytest.xfail("implement a simple test for event production")
assert not worker.use_callback # type: ignore[unreachable]
worker.pytester.makepyfile(
"""
def test_func():
pass
"""
)
worker.setup()
hookrec = testdir.getreportrecorder(worker.config)
hookrec = pytester.getreportrecorder(worker.config)
for data in worker.slp.channel:
worker.slp.process_from_remote(data)
worker.slp.process_from_remote(worker.slp.ENDMARK)
@@ -209,7 +211,9 @@ class TestWorkerInteractor:
]
)
def test_process_from_remote_error_handling(self, worker, capsys):
def test_process_from_remote_error_handling(
self, worker: WorkerSetup, capsys: pytest.CaptureFixture[str]
) -> None:
worker.use_callback = True
worker.setup()
worker.slp.process_from_remote(("<nonono>", ()))
@@ -219,8 +223,8 @@ class TestWorkerInteractor:
assert ev.name == "errordown"
def test_remote_env_vars(testdir):
testdir.makepyfile(
def test_remote_env_vars(pytester: pytest.Pytester) -> None:
pytester.makepyfile(
"""
import os
def test():
@@ -229,13 +233,13 @@ def test_remote_env_vars(testdir):
assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
"""
)
result = testdir.runpytest("-n2", "--max-worker-restart=0")
result = pytester.runpytest("-n2", "--max-worker-restart=0")
assert result.ret == 0
def test_remote_inner_argv(testdir):
def test_remote_inner_argv(pytester: pytest.Pytester) -> None:
"""Test/document the behavior due to execnet using `python -c`."""
testdir.makepyfile(
pytester.makepyfile(
"""
import sys
@@ -243,14 +247,14 @@ def test_remote_inner_argv(testdir):
assert sys.argv == ["-c"]
"""
)
result = testdir.runpytest("-n1")
result = pytester.runpytest("-n1")
assert result.ret == 0
def test_remote_mainargv(testdir):
def test_remote_mainargv(pytester: pytest.Pytester) -> None:
outer_argv = sys.argv
testdir.makepyfile(
pytester.makepyfile(
"""
def test_mainargv(request):
assert request.config.workerinput["mainargv"] == {!r}
@@ -258,14 +262,14 @@ def test_remote_mainargv(testdir):
outer_argv
)
)
result = testdir.runpytest("-n1")
result = pytester.runpytest("-n1")
assert result.ret == 0
def test_remote_usage_prog(testdir, request):
def test_remote_usage_prog(pytester: pytest.Pytester, request) -> None:
if not hasattr(request.config._parser, "prog"):
pytest.skip("prog not available in config parser")
testdir.makeconftest(
pytester.makeconftest(
"""
import pytest
@@ -280,7 +284,7 @@ def test_remote_usage_prog(testdir, request):
config_parser = config._parser
"""
)
testdir.makepyfile(
pytester.makepyfile(
"""
import sys
@@ -289,14 +293,14 @@ def test_remote_usage_prog(testdir, request):
"""
)
result = testdir.runpytest_subprocess("-n1")
result = pytester.runpytest_subprocess("-n1")
assert result.ret == 1
result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])
def test_remote_sys_path(testdir):
def test_remote_sys_path(pytester: pytest.Pytester) -> None:
"""Work around sys.path differences due to execnet using `python -c`."""
testdir.makepyfile(
pytester.makepyfile(
"""
import sys
@@ -304,5 +308,5 @@ def test_remote_sys_path(testdir):
assert "" not in sys.path
"""
)
result = testdir.runpytest("-n1")
result = pytester.runpytest("-n1")
assert result.ret == 0

259
testing/test_workermanage.py
View File

@@ -1,39 +1,42 @@
import execnet
import py
import pytest
import shutil
import textwrap
import execnet
from _pytest.pytester import HookRecorder
from xdist import workermanage, newhooks
from pathlib import Path
from xdist import workermanage
from xdist.workermanage import HostRSync, NodeManager
pytest_plugins = "pytester"
@pytest.fixture
def hookrecorder(request, config):
hookrecorder = HookRecorder(config.pluginmanager)
if hasattr(hookrecorder, "start_recording"):
hookrecorder.start_recording(newhooks)
request.addfinalizer(hookrecorder.finish_recording)
def hookrecorder(request, config, pytester: pytest.Pytester):
hookrecorder = pytester.make_hook_recorder(config.pluginmanager)
return hookrecorder
@pytest.fixture
def config(testdir):
return testdir.parseconfig()
def config(pytester: pytest.Pytester):
return pytester.parseconfig()
@pytest.fixture
def mysetup(tmpdir):
class mysetup:
source = tmpdir.mkdir("source")
dest = tmpdir.mkdir("dest")
return mysetup()
def source(tmp_path: Path) -> Path:
source = tmp_path / "source"
source.mkdir()
return source
@pytest.fixture
def workercontroller(monkeypatch):
def dest(tmp_path: Path) -> Path:
dest = tmp_path / "dest"
dest.mkdir()
return dest
@pytest.fixture
def workercontroller(monkeypatch: pytest.MonkeyPatch):
class MockController:
def __init__(self, *args):
pass
@@ -46,18 +49,20 @@ def workercontroller(monkeypatch):
class TestNodeManagerPopen:
def test_popen_no_default_chdir(self, config):
def test_popen_no_default_chdir(self, config) -> None:
gm = NodeManager(config, ["popen"])
assert gm.specs[0].chdir is None
def test_default_chdir(self, config):
def test_default_chdir(self, config) -> None:
specs = ["ssh=noco", "socket=xyz"]
for spec in NodeManager(config, specs).specs:
assert spec.chdir == "pyexecnetcache"
for spec in NodeManager(config, specs, defaultchdir="abc").specs:
assert spec.chdir == "abc"
def test_popen_makegateway_events(self, config, hookrecorder, workercontroller):
def test_popen_makegateway_events(
self, config, hookrecorder, workercontroller
) -> None:
hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None)
call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -72,15 +77,16 @@ class TestNodeManagerPopen:
hm.teardown_nodes()
assert not len(hm.group)
def test_popens_rsync(self, config, mysetup, workercontroller):
source = mysetup.source
def test_popens_rsync(
self, config, source: Path, dest: Path, workercontroller
) -> None:
hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None)
assert len(hm.group) == 2
for gw in hm.group:
class pseudoexec:
args = []
args = [] # type: ignore[var-annotated]
def __init__(self, *args):
self.args.extend(args)
@@ -97,30 +103,37 @@ class TestNodeManagerPopen:
assert not len(hm.group)
assert "sys.path.insert" in gw.remote_exec.args[0]
def test_rsync_popen_with_path(self, config, mysetup, workercontroller):
source, dest = mysetup.source, mysetup.dest
def test_rsync_popen_with_path(
self, config, source: Path, dest: Path, workercontroller
) -> None:
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello")
source.joinpath("dir1", "dir2").mkdir(parents=True)
source.joinpath("dir1", "dir2", "hello").touch()
notifications = []
for gw in hm.group:
hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
assert len(notifications) == 1
assert notifications[0] == ("rsyncrootready", hm.group["gw0"].spec, source)
hm.teardown_nodes()
dest = dest.join(source.basename)
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", "hello").check()
dest = dest.joinpath(source.name)
assert dest.joinpath("dir1").exists()
assert dest.joinpath("dir1", "dir2").exists()
assert dest.joinpath("dir1", "dir2", "hello").exists()
def test_rsync_same_popen_twice(
self, config, mysetup, hookrecorder, workercontroller
):
source, dest = mysetup.source, mysetup.dest
self,
config,
source: Path,
dest: Path,
hookrecorder,
workercontroller,
) -> None:
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
hm.roots = []
hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello")
source.joinpath("dir1", "dir2").mkdir(parents=True)
source.joinpath("dir1", "dir2", "hello").touch()
gw = hm.group[0]
hm.rsync(gw, source)
call = hookrecorder.popcall("pytest_xdist_rsyncstart")
@@ -131,83 +144,98 @@ class TestNodeManagerPopen:
class TestHRSync:
def test_hrsync_filter(self, mysetup):
source, _ = mysetup.source, mysetup.dest # noqa
source.ensure("dir", "file.txt")
source.ensure(".svn", "entries")
source.ensure(".somedotfile", "moreentries")
source.ensure("somedir", "editfile~")
def test_hrsync_filter(self, source: Path, dest: Path) -> None:
source.joinpath("dir").mkdir()
source.joinpath("dir", "file.txt").touch()
source.joinpath(".svn").mkdir()
source.joinpath(".svn", "entries").touch()
source.joinpath(".somedotfile").mkdir()
source.joinpath(".somedotfile", "moreentries").touch()
source.joinpath("somedir").mkdir()
source.joinpath("somedir", "editfile~").touch()
syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
files = list(source.visit(rec=syncer.filter, fil=syncer.filter))
files = list(py.path.local(source).visit(rec=syncer.filter, fil=syncer.filter))
assert len(files) == 3
basenames = [x.basename for x in files]
assert "dir" in basenames
assert "file.txt" in basenames
assert "somedir" in basenames
def test_hrsync_one_host(self, mysetup):
source, dest = mysetup.source, mysetup.dest
def test_hrsync_one_host(self, source: Path, dest: Path) -> None:
gw = execnet.makegateway("popen//chdir=%s" % dest)
finished = []
rsync = HostRSync(source)
rsync.add_target_host(gw, finished=lambda: finished.append(1))
source.join("hello.py").write("world")
source.joinpath("hello.py").write_text("world")
rsync.send()
gw.exit()
assert dest.join(source.basename, "hello.py").check()
assert dest.joinpath(source.name, "hello.py").exists()
assert len(finished) == 1
class TestNodeManager:
@py.test.mark.xfail(run=False)
def test_rsync_roots_no_roots(self, testdir, mysetup):
mysetup.source.ensure("dir1", "file1").write("hello")
config = testdir.parseconfig(mysetup.source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
@pytest.mark.xfail(run=False)
def test_rsync_roots_no_roots(
self, pytester: pytest.Pytester, source: Path, dest: Path
) -> None:
source.joinpath("dir1").mkdir()
source.joinpath("dir1", "file1").write_text("hello")
config = pytester.parseconfig(source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
# assert nodemanager.config.topdir == source == config.topdir
nodemanager.makegateways()
nodemanager.rsync_roots()
(p,) = nodemanager.gwmanager.multi_exec(
nodemanager.makegateways() # type: ignore[attr-defined]
nodemanager.rsync_roots() # type: ignore[call-arg]
(p,) = nodemanager.gwmanager.multi_exec( # type: ignore[attr-defined]
"import os ; channel.send(os.getcwd())"
).receive_each()
p = py.path.local(p)
p = Path(p)
print("remote curdir", p)
assert p == mysetup.dest.join(config.topdir.basename)
assert p.join("dir1").check()
assert p.join("dir1", "file1").check()
assert p == dest.joinpath(config.rootpath.name)
assert p.joinpath("dir1").check()
assert p.joinpath("dir1", "file1").check()
def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller):
source, dest = mysetup.source, mysetup.dest
dir1 = mysetup.source.mkdir("dir1")
dir2 = dir1.mkdir("dir2")
dir2.ensure("hello")
def test_popen_rsync_subdir(
self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
) -> None:
dir1 = source / "dir1"
dir1.mkdir()
dir2 = dir1 / "dir2"
dir2.mkdir()
dir2.joinpath("hello").touch()
for rsyncroot in (dir1, source):
dest.remove()
shutil.rmtree(str(dest), ignore_errors=True)
nodemanager = NodeManager(
testdir.parseconfig(
pytester.parseconfig(
"--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source
)
)
nodemanager.setup_nodes(None) # calls .rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", "hello").check()
dest = dest.joinpath("source")
assert dest.joinpath("dir1").exists()
assert dest.joinpath("dir1", "dir2").exists()
assert dest.joinpath("dir1", "dir2", "hello").exists()
nodemanager.teardown_nodes()
@pytest.mark.parametrize(
"flag, expects_report", [("-q", False), ("", False), ("-v", True)]
)
def test_rsync_report(
self, testdir, mysetup, workercontroller, capsys, flag, expects_report
):
source, dest = mysetup.source, mysetup.dest
dir1 = mysetup.source.mkdir("dir1")
args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source
self,
pytester: pytest.Pytester,
source: Path,
dest: Path,
workercontroller,
capsys: pytest.CaptureFixture[str],
flag: str,
expects_report: bool,
) -> None:
dir1 = source / "dir1"
dir1.mkdir()
args = ["--tx", "popen//chdir=%s" % dest, "--rsyncdir", str(dir1), str(source)]
if flag:
args += (flag,)
nodemanager = NodeManager(testdir.parseconfig(*args))
args.append(flag)
nodemanager = NodeManager(pytester.parseconfig(*args))
nodemanager.setup_nodes(None) # calls .rsync_roots()
out, _ = capsys.readouterr()
if expects_report:
@@ -215,13 +243,16 @@ class TestNodeManager:
else:
assert "<= pytest/__init__.py" not in out
def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir1", "somefile", dir=1)
dir2.ensure("hello")
source.ensure("bogusdir", "file")
source.join("tox.ini").write(
def test_init_rsync_roots(
self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
) -> None:
dir2 = source.joinpath("dir1", "dir2")
dir2.mkdir(parents=True)
source.joinpath("dir1", "somefile").mkdir()
dir2.joinpath("hello").touch()
source.joinpath("bogusdir").mkdir()
source.joinpath("bogusdir", "file").touch()
source.joinpath("tox.ini").write_text(
textwrap.dedent(
"""
[pytest]
@@ -229,22 +260,27 @@ class TestNodeManager:
"""
)
)
config = testdir.parseconfig(source)
config = pytester.parseconfig(source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir2").check()
assert not dest.join("dir1").check()
assert not dest.join("bogus").check()
assert dest.joinpath("dir2").exists()
assert not dest.joinpath("dir1").exists()
assert not dest.joinpath("bogus").exists()
def test_rsyncignore(self, testdir, mysetup, workercontroller):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir5", "dir6", "bogus")
source.ensure("dir5", "file")
dir2.ensure("hello")
source.ensure("foo", "bar")
source.ensure("bar", "foo")
source.join("tox.ini").write(
def test_rsyncignore(
self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
) -> None:
dir2 = source.joinpath("dir1", "dir2")
dir2.mkdir(parents=True)
source.joinpath("dir5", "dir6").mkdir(parents=True)
source.joinpath("dir5", "dir6", "bogus").touch()
source.joinpath("dir5", "file").touch()
dir2.joinpath("hello").touch()
source.joinpath("foo").mkdir()
source.joinpath("foo", "bar").touch()
source.joinpath("bar").mkdir()
source.joinpath("bar", "foo").touch()
source.joinpath("tox.ini").write_text(
textwrap.dedent(
"""
[pytest]
@@ -253,39 +289,40 @@ class TestNodeManager:
"""
)
)
config = testdir.parseconfig(source)
config = pytester.parseconfig(source)
config.option.rsyncignore = ["bar"]
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir1").check()
assert not dest.join("dir1", "dir2").check()
assert dest.join("dir5", "file").check()
assert not dest.join("dir6").check()
assert not dest.join("foo").check()
assert not dest.join("bar").check()
assert dest.joinpath("dir1").exists()
assert not dest.joinpath("dir1", "dir2").exists()
assert dest.joinpath("dir5", "file").exists()
assert not dest.joinpath("dir6").exists()
assert not dest.joinpath("foo").exists()
assert not dest.joinpath("bar").exists()
def test_optimise_popen(self, testdir, mysetup, workercontroller):
source = mysetup.source
def test_optimise_popen(
self, pytester: pytest.Pytester, source: Path, dest: Path, workercontroller
) -> None:
specs = ["popen"] * 3
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure("a", dir=1)
config = testdir.parseconfig(source)
source.joinpath("conftest.py").write_text("rsyncdirs = ['a']")
source.joinpath("a").mkdir()
config = pytester.parseconfig(source)
nodemanager = NodeManager(config, specs)
nodemanager.setup_nodes(None) # calls .rysnc_roots()
for gwspec in nodemanager.specs:
assert gwspec._samefilesystem()
assert not gwspec.chdir
def test_ssh_setup_nodes(self, specssh, testdir):
testdir.makepyfile(
def test_ssh_setup_nodes(self, specssh: str, pytester: pytest.Pytester) -> None:
pytester.makepyfile(
__init__="",
test_x="""
def test_one():
pass
""",
)
reprec = testdir.inline_run(
"-d", "--rsyncdir=%s" % testdir.tmpdir, "--tx", specssh, testdir.tmpdir
reprec = pytester.inline_run(
"-d", "--rsyncdir=%s" % pytester.path, "--tx", specssh, pytester.path
)
(rep,) = reprec.getreports("pytest_runtest_logreport")
assert rep.passed

1
tox.ini
View File

@@ -18,7 +18,6 @@ commands=
extras =
testing
psutil
deps = pytest
commands =
pytest {posargs:-k psutil}