From f23c717a36a21d9a3d78b9c61aeb7660b1bbc666 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Fri, 12 Jun 2020 17:59:07 +1000 Subject: [PATCH 01/10] Delete old issues file --- ISSUES.txt | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 ISSUES.txt diff --git a/ISSUES.txt b/ISSUES.txt deleted file mode 100644 index 58cf69d..0000000 --- a/ISSUES.txt +++ /dev/null @@ -1,31 +0,0 @@ -next release critical ------------------------------------------------ -tag: bug - -miserably fails: --dist=each --tx popen --tx socket=... - - -rename / hooks ------------------------------------------------ -tag: bug - -node -> slave -transition for hooks? -configure_node -> configure_slave - -allow to remotely run xdist tests with xdist ------------------------------------------------ -tag: feature - -allow to run xdist own tests using its own mechanism. -currently this doesn't work because the remote side -has no pytest plugin. How to configure/do -register "xdist.plugin" on the remote side? - -see to avoid any "from _pytest" internal imports ------------------------------------------------ -tag: feature - -currently tests and even xdist core code imports -names from the internal _pytest namespace. -See to avoid it. From de3e54fd278d49b9b5d64e64f5942512519545e5 Mon Sep 17 00:00:00 2001 From: Zac-HD Date: Fri, 12 Jun 2020 18:00:28 +1000 Subject: [PATCH 02/10] Finish removal of "slave" terminology See issue #234. It's time. --- .appveyor.yml | 1 - .travis.yml | 2 -- changelog/541.removal.rst | 1 + setup.py | 16 ++++------------ src/xdist/dsession.py | 3 --- src/xdist/plugin.py | 5 +---- src/xdist/remote.py | 3 --- src/xdist/workermanage.py | 6 +----- testing/acceptance_test.py | 16 ---------------- tox.ini | 2 +- 10 files changed, 8 insertions(+), 47 deletions(-) create mode 100644 changelog/541.removal.rst diff --git a/.appveyor.yml b/.appveyor.yml index 36c98c1..0c9cef7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,6 +1,5 @@ environment: matrix: - - TOXENV: "py27-pytestlatest" - TOXENV: "py35-pytestlatest" - TOXENV: "py36-pytestlatest" - TOXENV: "py37-pytestlatest" diff --git a/.travis.yml b/.travis.yml index 8932051..79afe10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,8 +32,6 @@ jobs: - $HOME/.cache/pre-commit - python: '3.8' env: TOXENV=py38-pytestlatest - - python: '2.7' - env: TOXENV=py27-pytestlatest - stage: test python: "3.5" diff --git a/changelog/541.removal.rst b/changelog/541.removal.rst new file mode 100644 index 0000000..0823a67 --- /dev/null +++ b/changelog/541.removal.rst @@ -0,0 +1 @@ +drop backward-compatibility "slave" aliases related to worker nodes. We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process. diff --git a/setup.py b/setup.py index ad4675c..fcba298 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,6 @@ from setuptools import setup, find_packages -install_requires = [ - "execnet>=1.1", - "psutil>=3.0.0", - "pytest>=4.4.0", - "pytest-forked", - "six", -] +install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=6.0.0", "pytest-forked"] with open("README.rst") as f: @@ -15,8 +9,7 @@ with open("README.rst") as f: setup( name="pytest-xdist", use_scm_version={"write_to": "src/xdist/_version.py"}, - description="pytest xdist plugin for distributed testing" - " and loop-on-failing modes", + description="pytest xdist plugin for distributed testing and loop-on-failing modes", long_description=long_description, license="MIT", author="holger krekel and contributors", @@ -30,7 +23,7 @@ setup( "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"] }, zip_safe=False, - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", + python_requires=">=3.5", install_requires=install_requires, setup_requires=["setuptools_scm"], classifiers=[ @@ -45,9 +38,8 @@ setup( "Topic :: Software Development :: Quality Assurance", "Topic :: Utilities", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 2f06cb1..7a84c8b 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -150,9 +150,6 @@ class DSession(object): node.workerinfo["id"] = node.gateway.id node.workerinfo["spec"] = node.gateway.spec - # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo - node.slaveinfo = node.workerinfo - self.config.hook.pytest_testnodeready(node=node) if self.shuttingdown: node.shutdown() diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 162f4c9..29062d9 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -43,14 +43,11 @@ def pytest_addoption(parser): ) group.addoption( "--max-worker-restart", - "--max-slave-restart", action="store", default=None, dest="maxworkerrestart", help="maximum number of workers that can be restarted " - "when crashed (set to zero to disable this feature)\n" - "'--max-slave-restart' option is deprecated and will be removed in " - "a future release", + "when crashed (set to zero to disable this feature)", ) group.addoption( "--dist", diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 86ba9cb..088ebeb 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -266,8 +266,5 @@ if __name__ == "__channelexec__": config._parser.prog = os.path.basename(workerinput["mainargv"][0]) config.workerinput = workerinput config.workeroutput = {} - # TODO: deprecated name, backward compatibility only. Remove it in future - config.slaveinput = config.workerinput - config.slaveoutput = config.workeroutput interactor = WorkerInteractor(config, channel) config.hook.pytest_cmdline_main(config=config) diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index e8b49bb..69b1fbd 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -228,13 +228,9 @@ class WorkerController(object): self.workerinput = { "workerid": gateway.id, "workercount": len(nodemanager.specs), - "slaveid": gateway.id, - "slavecount": len(nodemanager.specs), "testrunuid": nodemanager.testrunuid, "mainargv": sys.argv, } - # TODO: deprecated name, backward compatibility only. Remove it in future - self.slaveinput = self.workerinput self._down = False self._shutdown_sent = False self.log = py.log.Producer("workerctl-%s" % gateway.id) @@ -333,7 +329,7 @@ class WorkerController(object): self.notify_inproc(eventname, node=self, **kwargs) elif eventname == "workerfinished": self._down = True - self.workeroutput = self.slaveoutput = kwargs["workeroutput"] + self.workeroutput = kwargs["workeroutput"] self.notify_inproc("workerfinished", node=self) elif eventname in ("logstart", "logfinish"): self.notify_inproc(eventname, node=self, **kwargs) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 7f9a551..168bb97 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -245,22 +245,6 @@ class TestDistribution: result.stderr.fnmatch_lines(["--foobar=123 active! *"]) assert dest.join(subdir.basename).check(dir=1) - def test_backward_compatibility_worker_terminology(self, testdir): - """Ensure that we still support "config.slaveinput" for backward compatibility (#234). - - Keep in mind that removing this compatibility will break a ton of plugins and user code. - """ - testdir.makepyfile( - """ - def test(pytestconfig): - assert hasattr(pytestconfig, 'slaveinput') - assert hasattr(pytestconfig, 'workerinput') - """ - ) - result = testdir.runpytest("-n1") - result.stdout.fnmatch_lines("*1 passed*") - assert result.ret == 0 - def test_data_exchange(self, testdir): testdir.makeconftest( """ diff --git a/tox.ini b/tox.ini index a68913a..584d3ed 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist= linting - py{27,35,36,37,38}-pytestlatest + py{35,36,37,38}-pytestlatest py38-pytestmaster [testenv] From 6f315356beca70ca0733f9bd646379e1785f53bf Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 11 Jul 2020 16:07:24 +0300 Subject: [PATCH 03/10] Set py35 as target version for Black --- .pre-commit-config.yaml | 2 +- src/xdist/report.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04d0ce3..c735868 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ repos: rev: 19.10b0 hooks: - id: black - args: [--safe, --quiet] + args: [--safe, --quiet, --target-version, py35] language_version: python3.7 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 diff --git a/src/xdist/report.py b/src/xdist/report.py index 3d239f1..1a132d0 100644 --- a/src/xdist/report.py +++ b/src/xdist/report.py @@ -13,9 +13,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id): diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id) error_message = ( - u"Different tests were collected between {from_id} and {to_id}. " - u"The difference is:\n" - u"{diff}" + "Different tests were collected between {from_id} and {to_id}. " + "The difference is:\n" + "{diff}" ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff)) msg = "\n".join([x.rstrip() for x in error_message.split("\n")]) return msg From 22ae7ff7af01fc3f23cb069725a64b569b6990dc Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 11 Jul 2020 16:09:19 +0300 Subject: [PATCH 04/10] Upgrade Python syntax with pyupgrade --py3-plus --- src/xdist/dsession.py | 4 ++-- src/xdist/looponfail.py | 7 +++---- src/xdist/remote.py | 2 +- src/xdist/report.py | 1 - src/xdist/scheduler/each.py | 2 +- src/xdist/scheduler/load.py | 2 +- src/xdist/scheduler/loadfile.py | 2 +- src/xdist/scheduler/loadscope.py | 2 +- src/xdist/workermanage.py | 13 +++++-------- testing/test_dsession.py | 2 +- testing/test_slavemanage.py | 3 +-- 11 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 7a84c8b..8ea245c 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -17,7 +17,7 @@ class Interrupted(KeyboardInterrupt): """ signals an immediate interruption. """ -class DSession(object): +class DSession: """A pytest plugin which runs a distributed test session At the beginning of the test session this creates a NodeManager @@ -333,7 +333,7 @@ class DSession(object): self.config.hook.pytest_runtest_logreport(report=rep) -class TerminalDistReporter(object): +class TerminalDistReporter: def __init__(self, config): self.config = config self.tr = config.pluginmanager.getplugin("terminalreporter") diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index 7f01ad6..19b9313 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -6,7 +6,6 @@ processes) otherwise changes to source code can crash the controlling process which should best never happen. """ -from __future__ import print_function import py import pytest import sys @@ -55,7 +54,7 @@ def looponfail_main(config): print() -class RemoteControl(object): +class RemoteControl: def __init__(self, config): self.config = config self.failures = [] @@ -165,7 +164,7 @@ def init_worker_session(channel, args, option_dict): WorkerFailSession(config, channel).main() -class WorkerFailSession(object): +class WorkerFailSession: def __init__(self, config, channel): self.config = config self.channel = channel @@ -220,7 +219,7 @@ class WorkerFailSession(object): self.channel.send((trails, failreports, self.collection_failed)) -class StatRecorder(object): +class StatRecorder: def __init__(self, rootdirlist): self.rootdirlist = rootdirlist self.statcache = {} diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 088ebeb..2df96fa 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -18,7 +18,7 @@ from execnet.gateway_base import dumps, DumpError from _pytest.config import _prepareconfig, Config -class WorkerInteractor(object): +class WorkerInteractor: def __init__(self, config, channel): self.config = config self.workerid = config.workerinput.get("workerid", "?") diff --git a/src/xdist/report.py b/src/xdist/report.py index 1a132d0..8843b40 100644 --- a/src/xdist/report.py +++ b/src/xdist/report.py @@ -1,4 +1,3 @@ -from __future__ import print_function from difflib import unified_diff diff --git a/src/xdist/scheduler/each.py b/src/xdist/scheduler/each.py index 486d8b0..b2a0442 100644 --- a/src/xdist/scheduler/each.py +++ b/src/xdist/scheduler/each.py @@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config from xdist.report import report_collection_diff -class EachScheduling(object): +class EachScheduling: """Implement scheduling of test items on all nodes If a node gets added after the test run is started then it is diff --git a/src/xdist/scheduler/load.py b/src/xdist/scheduler/load.py index 4116984..e378d9a 100644 --- a/src/xdist/scheduler/load.py +++ b/src/xdist/scheduler/load.py @@ -7,7 +7,7 @@ from xdist.workermanage import parse_spec_config from xdist.report import report_collection_diff -class LoadScheduling(object): +class LoadScheduling: """Implement load scheduling across nodes. This distributes the tests collected across all nodes so each test diff --git a/src/xdist/scheduler/loadfile.py b/src/xdist/scheduler/loadfile.py index 52a28b1..867a94e 100644 --- a/src/xdist/scheduler/loadfile.py +++ b/src/xdist/scheduler/loadfile.py @@ -21,7 +21,7 @@ class LoadFileScheduling(LoadScopeScheduling): """ def __init__(self, config, log=None): - super(LoadFileScheduling, self).__init__(config, log) + super().__init__(config, log) if log is None: self.log = Producer("loadfilesched") else: diff --git a/src/xdist/scheduler/loadscope.py b/src/xdist/scheduler/loadscope.py index 6b50a48..31dbe26 100644 --- a/src/xdist/scheduler/loadscope.py +++ b/src/xdist/scheduler/loadscope.py @@ -6,7 +6,7 @@ from xdist.report import report_collection_diff from xdist.workermanage import parse_spec_config -class LoadScopeScheduling(object): +class LoadScopeScheduling: """Implement load scheduling across nodes, but grouping test by scope. This distributes the tests collected across all nodes so each test is run diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index 69b1fbd..ec7f3df 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -1,4 +1,3 @@ -from __future__ import print_function import fnmatch import os import re @@ -29,7 +28,7 @@ def parse_spec_config(config): return xspeclist -class NodeManager(object): +class NodeManager: EXIT_TIMEOUT = 10 DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"] @@ -167,7 +166,7 @@ class HostRSync(execnet.RSync): self._ignores = [ re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores ] - super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs) + super().__init__(sourcedir=sourcedir, **kwargs) def filter(self, path): path = py.path.local(path) @@ -179,9 +178,7 @@ class HostRSync(execnet.RSync): def add_target_host(self, gateway, finished=None): remotepath = os.path.basename(self._sourcedir) - super(HostRSync, self).add_target( - gateway, remotepath, finishedcallback=finished, delete=True - ) + super().add_target(gateway, remotepath, finishedcallback=finished, delete=True) def _report_send_file(self, gateway, modified_rel_path): if self._verbose: @@ -211,7 +208,7 @@ def make_reltoroot(roots, args): return result -class WorkerController(object): +class WorkerController: ENDMARK = -1 class RemoteHook: @@ -292,7 +289,7 @@ class WorkerController(object): if not self._down: try: self.sendcommand("shutdown") - except (IOError, OSError): + except OSError: pass self._shutdown_sent = True diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 8d0373e..b015c75 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -203,7 +203,7 @@ class TestLoadScheduling: different test ids are collected by workers. """ - class CollectHook(object): + class CollectHook: """ Dummy hook that stores collection reports. """ diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index 47d2cba..f8abcca 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -1,4 +1,3 @@ -from __future__ import print_function import py import pytest import textwrap @@ -35,7 +34,7 @@ def mysetup(tmpdir): @pytest.fixture def workercontroller(monkeypatch): - class MockController(object): + class MockController: def __init__(self, *args): pass From 9decca0023ccdb48a210c3d3635d2b7082bd3660 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 11 Jul 2020 16:12:42 +0300 Subject: [PATCH 05/10] Drop support for EOL Python 2 --- testing/acceptance_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 168bb97..cc9b3ee 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1,6 +1,5 @@ import os import re -import sys import textwrap import py @@ -883,9 +882,6 @@ class TestWarnings: """Check that warnings with unserializable _WARNING_DETAILS are handled correctly (#379). """ - if sys.version_info[0] < 3: - # The issue is only present in Python 3 warnings - return testdir.makepyfile( """ import warnings, pytest From dddac7c7e9fcc176508592dd3dbd7d437336a885 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 11 Jul 2020 16:17:20 +0300 Subject: [PATCH 06/10] Remove six library --- src/xdist/dsession.py | 2 +- testing/conftest.py | 11 ----------- testing/test_remote.py | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 8ea245c..d7ccb8c 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -10,7 +10,7 @@ from xdist.scheduler import ( ) -from six.moves.queue import Empty, Queue +from queue import Empty, Queue class Interrupted(KeyboardInterrupt): diff --git a/testing/conftest.py b/testing/conftest.py index 43c0399..52f0308 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,20 +1,9 @@ -import six import py import pytest import execnet pytest_plugins = "pytester" -if six.PY2: - - @pytest.fixture(scope="session", autouse=True) - def _ensure_imports(): - # we import some modules because pytest-2.8's testdir fixture - # will unload all modules after each test and this cause - # (unknown) problems with execnet.Group() - execnet.Group - execnet.makegateway - @pytest.fixture(autouse=True) def _divert_atexit(request, monkeypatch): diff --git a/testing/test_remote.py b/testing/test_remote.py index e223a66..da2f6a8 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -8,7 +8,7 @@ from xdist.workermanage import WorkerController import execnet import marshal -from six.moves.queue import Queue +from queue import Queue WAIT_TIMEOUT = 10.0 From 62bc333e3ecc8711d34cb77bb61f020b63b17e26 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 30 Jul 2020 09:46:55 +0300 Subject: [PATCH 07/10] Fix tests --- testing/acceptance_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index cc9b3ee..77e8ce4 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1,5 +1,6 @@ import os import re +import sys import textwrap import py From bdf66f2c8e00f5e731380f1f9ac3f3441f7b868e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 5 Aug 2020 18:57:25 -0300 Subject: [PATCH 08/10] Add CHANGELOG about requiring pytest 6.0+ --- changelog/541.trivial.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/541.trivial.rst diff --git a/changelog/541.trivial.rst b/changelog/541.trivial.rst new file mode 100644 index 0000000..5911940 --- /dev/null +++ b/changelog/541.trivial.rst @@ -0,0 +1 @@ +``pytest-xdist`` now requires ``pytest>=6.0``. From e5cdb771f7358ee5dbd07447e95e7d2bf66c0840 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 5 Aug 2020 19:01:33 -0300 Subject: [PATCH 09/10] Add CHANGELOG about dropping Python 2.7 Fix #569 --- changelog/569.removal.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/569.removal.rst diff --git a/changelog/569.removal.rst b/changelog/569.removal.rst new file mode 100644 index 0000000..9207182 --- /dev/null +++ b/changelog/569.removal.rst @@ -0,0 +1 @@ +``pytest-xdist`` no longer supports Python 2.7. From 2f014c39df9edea3c4ab952694aa0c8cc1f0303f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 5 Aug 2020 19:02:50 -0300 Subject: [PATCH 10/10] Minor changelog adjustment --- changelog/541.removal.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/541.removal.rst b/changelog/541.removal.rst index 0823a67..f8f3f44 100644 --- a/changelog/541.removal.rst +++ b/changelog/541.removal.rst @@ -1 +1 @@ -drop backward-compatibility "slave" aliases related to worker nodes. We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process. +Drop backward-compatibility "slave" aliases related to worker nodes. We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.