From e3b69d4f29a1dc376a0f331293c68910978ca002 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 3 Apr 2024 00:13:06 +0300 Subject: [PATCH 1/3] Sort imports Using pytest's style. --- src/xdist/__init__.py | 11 +++++------ src/xdist/_path.py | 5 +++-- src/xdist/dsession.py | 23 +++++++++++------------ src/xdist/looponfail.py | 9 +++++---- src/xdist/plugin.py | 2 +- src/xdist/remote.py | 9 +++++---- src/xdist/scheduler/__init__.py | 12 ++++++------ src/xdist/scheduler/each.py | 2 +- src/xdist/scheduler/load.py | 2 +- src/xdist/scheduler/loadfile.py | 3 ++- src/xdist/scheduler/loadgroup.py | 3 ++- src/xdist/scheduler/loadscope.py | 1 + src/xdist/scheduler/worksteal.py | 2 +- src/xdist/workermanage.py | 18 ++++++++++++------ testing/acceptance_test.py | 1 + testing/conftest.py | 6 ++++-- testing/test_dsession.py | 20 +++++++++++--------- testing/test_looponfail.py | 8 ++++---- testing/test_plugin.py | 6 +++--- testing/test_remote.py | 9 +++++---- testing/test_workermanage.py | 13 +++++++++---- 21 files changed, 93 insertions(+), 72 deletions(-) diff --git a/src/xdist/__init__.py b/src/xdist/__init__.py index 031a3d3..ed5a4d9 100644 --- a/src/xdist/__init__.py +++ b/src/xdist/__init__.py @@ -1,10 +1,9 @@ -from xdist.plugin import ( - is_xdist_worker, - is_xdist_master, - get_xdist_worker_id, - is_xdist_controller, -) from xdist._version import version as __version__ +from xdist.plugin import get_xdist_worker_id +from xdist.plugin import is_xdist_controller +from xdist.plugin import is_xdist_master +from xdist.plugin import is_xdist_worker + __all__ = [ "__version__", diff --git a/src/xdist/_path.py b/src/xdist/_path.py index 0af3209..1732300 100644 --- a/src/xdist/_path.py +++ b/src/xdist/_path.py @@ -1,7 +1,8 @@ -import os from itertools import chain +import os from pathlib import Path -from typing import Callable, Iterator +from typing import Callable +from typing import Iterator def visit_path( diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index bae9279..6be93e0 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -1,23 +1,22 @@ from __future__ import annotations + +from enum import auto +from enum import Enum +from queue import Empty +from queue import Queue import sys -from enum import Enum, auto from typing import Sequence import pytest from xdist.remote import Producer +from xdist.scheduler import EachScheduling +from xdist.scheduler import LoadFileScheduling +from xdist.scheduler import LoadGroupScheduling +from xdist.scheduler import LoadScheduling +from xdist.scheduler import LoadScopeScheduling +from xdist.scheduler import WorkStealingScheduling from xdist.workermanage import NodeManager -from xdist.scheduler import ( - EachScheduling, - LoadScheduling, - LoadScopeScheduling, - LoadFileScheduling, - LoadGroupScheduling, - WorkStealingScheduling, -) - - -from queue import Empty, Queue class Interrupted(KeyboardInterrupt): diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index 370cb8b..929f3d4 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -9,13 +9,14 @@ import os from pathlib import Path -from typing import Dict, Sequence - -import pytest import sys import time -import execnet +from typing import Dict +from typing import Sequence + from _pytest._io import TerminalWriter +import execnet +import pytest from xdist._path import visit_path diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index c1350b4..cef61c0 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -1,6 +1,6 @@ import os -import uuid import sys +import uuid import warnings import pytest diff --git a/src/xdist/remote.py b/src/xdist/remote.py index cba91bc..70aa870 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -7,15 +7,16 @@ """ import contextlib -import sys import os +import sys import time from typing import Any -import pytest -from execnet.gateway_base import dumps, DumpError - from _pytest.config import _prepareconfig +from execnet.gateway_base import DumpError +from execnet.gateway_base import dumps +import pytest + try: from setproctitle import setproctitle diff --git a/src/xdist/scheduler/__init__.py b/src/xdist/scheduler/__init__.py index 9201cda..54be9ad 100644 --- a/src/xdist/scheduler/__init__.py +++ b/src/xdist/scheduler/__init__.py @@ -1,6 +1,6 @@ -from xdist.scheduler.each import EachScheduling # noqa -from xdist.scheduler.load import LoadScheduling # noqa -from xdist.scheduler.loadfile import LoadFileScheduling # noqa -from xdist.scheduler.loadscope import LoadScopeScheduling # noqa -from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa -from xdist.scheduler.worksteal import WorkStealingScheduling # noqa +from xdist.scheduler.each import EachScheduling as EachScheduling +from xdist.scheduler.load import LoadScheduling as LoadScheduling +from xdist.scheduler.loadfile import LoadFileScheduling as LoadFileScheduling +from xdist.scheduler.loadgroup import LoadGroupScheduling as LoadGroupScheduling +from xdist.scheduler.loadscope import LoadScopeScheduling as LoadScopeScheduling +from xdist.scheduler.worksteal import WorkStealingScheduling as WorkStealingScheduling diff --git a/src/xdist/scheduler/each.py b/src/xdist/scheduler/each.py index 4579102..329ce68 100644 --- a/src/xdist/scheduler/each.py +++ b/src/xdist/scheduler/each.py @@ -1,6 +1,6 @@ from xdist.remote import Producer -from xdist.workermanage import parse_spec_config from xdist.report import report_collection_diff +from xdist.workermanage import parse_spec_config class EachScheduling: diff --git a/src/xdist/scheduler/load.py b/src/xdist/scheduler/load.py index ccca68b..87d9cb2 100644 --- a/src/xdist/scheduler/load.py +++ b/src/xdist/scheduler/load.py @@ -3,8 +3,8 @@ from itertools import cycle from _pytest.runner import CollectReport from xdist.remote import Producer -from xdist.workermanage import parse_spec_config from xdist.report import report_collection_diff +from xdist.workermanage import parse_spec_config class LoadScheduling: diff --git a/src/xdist/scheduler/loadfile.py b/src/xdist/scheduler/loadfile.py index 91b5938..9ddd535 100644 --- a/src/xdist/scheduler/loadfile.py +++ b/src/xdist/scheduler/loadfile.py @@ -1,6 +1,7 @@ -from .loadscope import LoadScopeScheduling from xdist.remote import Producer +from .loadscope import LoadScopeScheduling + class LoadFileScheduling(LoadScopeScheduling): """Implement load scheduling across nodes, but grouping test test file. diff --git a/src/xdist/scheduler/loadgroup.py b/src/xdist/scheduler/loadgroup.py index ecefa49..1dee40e 100644 --- a/src/xdist/scheduler/loadgroup.py +++ b/src/xdist/scheduler/loadgroup.py @@ -1,6 +1,7 @@ -from .loadscope import LoadScopeScheduling from xdist.remote import Producer +from .loadscope import LoadScopeScheduling + class LoadGroupScheduling(LoadScopeScheduling): """Implement load scheduling across nodes, but grouping test by xdist_group mark. diff --git a/src/xdist/scheduler/loadscope.py b/src/xdist/scheduler/loadscope.py index bcfe11f..af92935 100644 --- a/src/xdist/scheduler/loadscope.py +++ b/src/xdist/scheduler/loadscope.py @@ -1,6 +1,7 @@ from collections import OrderedDict from _pytest.runner import CollectReport + from xdist.remote import Producer from xdist.report import report_collection_diff from xdist.workermanage import parse_spec_config diff --git a/src/xdist/scheduler/worksteal.py b/src/xdist/scheduler/worksteal.py index 01619ea..055efa4 100644 --- a/src/xdist/scheduler/worksteal.py +++ b/src/xdist/scheduler/worksteal.py @@ -3,8 +3,8 @@ from collections import namedtuple from _pytest.runner import CollectReport from xdist.remote import Producer -from xdist.workermanage import parse_spec_config from xdist.report import report_collection_diff +from xdist.workermanage import parse_spec_config NodePending = namedtuple("NodePending", ["node", "pending"]) diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index 9c72431..1de00ad 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -1,17 +1,23 @@ import fnmatch import os +from pathlib import Path import re import sys +from typing import Any +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Union import uuid -from pathlib import Path -from typing import List, Union, Sequence, Optional, Any, Tuple, Set -import pytest import execnet +import pytest +from xdist.plugin import _sys_path import xdist.remote from xdist.remote import Producer -from xdist.plugin import _sys_path def parse_spec_config(config): @@ -89,8 +95,8 @@ class NodeManager: break else: return [] - import pytest import _pytest + import pytest def get_dir(p): """Return the directory path if p is a package or the path to the .py file otherwise.""" @@ -405,8 +411,8 @@ class WorkerController: def unserialize_warning_message(data): - import warnings import importlib + import warnings if data["message_module"]: mod = importlib.import_module(data["message_module"]) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index acfa8d4..40745f6 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -6,6 +6,7 @@ from typing import List from typing import Tuple import pytest + import xdist diff --git a/testing/conftest.py b/testing/conftest.py index dd7293d..195fb87 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,8 +1,10 @@ -import execnet -import pytest import shutil from typing import List +import execnet +import pytest + + pytest_plugins = "pytester" diff --git a/testing/test_dsession.py b/testing/test_dsession.py index f809dc4..432f167 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -1,16 +1,18 @@ from __future__ import annotations -from xdist.dsession import ( - DSession, - get_default_max_worker_restart, - get_workers_status_line, - WorkerStatus, -) -from xdist.report import report_collection_diff -from xdist.scheduler import EachScheduling, LoadScheduling, WorkStealingScheduling + from typing import Sequence -import pytest import execnet +import pytest + +from xdist.dsession import DSession +from xdist.dsession import get_default_max_worker_restart +from xdist.dsession import get_workers_status_line +from xdist.dsession import WorkerStatus +from xdist.report import report_collection_diff +from xdist.scheduler import EachScheduling +from xdist.scheduler import LoadScheduling +from xdist.scheduler import WorkStealingScheduling class MockGateway: diff --git a/testing/test_looponfail.py b/testing/test_looponfail.py index 65a89fb..2879e4d 100644 --- a/testing/test_looponfail.py +++ b/testing/test_looponfail.py @@ -1,12 +1,12 @@ import pathlib +from pathlib import Path +import shutil import tempfile -import unittest.mock +import textwrap from typing import List +import unittest.mock import pytest -import shutil -import textwrap -from pathlib import Path from xdist.looponfail import RemoteControl from xdist.looponfail import StatRecorder diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 4bf514b..951b882 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -1,13 +1,13 @@ from contextlib import suppress +import os from pathlib import Path import sys -import os import execnet -from xdist.workermanage import NodeManager - import pytest +from xdist.workermanage import NodeManager + @pytest.fixture def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch): diff --git a/testing/test_remote.py b/testing/test_remote.py index 4e37262..26ded17 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -1,13 +1,14 @@ +import marshal import pprint -import pytest +from queue import Queue import sys import uuid -from xdist.workermanage import WorkerController import execnet -import marshal +import pytest + +from xdist.workermanage import WorkerController -from queue import Queue WAIT_TIMEOUT = 10.0 diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index 6f5a3a4..367ce8a 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -1,14 +1,19 @@ -import execnet -import pytest +from pathlib import Path import shutil import textwrap import warnings -from pathlib import Path + +import execnet +import pytest from util import generate_warning + from xdist import workermanage from xdist._path import visit_path from xdist.remote import serialize_warning_message -from xdist.workermanage import HostRSync, NodeManager, unserialize_warning_message +from xdist.workermanage import HostRSync +from xdist.workermanage import NodeManager +from xdist.workermanage import unserialize_warning_message + pytest_plugins = "pytester" From c01de1c73ebe002318118c0ab924434946313489 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 3 Apr 2024 00:15:21 +0300 Subject: [PATCH 2/3] Make docstring style more consistent --- src/xdist/dsession.py | 8 ++++---- src/xdist/looponfail.py | 10 +++++----- src/xdist/newhooks.py | 16 ++++++++-------- src/xdist/plugin.py | 4 ++-- src/xdist/remote.py | 10 +++++----- src/xdist/scheduler/each.py | 8 ++++---- src/xdist/scheduler/load.py | 14 +++++++------- src/xdist/scheduler/loadfile.py | 5 ++++- src/xdist/scheduler/loadscope.py | 2 +- src/xdist/scheduler/worksteal.py | 12 ++++++------ src/xdist/workermanage.py | 4 ++-- testing/acceptance_test.py | 28 ++++++++++------------------ testing/test_newhooks.py | 6 +++--- testing/test_plugin.py | 2 +- testing/test_workermanage.py | 6 ++---- 15 files changed, 64 insertions(+), 71 deletions(-) diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 6be93e0..1f55239 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -24,7 +24,7 @@ class Interrupted(KeyboardInterrupt): class DSession: - """A pytest plugin which runs a distributed test session + """A pytest plugin which runs a distributed test session. At the beginning of the test session this creates a NodeManager instance which creates and starts all nodes. Nodes then emit @@ -60,7 +60,7 @@ class DSession: @property def session_finished(self): - """Return True if the distributed session has finished + """Return True if the distributed session has finished. This means all nodes have executed all test items. This is used by pytest_runtestloop to break out of its loop. @@ -250,7 +250,7 @@ class DSession: terminalreporter.write_sep("=", f"xdist: {self._summary_report}") def worker_collectionfinish(self, node, ids): - """worker has finished test collection. + """Worker has finished test collection. This adds the collection for this node to the scheduler. If the scheduler indicates collection is finished (i.e. all @@ -490,7 +490,7 @@ class TerminalDistReporter: def get_default_max_worker_restart(config): - """gets the default value of --max-worker-restart option if it is not provided. + """Gets the default value of --max-worker-restart option if it is not provided. Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226). """ diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index 929f3d4..e88d2ac 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -1,10 +1,10 @@ """ - Implement -f aka looponfailing for pytest. +Implement -f aka looponfailing for pytest. - NOTE that we try to avoid loading and depending on application modules - within the controlling process (the one that starts repeatedly test - processes) otherwise changes to source code can crash - the controlling process which should best never happen. +NOTE that we try to avoid loading and depending on application modules +within the controlling process (the one that starts repeatedly test +processes) otherwise changes to source code can crash +the controlling process which should best never happen. """ import os diff --git a/src/xdist/newhooks.py b/src/xdist/newhooks.py index 1603951..ceac11e 100644 --- a/src/xdist/newhooks.py +++ b/src/xdist/newhooks.py @@ -17,12 +17,12 @@ import pytest @pytest.hookspec() def pytest_xdist_setupnodes(config, specs): - """called before any remote node is set up.""" + """Called before any remote node is set up.""" @pytest.hookspec() def pytest_xdist_newgateway(gateway): - """called on new raw gateway creation.""" + """Called on new raw gateway creation.""" @pytest.hookspec( @@ -31,7 +31,7 @@ def pytest_xdist_newgateway(gateway): ) ) def pytest_xdist_rsyncstart(source, gateways): - """called before rsyncing a directory to remote gateways takes place.""" + """Called before rsyncing a directory to remote gateways takes place.""" @pytest.hookspec( @@ -40,17 +40,17 @@ def pytest_xdist_rsyncstart(source, gateways): ) ) def pytest_xdist_rsyncfinish(source, gateways): - """called after rsyncing a directory to remote gateways takes place.""" + """Called after rsyncing a directory to remote gateways takes place.""" @pytest.hookspec(firstresult=True) def pytest_xdist_getremotemodule(): - """called when creating remote node""" + """Called when creating remote node.""" @pytest.hookspec() def pytest_configure_node(node): - """configure node information before it gets instantiated.""" + """Configure node information before it gets instantiated.""" @pytest.hookspec() @@ -65,12 +65,12 @@ def pytest_testnodedown(node, error): @pytest.hookspec() def pytest_xdist_node_collection_finished(node, ids): - """called by the controller node when a worker node finishes collecting.""" + """Called by the controller node when a worker node finishes collecting.""" @pytest.hookspec(firstresult=True) def pytest_xdist_make_scheduler(config, log): - """return a node scheduler implementation""" + """Return a node scheduler implementation.""" @pytest.hookspec(firstresult=True) diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index cef61c0..116a0f0 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -305,7 +305,7 @@ def pytest_cmdline_main(config): def is_xdist_worker(request_or_session) -> bool: - """Return `True` if this is an xdist worker, `False` otherwise + """Return `True` if this is an xdist worker, `False` otherwise. :param request_or_session: the `pytest` `request` or `session` object """ @@ -313,7 +313,7 @@ def is_xdist_worker(request_or_session) -> bool: def is_xdist_controller(request_or_session) -> bool: - """Return `True` if this is the xdist controller, `False` otherwise + """Return `True` if this is the xdist controller, `False` otherwise. Note: this method also returns `False` when distribution has not been activated at all. diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 70aa870..93f722e 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -1,9 +1,9 @@ """ - This module is executed in remote subprocesses and helps to - control a remote testing session and relay back information. - It assumes that 'py' is importable and does not have dependencies - on the rest of the xdist code. This means that the xdist-plugin - needs not to be installed in remote environments. +This module is executed in remote subprocesses and helps to +control a remote testing session and relay back information. +It assumes that 'py' is importable and does not have dependencies +on the rest of the xdist code. This means that the xdist-plugin +needs not to be installed in remote environments. """ import contextlib diff --git a/src/xdist/scheduler/each.py b/src/xdist/scheduler/each.py index 329ce68..91084bf 100644 --- a/src/xdist/scheduler/each.py +++ b/src/xdist/scheduler/each.py @@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config class EachScheduling: - """Implement scheduling of test items on all nodes + """Implement scheduling of test items on all nodes. If a node gets added after the test run is started then it is assumed to replace a node which got removed before it finished @@ -48,7 +48,7 @@ class EachScheduling: @property def has_pending(self): - """Return True if there are pending test items + """Return True if there are pending test items. This indicates that collection has finished and nodes are still processing test items, so this can be thought of as @@ -64,7 +64,7 @@ class EachScheduling: self.node2pending[node] = [] def add_node_collection(self, node, collection): - """Add the collected test items from a node + """Add the collected test items from a node. Collection is complete once all nodes have submitted their collection. In this case its pending list is set to an empty @@ -119,7 +119,7 @@ class EachScheduling: return crashitem def schedule(self): - """Schedule the test items on the nodes + """Schedule the test items on the nodes. If the node's pending list is empty it is a new node which needs to run all the tests. If the pending list is already diff --git a/src/xdist/scheduler/load.py b/src/xdist/scheduler/load.py index 87d9cb2..6d7b231 100644 --- a/src/xdist/scheduler/load.py +++ b/src/xdist/scheduler/load.py @@ -23,7 +23,7 @@ class LoadScheduling: submit a collection. This is used to know when all nodes have finished collection or how large the chunks need to be created. - Attributes: + Attributes:: :numnodes: The expected number of nodes taking part. The actual number of nodes will vary during the scheduler's lifetime as @@ -95,7 +95,7 @@ class LoadScheduling: @property def has_pending(self): - """Return True if there are pending test items + """Return True if there are pending test items. This indicates that collection has finished and nodes are still processing test items, so this can be thought of as @@ -121,7 +121,7 @@ class LoadScheduling: self.node2pending[node] = [] def add_node_collection(self, node, collection): - """Add the collected test items from a node + """Add the collected test items from a node. The collection is stored in the ``.node2collection`` map. Called by the ``DSession.worker_collectionfinish`` hook. @@ -142,7 +142,7 @@ class LoadScheduling: self.node2collection[node] = list(collection) def mark_test_complete(self, node, item_index, duration=0): - """Mark test item as completed by node + """Mark test item as completed by node. The duration it took to execute the item is used as a hint to the scheduler. @@ -161,7 +161,7 @@ class LoadScheduling: self.check_schedule(node) def check_schedule(self, node, duration=0): - """Maybe schedule new items on the node + """Maybe schedule new items on the node. If there are any globally pending nodes left then this will check if the given node should be given any more tests. The @@ -195,7 +195,7 @@ class LoadScheduling: self.log("num items waiting for node:", len(self.pending)) def remove_node(self, node): - """Remove a node from the scheduler + """Remove a node from the scheduler. This should be called either when the node crashed or at shutdown time. In the former case any pending items assigned @@ -219,7 +219,7 @@ class LoadScheduling: return crashitem def schedule(self): - """Initiate distribution of the test collection + """Initiate distribution of the test collection. Initiate scheduling of the items across the nodes. If this gets called again later it behaves the same as calling diff --git a/src/xdist/scheduler/loadfile.py b/src/xdist/scheduler/loadfile.py index 9ddd535..25b72da 100644 --- a/src/xdist/scheduler/loadfile.py +++ b/src/xdist/scheduler/loadfile.py @@ -44,7 +44,10 @@ class LoadFileScheduling(LoadScopeScheduling): This function will group tests with the scope determined by splitting the first ``::`` from the left. That is, test will be grouped in a single work unit when they reside in the same file. - In the above example, scopes will be:: + + In the above example, scopes will be:: + + .. code-block:: text example/loadsuite/test/test_beta.py example/loadsuite/test/test_delta.py diff --git a/src/xdist/scheduler/loadscope.py b/src/xdist/scheduler/loadscope.py index af92935..ad94714 100644 --- a/src/xdist/scheduler/loadscope.py +++ b/src/xdist/scheduler/loadscope.py @@ -22,7 +22,7 @@ class LoadScopeScheduling: When created, ``numnodes`` defines how many nodes are expected to submit a collection. This is used to know when all nodes have finished collection. - Attributes: + Attributes:: :numnodes: The expected number of nodes taking part. The actual number of nodes will vary during the scheduler's lifetime as nodes are added by diff --git a/src/xdist/scheduler/worksteal.py b/src/xdist/scheduler/worksteal.py index 055efa4..fc102bd 100644 --- a/src/xdist/scheduler/worksteal.py +++ b/src/xdist/scheduler/worksteal.py @@ -22,7 +22,7 @@ class WorkStealingScheduling: test remains), an attempt is made to reassign ("steal") some tests from other nodes to this node. - Attributes: + Attributes:: :numnodes: The expected number of nodes taking part. The actual number of nodes will vary during the scheduler's lifetime as @@ -101,7 +101,7 @@ class WorkStealingScheduling: @property def has_pending(self): - """Return True if there are pending test items + """Return True if there are pending test items. This indicates that collection has finished and nodes are still processing test items, so this can be thought of as @@ -127,7 +127,7 @@ class WorkStealingScheduling: self.node2pending[node] = [] def add_node_collection(self, node, collection): - """Add the collected test items from a node + """Add the collected test items from a node. The collection is stored in the ``.node2collection`` map. Called by the ``DSession.worker_collectionfinish`` hook. @@ -148,7 +148,7 @@ class WorkStealingScheduling: self.node2collection[node] = list(collection) def mark_test_complete(self, node, item_index, duration=None): - """Mark test item as completed by node + """Mark test item as completed by node. This is called by the ``DSession.worker_testreport`` hook. """ @@ -233,7 +233,7 @@ class WorkStealingScheduling: self.steal_requested_from_node = steal_from.node def remove_node(self, node): - """Remove a node from the scheduler + """Remove a node from the scheduler. This should be called either when the node crashed or at shutdown time. In the former case any pending items assigned @@ -263,7 +263,7 @@ class WorkStealingScheduling: return crashitem def schedule(self): - """Initiate distribution of the test collection + """Initiate distribution of the test collection. Initiate scheduling of the items across the nodes. If this gets called again later it behaves the same as calling diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index 1de00ad..fcb70e8 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -166,7 +166,7 @@ class NodeManager: class HostRSync(execnet.RSync): - """RSyncer that filters out common files""" + """RSyncer that filters out common files.""" PathLike = Union[str, "os.PathLike[str]"] @@ -314,7 +314,7 @@ class WorkerController: self._shutdown_sent = True def sendcommand(self, name, **kwargs): - """send a named parametrized command to the other side.""" + """Send a named parametrized command to the other side.""" self.log(f"sending command {name}(**{kwargs})") self.channel.send((name, kwargs)) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 40745f6..d82562e 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -482,7 +482,7 @@ class TestTerminalReporting: ) def test_logfinish_hook(self, pytester: pytest.Pytester) -> None: - """Ensure the pytest_runtest_logfinish hook is being properly handled""" + """Ensure the pytest_runtest_logfinish hook is being properly handled.""" pytester.makeconftest( """ def pytest_runtest_logfinish(): @@ -613,7 +613,7 @@ def test_fixture_teardown_failure(pytester: pytest.Pytester) -> None: def test_config_initialization( pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, pytestconfig ) -> None: - """Ensure workers and controller are initialized consistently. Integration test for #445""" + """Ensure workers and controller are initialized consistently. Integration test for #445.""" pytester.makepyfile( **{ "dir_a/test_foo.py": """ @@ -637,7 +637,7 @@ def test_config_initialization( @pytest.mark.parametrize("when", ["setup", "call", "teardown"]) def test_crashing_item(pytester, when) -> None: - """Ensure crashing item is correctly reported during all testing stages""" + """Ensure crashing item is correctly reported during all testing stages.""" code = dict(setup="", call="", teardown="") code[when] = "os._exit(1)" p = pytester.makepyfile( @@ -770,7 +770,7 @@ def test_tmpdir_disabled(pytester: pytest.Pytester) -> None: @pytest.mark.parametrize("plugin", ["xdist.looponfail"]) def test_sub_plugins_disabled(pytester, plugin) -> None: - """Test that xdist doesn't break if we disable any of its sub-plugins. (#32)""" + """Test that xdist doesn't break if we disable any of its sub-plugins (#32).""" p1 = pytester.makepyfile( """ def test_ok(): @@ -800,9 +800,7 @@ class TestWarnings: def test_warning_captured_deprecated_in_pytest_6( self, pytester: pytest.Pytester ) -> None: - """ - Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562) - """ + """Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562).""" from _pytest import hookspec if not hasattr(hookspec, "pytest_warning_captured"): @@ -834,7 +832,7 @@ class TestWarnings: @pytest.mark.parametrize("n", ["-n0", "-n1"]) def test_custom_subclass(self, pytester, n) -> None: """Check that warning subclasses that don't honor the args attribute don't break - pytest-xdist (#344) + pytest-xdist (#344). """ pytester.makepyfile( """ @@ -1117,7 +1115,7 @@ def test_error_report_styles(pytester, tb) -> None: def test_color_yes_collection_on_non_atty(pytester, request) -> None: - """skip collect progress report when working on non-terminals. + """Skip collect progress report when working on non-terminals. Similar to pytest-dev/pytest#1397 """ @@ -1142,9 +1140,7 @@ def test_color_yes_collection_on_non_atty(pytester, request) -> None: def test_without_terminal_plugin(pytester, request) -> None: - """ - No output when terminal plugin is disabled - """ + """No output when terminal plugin is disabled.""" pytester.makepyfile( """ def test_1(): @@ -1158,9 +1154,7 @@ def test_without_terminal_plugin(pytester, request) -> None: def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None: - """ - Internal error when using --maxfail option (#62, #65). - """ + """Internal error when using --maxfail option (#62, #65).""" pytester.makepyfile( """ import pytest @@ -1181,9 +1175,7 @@ def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None: def test_maxfail_causes_early_termination(pytester: pytest.Pytester) -> None: - """ - Ensure subsequent tests on a worker aren't run when using --maxfail (#1024). - """ + """Ensure subsequent tests on a worker aren't run when using --maxfail (#1024).""" pytester.makepyfile( """ def test1(): diff --git a/testing/test_newhooks.py b/testing/test_newhooks.py index dcd2bc0..e3a8ac6 100644 --- a/testing/test_newhooks.py +++ b/testing/test_newhooks.py @@ -14,9 +14,9 @@ class TestHooks: ) def test_runtest_logreport(self, pytester: pytest.Pytester) -> None: - """Test that log reports from pytest_runtest_logreport when running - with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8) - """ + """Test that log reports from pytest_runtest_logreport when running with + xdist contain "node", "nodeid", "worker_id", and "testrun_uid" + attributes (#8).""" pytester.makeconftest( """ def pytest_runtest_logreport(report): diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 951b882..74de0d4 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -11,7 +11,7 @@ from xdist.workermanage import NodeManager @pytest.fixture def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch): - """Make pytest-xdist believe the system has 3 CPUs""" + """Make pytest-xdist believe the system has 3 CPUs.""" # block import monkeypatch.setitem(sys.modules, "psutil", None) # type: ignore monkeypatch.delattr(os, "sched_getaffinity", raising=False) diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index 367ce8a..e7fa3f5 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -350,8 +350,7 @@ class MyWarning(UserWarning): ], ) def test_unserialize_warning_msg(w_cls): - """Test that warning serialization process works well""" - + """Test that warning serialization process works well.""" # Create a test warning message with pytest.warns(UserWarning) as w: if not isinstance(w_cls, str): @@ -392,8 +391,7 @@ class MyWarningUnknown(UserWarning): def test_warning_serialization_tweaked_module(): - """Test for GH#404""" - + """Test for GH#404.""" # Create a test warning message with pytest.warns(UserWarning) as w: warnings.warn("hello", MyWarningUnknown) From 816c9dcda1b5b70ef7d4dcfd444e066a932a88ba Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Tue, 2 Apr 2024 23:23:31 +0300 Subject: [PATCH 3/3] Use ruff instead of black, flake8, autoflake, pyupgrade Config adapted from pytest. --- .pre-commit-config.yaml | 28 +++---------- pyproject.toml | 70 ++++++++++++++++++++++++++++++-- src/xdist/dsession.py | 8 ++-- src/xdist/looponfail.py | 2 +- src/xdist/plugin.py | 2 +- src/xdist/remote.py | 2 +- src/xdist/scheduler/load.py | 4 +- src/xdist/scheduler/loadscope.py | 2 - src/xdist/scheduler/worksteal.py | 12 ++++-- src/xdist/workermanage.py | 8 ++-- testing/acceptance_test.py | 12 ++---- testing/test_dsession.py | 12 +++--- testing/test_plugin.py | 2 +- testing/test_remote.py | 14 +++---- tox.ini | 4 -- 15 files changed, 109 insertions(+), 73 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8210d6..70d69c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,10 @@ repos: -- repo: https://github.com/PyCQA/autoflake - rev: v2.3.1 - hooks: - - id: autoflake - args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"] -- repo: https://github.com/psf/black - rev: 24.3.0 - hooks: - - id: black - args: [--safe, --quiet, --target-version, py35] +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.3.5" + hooks: + - id: ruff + args: ["--fix"] + - id: ruff-format - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 hooks: @@ -17,19 +13,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - id: check-yaml - - id: debug-statements -- repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 - hooks: - - id: flake8 -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 - hooks: - - id: pyupgrade - args: [--py3-plus] - repo: local hooks: - id: rst diff --git a/pyproject.toml b/pyproject.toml index bb6cc5e..ef5a35a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,10 +67,72 @@ include-package-data = false [tool.setuptools_scm] write_to = "src/xdist/_version.py" -[tool.flake8] -# Ignore any errors related to formatting, let black worry/fix them. -ignore = ["E501", "W503", "E203"] -max-line-length = 100 +[tool.ruff] +src = ["src"] + +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +select = [ + "B", # bugbear + "D", # pydocstyle + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "PYI", # flake8-pyi + "UP", # pyupgrade + "RUF", # ruff + "W", # pycodestyle + "T10", # flake8-debugger + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PLE", # pylint error + "PLW", # pylint warning + "PLR1714", # Consider merging multiple comparisons +] +ignore = [ + # bugbear ignore + "B011", # Do not `assert False` (`python -O` removes these calls) + "B028", # No explicit `stacklevel` keyword argument found + # pydocstyle ignore + "D100", # Missing docstring in public module + "D101", # Missing docstring in public class + "D102", # Missing docstring in public method + "D103", # Missing docstring in public function + "D104", # Missing docstring in public package + "D105", # Missing docstring in magic method + "D106", # Missing docstring in public nested class + "D107", # Missing docstring in `__init__` + "D209", # Multi-line docstring closing quotes should be on a separate line + "D205", # 1 blank line required between summary line and description + "D400", # First line should end with a period + "D401", # First line of docstring should be in imperative mood + # ruff ignore + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` + # pylint ignore + "PLW0603", # Using the global statement + "PLW0120", # remove the else and dedent its contents + "PLW2901", # for loop variable overwritten by assignment target + "PLR5501", # Use `elif` instead of `else` then `if` +] + +[tool.ruff.lint.pycodestyle] +# In order to be able to format for 88 char in ruff format +max-line-length = 120 + +[tool.ruff.lint.pydocstyle] +convention = "pep257" + +[tool.ruff.lint.isort] +force-single-line = true +combine-as-imports = true +force-sort-within-sections = true +order-by-type = false +lines-after-imports = 2 + +[tool.ruff.lint.per-file-ignores] +"src/xdist/_version.py" = ["I001"] [tool.mypy] mypy_path = ["src"] diff --git a/src/xdist/dsession.py b/src/xdist/dsession.py index 1f55239..ebba5e2 100644 --- a/src/xdist/dsession.py +++ b/src/xdist/dsession.py @@ -230,9 +230,7 @@ class DSession: ) if maximum_reached: if self._max_worker_restart == 0: - msg = "worker {} crashed and worker restarting disabled".format( - node.gateway.id - ) + msg = f"worker {node.gateway.id} crashed and worker restarting disabled" else: msg = "maximum crashed workers reached: %d" % self._max_worker_restart self._summary_report = msg @@ -463,7 +461,7 @@ class TerminalDistReporter: rinfo = gateway._rinfo() different_interpreter = rinfo.executable != sys.executable if different_interpreter: - version = "%s.%s.%s" % rinfo.version_info[:3] + version = "{}.{}.{}".format(*rinfo.version_info[:3]) self.rewrite( f"[{gateway.id}] {rinfo.platform} Python {version} cwd: {rinfo.cwd}", newline=True, @@ -504,7 +502,7 @@ def get_default_max_worker_restart(config): def get_workers_status_line( - status_and_items: Sequence[tuple[WorkerStatus, int]] + status_and_items: Sequence[tuple[WorkerStatus, int]], ) -> str: """ Return the line to display during worker setup/collection based on the diff --git a/src/xdist/looponfail.py b/src/xdist/looponfail.py index e88d2ac..b67bbe6 100644 --- a/src/xdist/looponfail.py +++ b/src/xdist/looponfail.py @@ -254,7 +254,7 @@ class StatRecorder: return time.sleep(checkinterval) - def check(self, removepycfiles: bool = True) -> bool: # noqa, too complex + def check(self, removepycfiles: bool = True) -> bool: changed = False newstat: Dict[Path, os.stat_result] = {} for rootdir in self.rootdirlist: diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index 116a0f0..c9d4c49 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -296,7 +296,7 @@ def pytest_cmdline_main(config): if not val("collectonly") and _is_distribution_mode(config) and usepdb: raise pytest.UsageError( "--pdb is incompatible with distributing tests; try using -n0 or -nauto." - ) # noqa: E501 + ) # ------------------------------------------------------------------------- diff --git a/src/xdist/remote.py b/src/xdist/remote.py index 93f722e..ba18da8 100644 --- a/src/xdist/remote.py +++ b/src/xdist/remote.py @@ -325,7 +325,7 @@ def setup_config(config, basetemp): if __name__ == "__channelexec__": - channel = channel # type: ignore[name-defined] # noqa: F821 + channel = channel # type: ignore[name-defined] # noqa: F821, PLW0127 workerinput, args, option_dict, change_sys_path = channel.receive() # type: ignore[name-defined] if change_sys_path is None: diff --git a/src/xdist/scheduler/load.py b/src/xdist/scheduler/load.py index 6d7b231..422528c 100644 --- a/src/xdist/scheduler/load.py +++ b/src/xdist/scheduler/load.py @@ -243,7 +243,7 @@ class LoadScheduling: return # Collections are identical, create the index of pending items. - self.collection = list(self.node2collection.values())[0] + self.collection = next(iter(self.node2collection.values())) self.pending[:] = range(len(self.collection)) if not self.collection: return @@ -260,7 +260,7 @@ class LoadScheduling: # to each node - which is suboptimal when you have less than # 2 * len(nodes) tests. nodes = cycle(self.nodes) - for i in range(len(self.pending)): + for _ in range(len(self.pending)): self._send_tests(next(nodes), 1) else: # Send batches of consecutive tests. By default, pytest sorts tests diff --git a/src/xdist/scheduler/loadscope.py b/src/xdist/scheduler/loadscope.py index ad94714..fb4bc63 100644 --- a/src/xdist/scheduler/loadscope.py +++ b/src/xdist/scheduler/loadscope.py @@ -208,7 +208,6 @@ class LoadScopeScheduling: - ``DSession.worker_collectionfinish``. """ - # Check that add_node() was called on the node before assert node in self.assigned_work @@ -301,7 +300,6 @@ class LoadScopeScheduling: If there are any globally pending work units left then this will check if the given node should be given any more tests. """ - # Do not add more work to a node shutting down if node.shutting_down: return diff --git a/src/xdist/scheduler/worksteal.py b/src/xdist/scheduler/worksteal.py index fc102bd..4a2c2fe 100644 --- a/src/xdist/scheduler/worksteal.py +++ b/src/xdist/scheduler/worksteal.py @@ -1,4 +1,7 @@ -from collections import namedtuple +from __future__ import annotations + +from typing import Any +from typing import NamedTuple from _pytest.runner import CollectReport @@ -7,7 +10,10 @@ from xdist.report import report_collection_diff from xdist.workermanage import parse_spec_config -NodePending = namedtuple("NodePending", ["node", "pending"]) +class NodePending(NamedTuple): + node: Any + pending: list[int] + # Every worker needs at least 2 tests in queue - the current and the next one. MIN_PENDING = 2 @@ -285,7 +291,7 @@ class WorkStealingScheduling: return # Collections are identical, create the index of pending items. - self.collection = list(self.node2collection.values())[0] + self.collection = next(iter(self.node2collection.values())) self.pending[:] = range(len(self.collection)) if not self.collection: return diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index fcb70e8..d1cef01 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -175,7 +175,7 @@ class HostRSync(execnet.RSync): sourcedir: PathLike, *, ignores: Optional[Sequence[PathLike]] = None, - **kwargs: object + **kwargs: object, ) -> None: if ignores is None: ignores = [] @@ -322,8 +322,8 @@ class WorkerController: self.log(f"queuing {eventname}(**{kwargs})") self.putevent((eventname, kwargs)) - def process_from_remote(self, eventcall): # noqa too complex - """this gets called for each object we receive from + def process_from_remote(self, eventcall): + """This gets called for each object we receive from the other side and if the channel closes. Note that channel callbacks run in the receiver @@ -400,7 +400,7 @@ class WorkerController: except KeyboardInterrupt: # should not land in receiver-thread raise - except: # noqa + except BaseException: from _pytest._code import ExceptionInfo excinfo = ExceptionInfo.from_current() diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index d82562e..6cb85c0 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -266,8 +266,8 @@ class TestDistribution: "-pfoobarplugin", "--foobar=123", "--dist=load", - "--rsyncdir=%(subdir)s" % locals(), - "--tx=popen//chdir=%(dest)s" % locals(), + f"--rsyncdir={subdir}", + f"--tx=popen//chdir={dest}", p, ) assert result.ret == 0 @@ -657,9 +657,7 @@ def test_crashing_item(pytester, when) -> None: def test_ok(): pass - """.format( - **code - ) + """.format(**code) ) passes = 2 if when == "teardown" else 1 result = pytester.runpytest("-n2", p) @@ -1513,9 +1511,7 @@ class TestLocking: FILE_LOCK = filelock.FileLock("test.lock") - """ + ( - (_test_content * 4) % ("A", "B", "C", "D") - ) + """ + ((_test_content * 4) % ("A", "B", "C", "D")) @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"]) def test_single_file(self, pytester, scope) -> None: diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 432f167..3ce205c 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -165,7 +165,7 @@ class TestLoadScheduling: for i in range(7, 16): sched.mark_test_complete(node1, i - 3) - assert node1.sent == [0, 1] + list(range(4, i)) + assert node1.sent == [0, 1, *range(4, i)] assert node2.sent == [2, 3] assert sched.pending == list(range(i, 16)) @@ -187,7 +187,7 @@ class TestLoadScheduling: for complete_index, first_pending in enumerate(range(5, 16)): sched.mark_test_complete(node1, node1.sent[complete_index]) - assert node1.sent == [0, 1] + list(range(4, first_pending)) + assert node1.sent == [0, 1, *range(4, first_pending)] assert node2.sent == [2, 3] assert sched.pending == list(range(first_pending, 16)) @@ -251,9 +251,7 @@ class TestLoadScheduling: """ class CollectHook: - """ - Dummy hook that stores collection reports. - """ + """Dummy hook that stores collection reports.""" def __init__(self): self.reports = [] @@ -295,7 +293,7 @@ class TestWorkStealingScheduling: sched.schedule() assert not sched.pending assert not sched.tests_finished - assert node1.sent == list(range(0, 8)) + assert node1.sent == list(range(8)) assert node2.sent == list(range(8, 16)) for i in range(8): sched.mark_test_complete(node1, node1.sent[i]) @@ -315,7 +313,7 @@ class TestWorkStealingScheduling: sched.add_node_collection(node2, collection) assert sched.collection_is_completed sched.schedule() - assert node1.sent == list(range(0, 8)) + assert node1.sent == list(range(8)) assert node2.sent == list(range(8, 16)) for i in range(8): sched.mark_test_complete(node1, node1.sent[i]) diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 74de0d4..687a3d7 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -13,7 +13,7 @@ from xdist.workermanage import NodeManager def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch): """Make pytest-xdist believe the system has 3 CPUs.""" # block import - monkeypatch.setitem(sys.modules, "psutil", None) # type: ignore + monkeypatch.setitem(sys.modules, "psutil", None) monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.setattr(os, "cpu_count", lambda: 3) diff --git a/testing/test_remote.py b/testing/test_remote.py index 26ded17..8645029 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -16,9 +16,9 @@ WAIT_TIMEOUT = 10.0 def check_marshallable(d): try: marshal.dumps(d) - except ValueError: + except ValueError as e: pprint.pprint(d) - raise ValueError("not marshallable") + raise ValueError("not marshallable") from e class EventCall: @@ -164,7 +164,7 @@ class TestWorkerInteractor: worker.sendcommand("runtests_all") worker.sendcommand("shutdown") for func in "::test_func", "::test_func2": - for i in range(3): # setup/call/teardown + for _ in range(3): # setup/call/teardown ev = worker.popevent("testreport") assert ev.name == "testreport" rep = unserialize_report(ev.kwargs["data"]) @@ -329,12 +329,10 @@ def test_remote_mainargv(pytester: pytest.Pytester) -> None: outer_argv = sys.argv pytester.makepyfile( - """ + f""" def test_mainargv(request): - assert request.config.workerinput["mainargv"] == {!r} - """.format( - outer_argv - ) + assert request.config.workerinput["mainargv"] == {outer_argv!r} + """ ) result = pytester.runpytest("-n1") assert result.ret == 0 diff --git a/tox.ini b/tox.ini index ca1310a..5f46dca 100644 --- a/tox.ini +++ b/tox.ini @@ -63,7 +63,3 @@ commands = # it so they don't conflict with each other (#611). addopts = -ra -p no:pytest-services testpaths = testing - -[flake8] -max-line-length = 120 -ignore = E203,W503