Merge pull request #1051 from bluetech/ruff

Use ruff instead of black, flake8, autoflake, pyupgrade
This commit is contained in:
Ran Benita
2024-04-03 09:42:26 +03:00
committed by GitHub
26 changed files with 266 additions and 216 deletions

View File

@@ -1,14 +1,10 @@
repos: repos:
- repo: https://github.com/PyCQA/autoflake - repo: https://github.com/astral-sh/ruff-pre-commit
rev: v2.3.1 rev: "v0.3.5"
hooks: hooks:
- id: autoflake - id: ruff
args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"] args: ["--fix"]
- repo: https://github.com/psf/black - id: ruff-format
rev: 24.3.0
hooks:
- id: black
args: [--safe, --quiet, --target-version, py35]
- repo: https://github.com/asottile/blacken-docs - repo: https://github.com/asottile/blacken-docs
rev: 1.16.0 rev: 1.16.0
hooks: hooks:
@@ -17,19 +13,7 @@ repos:
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.5.0 rev: v4.5.0
hooks: hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml - id: check-yaml
- id: debug-statements
- repo: https://github.com/PyCQA/flake8
rev: 7.0.0
hooks:
- id: flake8
- repo: https://github.com/asottile/pyupgrade
rev: v3.15.1
hooks:
- id: pyupgrade
args: [--py3-plus]
- repo: local - repo: local
hooks: hooks:
- id: rst - id: rst

View File

@@ -67,10 +67,72 @@ include-package-data = false
[tool.setuptools_scm] [tool.setuptools_scm]
write_to = "src/xdist/_version.py" write_to = "src/xdist/_version.py"
[tool.flake8] [tool.ruff]
# Ignore any errors related to formatting, let black worry/fix them. src = ["src"]
ignore = ["E501", "W503", "E203"]
max-line-length = 100 [tool.ruff.format]
docstring-code-format = true
[tool.ruff.lint]
select = [
"B", # bugbear
"D", # pydocstyle
"E", # pycodestyle
"F", # pyflakes
"I", # isort
"PYI", # flake8-pyi
"UP", # pyupgrade
"RUF", # ruff
"W", # pycodestyle
"T10", # flake8-debugger
"PIE", # flake8-pie
"PGH", # pygrep-hooks
"PLE", # pylint error
"PLW", # pylint warning
"PLR1714", # Consider merging multiple comparisons
]
ignore = [
# bugbear ignore
"B011", # Do not `assert False` (`python -O` removes these calls)
"B028", # No explicit `stacklevel` keyword argument found
# pydocstyle ignore
"D100", # Missing docstring in public module
"D101", # Missing docstring in public class
"D102", # Missing docstring in public method
"D103", # Missing docstring in public function
"D104", # Missing docstring in public package
"D105", # Missing docstring in magic method
"D106", # Missing docstring in public nested class
"D107", # Missing docstring in `__init__`
"D209", # Multi-line docstring closing quotes should be on a separate line
"D205", # 1 blank line required between summary line and description
"D400", # First line should end with a period
"D401", # First line of docstring should be in imperative mood
# ruff ignore
"RUF012", # Mutable class attributes should be annotated with `typing.ClassVar`
# pylint ignore
"PLW0603", # Using the global statement
"PLW0120", # remove the else and dedent its contents
"PLW2901", # for loop variable overwritten by assignment target
"PLR5501", # Use `elif` instead of `else` then `if`
]
[tool.ruff.lint.pycodestyle]
# In order to be able to format for 88 char in ruff format
max-line-length = 120
[tool.ruff.lint.pydocstyle]
convention = "pep257"
[tool.ruff.lint.isort]
force-single-line = true
combine-as-imports = true
force-sort-within-sections = true
order-by-type = false
lines-after-imports = 2
[tool.ruff.lint.per-file-ignores]
"src/xdist/_version.py" = ["I001"]
[tool.mypy] [tool.mypy]
mypy_path = ["src"] mypy_path = ["src"]

View File

@@ -1,10 +1,9 @@
from xdist.plugin import (
is_xdist_worker,
is_xdist_master,
get_xdist_worker_id,
is_xdist_controller,
)
from xdist._version import version as __version__ from xdist._version import version as __version__
from xdist.plugin import get_xdist_worker_id
from xdist.plugin import is_xdist_controller
from xdist.plugin import is_xdist_master
from xdist.plugin import is_xdist_worker
__all__ = [ __all__ = [
"__version__", "__version__",

View File

@@ -1,7 +1,8 @@
import os
from itertools import chain from itertools import chain
import os
from pathlib import Path from pathlib import Path
from typing import Callable, Iterator from typing import Callable
from typing import Iterator
def visit_path( def visit_path(

View File

@@ -1,23 +1,22 @@
from __future__ import annotations from __future__ import annotations
from enum import auto
from enum import Enum
from queue import Empty
from queue import Queue
import sys import sys
from enum import Enum, auto
from typing import Sequence from typing import Sequence
import pytest import pytest
from xdist.remote import Producer from xdist.remote import Producer
from xdist.scheduler import EachScheduling
from xdist.scheduler import LoadFileScheduling
from xdist.scheduler import LoadGroupScheduling
from xdist.scheduler import LoadScheduling
from xdist.scheduler import LoadScopeScheduling
from xdist.scheduler import WorkStealingScheduling
from xdist.workermanage import NodeManager from xdist.workermanage import NodeManager
from xdist.scheduler import (
EachScheduling,
LoadScheduling,
LoadScopeScheduling,
LoadFileScheduling,
LoadGroupScheduling,
WorkStealingScheduling,
)
from queue import Empty, Queue
class Interrupted(KeyboardInterrupt): class Interrupted(KeyboardInterrupt):
@@ -25,7 +24,7 @@ class Interrupted(KeyboardInterrupt):
class DSession: class DSession:
"""A pytest plugin which runs a distributed test session """A pytest plugin which runs a distributed test session.
At the beginning of the test session this creates a NodeManager At the beginning of the test session this creates a NodeManager
instance which creates and starts all nodes. Nodes then emit instance which creates and starts all nodes. Nodes then emit
@@ -61,7 +60,7 @@ class DSession:
@property @property
def session_finished(self): def session_finished(self):
"""Return True if the distributed session has finished """Return True if the distributed session has finished.
This means all nodes have executed all test items. This is This means all nodes have executed all test items. This is
used by pytest_runtestloop to break out of its loop. used by pytest_runtestloop to break out of its loop.
@@ -231,9 +230,7 @@ class DSession:
) )
if maximum_reached: if maximum_reached:
if self._max_worker_restart == 0: if self._max_worker_restart == 0:
msg = "worker {} crashed and worker restarting disabled".format( msg = f"worker {node.gateway.id} crashed and worker restarting disabled"
node.gateway.id
)
else: else:
msg = "maximum crashed workers reached: %d" % self._max_worker_restart msg = "maximum crashed workers reached: %d" % self._max_worker_restart
self._summary_report = msg self._summary_report = msg
@@ -251,7 +248,7 @@ class DSession:
terminalreporter.write_sep("=", f"xdist: {self._summary_report}") terminalreporter.write_sep("=", f"xdist: {self._summary_report}")
def worker_collectionfinish(self, node, ids): def worker_collectionfinish(self, node, ids):
"""worker has finished test collection. """Worker has finished test collection.
This adds the collection for this node to the scheduler. If This adds the collection for this node to the scheduler. If
the scheduler indicates collection is finished (i.e. all the scheduler indicates collection is finished (i.e. all
@@ -464,7 +461,7 @@ class TerminalDistReporter:
rinfo = gateway._rinfo() rinfo = gateway._rinfo()
different_interpreter = rinfo.executable != sys.executable different_interpreter = rinfo.executable != sys.executable
if different_interpreter: if different_interpreter:
version = "%s.%s.%s" % rinfo.version_info[:3] version = "{}.{}.{}".format(*rinfo.version_info[:3])
self.rewrite( self.rewrite(
f"[{gateway.id}] {rinfo.platform} Python {version} cwd: {rinfo.cwd}", f"[{gateway.id}] {rinfo.platform} Python {version} cwd: {rinfo.cwd}",
newline=True, newline=True,
@@ -491,7 +488,7 @@ class TerminalDistReporter:
def get_default_max_worker_restart(config): def get_default_max_worker_restart(config):
"""gets the default value of --max-worker-restart option if it is not provided. """Gets the default value of --max-worker-restart option if it is not provided.
Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226). Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
""" """
@@ -505,7 +502,7 @@ def get_default_max_worker_restart(config):
def get_workers_status_line( def get_workers_status_line(
status_and_items: Sequence[tuple[WorkerStatus, int]] status_and_items: Sequence[tuple[WorkerStatus, int]],
) -> str: ) -> str:
""" """
Return the line to display during worker setup/collection based on the Return the line to display during worker setup/collection based on the

View File

@@ -1,21 +1,22 @@
""" """
Implement -f aka looponfailing for pytest. Implement -f aka looponfailing for pytest.
NOTE that we try to avoid loading and depending on application modules NOTE that we try to avoid loading and depending on application modules
within the controlling process (the one that starts repeatedly test within the controlling process (the one that starts repeatedly test
processes) otherwise changes to source code can crash processes) otherwise changes to source code can crash
the controlling process which should best never happen. the controlling process which should best never happen.
""" """
import os import os
from pathlib import Path from pathlib import Path
from typing import Dict, Sequence
import pytest
import sys import sys
import time import time
import execnet from typing import Dict
from typing import Sequence
from _pytest._io import TerminalWriter from _pytest._io import TerminalWriter
import execnet
import pytest
from xdist._path import visit_path from xdist._path import visit_path
@@ -253,7 +254,7 @@ class StatRecorder:
return return
time.sleep(checkinterval) time.sleep(checkinterval)
def check(self, removepycfiles: bool = True) -> bool: # noqa, too complex def check(self, removepycfiles: bool = True) -> bool:
changed = False changed = False
newstat: Dict[Path, os.stat_result] = {} newstat: Dict[Path, os.stat_result] = {}
for rootdir in self.rootdirlist: for rootdir in self.rootdirlist:

View File

@@ -17,12 +17,12 @@ import pytest
@pytest.hookspec() @pytest.hookspec()
def pytest_xdist_setupnodes(config, specs): def pytest_xdist_setupnodes(config, specs):
"""called before any remote node is set up.""" """Called before any remote node is set up."""
@pytest.hookspec() @pytest.hookspec()
def pytest_xdist_newgateway(gateway): def pytest_xdist_newgateway(gateway):
"""called on new raw gateway creation.""" """Called on new raw gateway creation."""
@pytest.hookspec( @pytest.hookspec(
@@ -31,7 +31,7 @@ def pytest_xdist_newgateway(gateway):
) )
) )
def pytest_xdist_rsyncstart(source, gateways): def pytest_xdist_rsyncstart(source, gateways):
"""called before rsyncing a directory to remote gateways takes place.""" """Called before rsyncing a directory to remote gateways takes place."""
@pytest.hookspec( @pytest.hookspec(
@@ -40,17 +40,17 @@ def pytest_xdist_rsyncstart(source, gateways):
) )
) )
def pytest_xdist_rsyncfinish(source, gateways): def pytest_xdist_rsyncfinish(source, gateways):
"""called after rsyncing a directory to remote gateways takes place.""" """Called after rsyncing a directory to remote gateways takes place."""
@pytest.hookspec(firstresult=True) @pytest.hookspec(firstresult=True)
def pytest_xdist_getremotemodule(): def pytest_xdist_getremotemodule():
"""called when creating remote node""" """Called when creating remote node."""
@pytest.hookspec() @pytest.hookspec()
def pytest_configure_node(node): def pytest_configure_node(node):
"""configure node information before it gets instantiated.""" """Configure node information before it gets instantiated."""
@pytest.hookspec() @pytest.hookspec()
@@ -65,12 +65,12 @@ def pytest_testnodedown(node, error):
@pytest.hookspec() @pytest.hookspec()
def pytest_xdist_node_collection_finished(node, ids): def pytest_xdist_node_collection_finished(node, ids):
"""called by the controller node when a worker node finishes collecting.""" """Called by the controller node when a worker node finishes collecting."""
@pytest.hookspec(firstresult=True) @pytest.hookspec(firstresult=True)
def pytest_xdist_make_scheduler(config, log): def pytest_xdist_make_scheduler(config, log):
"""return a node scheduler implementation""" """Return a node scheduler implementation."""
@pytest.hookspec(firstresult=True) @pytest.hookspec(firstresult=True)

View File

@@ -1,6 +1,6 @@
import os import os
import uuid
import sys import sys
import uuid
import warnings import warnings
import pytest import pytest
@@ -296,7 +296,7 @@ def pytest_cmdline_main(config):
if not val("collectonly") and _is_distribution_mode(config) and usepdb: if not val("collectonly") and _is_distribution_mode(config) and usepdb:
raise pytest.UsageError( raise pytest.UsageError(
"--pdb is incompatible with distributing tests; try using -n0 or -nauto." "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
) # noqa: E501 )
# ------------------------------------------------------------------------- # -------------------------------------------------------------------------
@@ -305,7 +305,7 @@ def pytest_cmdline_main(config):
def is_xdist_worker(request_or_session) -> bool: def is_xdist_worker(request_or_session) -> bool:
"""Return `True` if this is an xdist worker, `False` otherwise """Return `True` if this is an xdist worker, `False` otherwise.
:param request_or_session: the `pytest` `request` or `session` object :param request_or_session: the `pytest` `request` or `session` object
""" """
@@ -313,7 +313,7 @@ def is_xdist_worker(request_or_session) -> bool:
def is_xdist_controller(request_or_session) -> bool: def is_xdist_controller(request_or_session) -> bool:
"""Return `True` if this is the xdist controller, `False` otherwise """Return `True` if this is the xdist controller, `False` otherwise.
Note: this method also returns `False` when distribution has not been Note: this method also returns `False` when distribution has not been
activated at all. activated at all.

View File

@@ -1,21 +1,22 @@
""" """
This module is executed in remote subprocesses and helps to This module is executed in remote subprocesses and helps to
control a remote testing session and relay back information. control a remote testing session and relay back information.
It assumes that 'py' is importable and does not have dependencies It assumes that 'py' is importable and does not have dependencies
on the rest of the xdist code. This means that the xdist-plugin on the rest of the xdist code. This means that the xdist-plugin
needs not to be installed in remote environments. needs not to be installed in remote environments.
""" """
import contextlib import contextlib
import sys
import os import os
import sys
import time import time
from typing import Any from typing import Any
import pytest
from execnet.gateway_base import dumps, DumpError
from _pytest.config import _prepareconfig from _pytest.config import _prepareconfig
from execnet.gateway_base import DumpError
from execnet.gateway_base import dumps
import pytest
try: try:
from setproctitle import setproctitle from setproctitle import setproctitle
@@ -324,7 +325,7 @@ def setup_config(config, basetemp):
if __name__ == "__channelexec__": if __name__ == "__channelexec__":
channel = channel # type: ignore[name-defined] # noqa: F821 channel = channel # type: ignore[name-defined] # noqa: F821, PLW0127
workerinput, args, option_dict, change_sys_path = channel.receive() # type: ignore[name-defined] workerinput, args, option_dict, change_sys_path = channel.receive() # type: ignore[name-defined]
if change_sys_path is None: if change_sys_path is None:

View File

@@ -1,6 +1,6 @@
from xdist.scheduler.each import EachScheduling # noqa from xdist.scheduler.each import EachScheduling as EachScheduling
from xdist.scheduler.load import LoadScheduling # noqa from xdist.scheduler.load import LoadScheduling as LoadScheduling
from xdist.scheduler.loadfile import LoadFileScheduling # noqa from xdist.scheduler.loadfile import LoadFileScheduling as LoadFileScheduling
from xdist.scheduler.loadscope import LoadScopeScheduling # noqa from xdist.scheduler.loadgroup import LoadGroupScheduling as LoadGroupScheduling
from xdist.scheduler.loadgroup import LoadGroupScheduling # noqa from xdist.scheduler.loadscope import LoadScopeScheduling as LoadScopeScheduling
from xdist.scheduler.worksteal import WorkStealingScheduling # noqa from xdist.scheduler.worksteal import WorkStealingScheduling as WorkStealingScheduling

View File

@@ -1,10 +1,10 @@
from xdist.remote import Producer from xdist.remote import Producer
from xdist.workermanage import parse_spec_config
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config
class EachScheduling: class EachScheduling:
"""Implement scheduling of test items on all nodes """Implement scheduling of test items on all nodes.
If a node gets added after the test run is started then it is If a node gets added after the test run is started then it is
assumed to replace a node which got removed before it finished assumed to replace a node which got removed before it finished
@@ -48,7 +48,7 @@ class EachScheduling:
@property @property
def has_pending(self): def has_pending(self):
"""Return True if there are pending test items """Return True if there are pending test items.
This indicates that collection has finished and nodes are This indicates that collection has finished and nodes are
still processing test items, so this can be thought of as still processing test items, so this can be thought of as
@@ -64,7 +64,7 @@ class EachScheduling:
self.node2pending[node] = [] self.node2pending[node] = []
def add_node_collection(self, node, collection): def add_node_collection(self, node, collection):
"""Add the collected test items from a node """Add the collected test items from a node.
Collection is complete once all nodes have submitted their Collection is complete once all nodes have submitted their
collection. In this case its pending list is set to an empty collection. In this case its pending list is set to an empty
@@ -119,7 +119,7 @@ class EachScheduling:
return crashitem return crashitem
def schedule(self): def schedule(self):
"""Schedule the test items on the nodes """Schedule the test items on the nodes.
If the node's pending list is empty it is a new node which If the node's pending list is empty it is a new node which
needs to run all the tests. If the pending list is already needs to run all the tests. If the pending list is already

View File

@@ -3,8 +3,8 @@ from itertools import cycle
from _pytest.runner import CollectReport from _pytest.runner import CollectReport
from xdist.remote import Producer from xdist.remote import Producer
from xdist.workermanage import parse_spec_config
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config
class LoadScheduling: class LoadScheduling:
@@ -23,7 +23,7 @@ class LoadScheduling:
submit a collection. This is used to know when all nodes have submit a collection. This is used to know when all nodes have
finished collection or how large the chunks need to be created. finished collection or how large the chunks need to be created.
Attributes: Attributes::
:numnodes: The expected number of nodes taking part. The actual :numnodes: The expected number of nodes taking part. The actual
number of nodes will vary during the scheduler's lifetime as number of nodes will vary during the scheduler's lifetime as
@@ -95,7 +95,7 @@ class LoadScheduling:
@property @property
def has_pending(self): def has_pending(self):
"""Return True if there are pending test items """Return True if there are pending test items.
This indicates that collection has finished and nodes are This indicates that collection has finished and nodes are
still processing test items, so this can be thought of as still processing test items, so this can be thought of as
@@ -121,7 +121,7 @@ class LoadScheduling:
self.node2pending[node] = [] self.node2pending[node] = []
def add_node_collection(self, node, collection): def add_node_collection(self, node, collection):
"""Add the collected test items from a node """Add the collected test items from a node.
The collection is stored in the ``.node2collection`` map. The collection is stored in the ``.node2collection`` map.
Called by the ``DSession.worker_collectionfinish`` hook. Called by the ``DSession.worker_collectionfinish`` hook.
@@ -142,7 +142,7 @@ class LoadScheduling:
self.node2collection[node] = list(collection) self.node2collection[node] = list(collection)
def mark_test_complete(self, node, item_index, duration=0): def mark_test_complete(self, node, item_index, duration=0):
"""Mark test item as completed by node """Mark test item as completed by node.
The duration it took to execute the item is used as a hint to The duration it took to execute the item is used as a hint to
the scheduler. the scheduler.
@@ -161,7 +161,7 @@ class LoadScheduling:
self.check_schedule(node) self.check_schedule(node)
def check_schedule(self, node, duration=0): def check_schedule(self, node, duration=0):
"""Maybe schedule new items on the node """Maybe schedule new items on the node.
If there are any globally pending nodes left then this will If there are any globally pending nodes left then this will
check if the given node should be given any more tests. The check if the given node should be given any more tests. The
@@ -195,7 +195,7 @@ class LoadScheduling:
self.log("num items waiting for node:", len(self.pending)) self.log("num items waiting for node:", len(self.pending))
def remove_node(self, node): def remove_node(self, node):
"""Remove a node from the scheduler """Remove a node from the scheduler.
This should be called either when the node crashed or at This should be called either when the node crashed or at
shutdown time. In the former case any pending items assigned shutdown time. In the former case any pending items assigned
@@ -219,7 +219,7 @@ class LoadScheduling:
return crashitem return crashitem
def schedule(self): def schedule(self):
"""Initiate distribution of the test collection """Initiate distribution of the test collection.
Initiate scheduling of the items across the nodes. If this Initiate scheduling of the items across the nodes. If this
gets called again later it behaves the same as calling gets called again later it behaves the same as calling
@@ -243,7 +243,7 @@ class LoadScheduling:
return return
# Collections are identical, create the index of pending items. # Collections are identical, create the index of pending items.
self.collection = list(self.node2collection.values())[0] self.collection = next(iter(self.node2collection.values()))
self.pending[:] = range(len(self.collection)) self.pending[:] = range(len(self.collection))
if not self.collection: if not self.collection:
return return
@@ -260,7 +260,7 @@ class LoadScheduling:
# to each node - which is suboptimal when you have less than # to each node - which is suboptimal when you have less than
# 2 * len(nodes) tests. # 2 * len(nodes) tests.
nodes = cycle(self.nodes) nodes = cycle(self.nodes)
for i in range(len(self.pending)): for _ in range(len(self.pending)):
self._send_tests(next(nodes), 1) self._send_tests(next(nodes), 1)
else: else:
# Send batches of consecutive tests. By default, pytest sorts tests # Send batches of consecutive tests. By default, pytest sorts tests

View File

@@ -1,6 +1,7 @@
from .loadscope import LoadScopeScheduling
from xdist.remote import Producer from xdist.remote import Producer
from .loadscope import LoadScopeScheduling
class LoadFileScheduling(LoadScopeScheduling): class LoadFileScheduling(LoadScopeScheduling):
"""Implement load scheduling across nodes, but grouping test test file. """Implement load scheduling across nodes, but grouping test test file.
@@ -43,8 +44,11 @@ class LoadFileScheduling(LoadScopeScheduling):
This function will group tests with the scope determined by splitting This function will group tests with the scope determined by splitting
the first ``::`` from the left. That is, test will be grouped in a the first ``::`` from the left. That is, test will be grouped in a
single work unit when they reside in the same file. single work unit when they reside in the same file.
In the above example, scopes will be:: In the above example, scopes will be::
.. code-block:: text
example/loadsuite/test/test_beta.py example/loadsuite/test/test_beta.py
example/loadsuite/test/test_delta.py example/loadsuite/test/test_delta.py
example/loadsuite/epsilon/__init__.py example/loadsuite/epsilon/__init__.py

View File

@@ -1,6 +1,7 @@
from .loadscope import LoadScopeScheduling
from xdist.remote import Producer from xdist.remote import Producer
from .loadscope import LoadScopeScheduling
class LoadGroupScheduling(LoadScopeScheduling): class LoadGroupScheduling(LoadScopeScheduling):
"""Implement load scheduling across nodes, but grouping test by xdist_group mark. """Implement load scheduling across nodes, but grouping test by xdist_group mark.

View File

@@ -1,6 +1,7 @@
from collections import OrderedDict from collections import OrderedDict
from _pytest.runner import CollectReport from _pytest.runner import CollectReport
from xdist.remote import Producer from xdist.remote import Producer
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config from xdist.workermanage import parse_spec_config
@@ -21,7 +22,7 @@ class LoadScopeScheduling:
When created, ``numnodes`` defines how many nodes are expected to submit a When created, ``numnodes`` defines how many nodes are expected to submit a
collection. This is used to know when all nodes have finished collection. collection. This is used to know when all nodes have finished collection.
Attributes: Attributes::
:numnodes: The expected number of nodes taking part. The actual number of :numnodes: The expected number of nodes taking part. The actual number of
nodes will vary during the scheduler's lifetime as nodes are added by nodes will vary during the scheduler's lifetime as nodes are added by
@@ -207,7 +208,6 @@ class LoadScopeScheduling:
- ``DSession.worker_collectionfinish``. - ``DSession.worker_collectionfinish``.
""" """
# Check that add_node() was called on the node before # Check that add_node() was called on the node before
assert node in self.assigned_work assert node in self.assigned_work
@@ -300,7 +300,6 @@ class LoadScopeScheduling:
If there are any globally pending work units left then this will check If there are any globally pending work units left then this will check
if the given node should be given any more tests. if the given node should be given any more tests.
""" """
# Do not add more work to a node shutting down # Do not add more work to a node shutting down
if node.shutting_down: if node.shutting_down:
return return

View File

@@ -1,13 +1,19 @@
from collections import namedtuple from __future__ import annotations
from typing import Any
from typing import NamedTuple
from _pytest.runner import CollectReport from _pytest.runner import CollectReport
from xdist.remote import Producer from xdist.remote import Producer
from xdist.workermanage import parse_spec_config
from xdist.report import report_collection_diff from xdist.report import report_collection_diff
from xdist.workermanage import parse_spec_config
NodePending = namedtuple("NodePending", ["node", "pending"]) class NodePending(NamedTuple):
node: Any
pending: list[int]
# Every worker needs at least 2 tests in queue - the current and the next one. # Every worker needs at least 2 tests in queue - the current and the next one.
MIN_PENDING = 2 MIN_PENDING = 2
@@ -22,7 +28,7 @@ class WorkStealingScheduling:
test remains), an attempt is made to reassign ("steal") some tests from test remains), an attempt is made to reassign ("steal") some tests from
other nodes to this node. other nodes to this node.
Attributes: Attributes::
:numnodes: The expected number of nodes taking part. The actual :numnodes: The expected number of nodes taking part. The actual
number of nodes will vary during the scheduler's lifetime as number of nodes will vary during the scheduler's lifetime as
@@ -101,7 +107,7 @@ class WorkStealingScheduling:
@property @property
def has_pending(self): def has_pending(self):
"""Return True if there are pending test items """Return True if there are pending test items.
This indicates that collection has finished and nodes are This indicates that collection has finished and nodes are
still processing test items, so this can be thought of as still processing test items, so this can be thought of as
@@ -127,7 +133,7 @@ class WorkStealingScheduling:
self.node2pending[node] = [] self.node2pending[node] = []
def add_node_collection(self, node, collection): def add_node_collection(self, node, collection):
"""Add the collected test items from a node """Add the collected test items from a node.
The collection is stored in the ``.node2collection`` map. The collection is stored in the ``.node2collection`` map.
Called by the ``DSession.worker_collectionfinish`` hook. Called by the ``DSession.worker_collectionfinish`` hook.
@@ -148,7 +154,7 @@ class WorkStealingScheduling:
self.node2collection[node] = list(collection) self.node2collection[node] = list(collection)
def mark_test_complete(self, node, item_index, duration=None): def mark_test_complete(self, node, item_index, duration=None):
"""Mark test item as completed by node """Mark test item as completed by node.
This is called by the ``DSession.worker_testreport`` hook. This is called by the ``DSession.worker_testreport`` hook.
""" """
@@ -233,7 +239,7 @@ class WorkStealingScheduling:
self.steal_requested_from_node = steal_from.node self.steal_requested_from_node = steal_from.node
def remove_node(self, node): def remove_node(self, node):
"""Remove a node from the scheduler """Remove a node from the scheduler.
This should be called either when the node crashed or at This should be called either when the node crashed or at
shutdown time. In the former case any pending items assigned shutdown time. In the former case any pending items assigned
@@ -263,7 +269,7 @@ class WorkStealingScheduling:
return crashitem return crashitem
def schedule(self): def schedule(self):
"""Initiate distribution of the test collection """Initiate distribution of the test collection.
Initiate scheduling of the items across the nodes. If this Initiate scheduling of the items across the nodes. If this
gets called again later it behaves the same as calling gets called again later it behaves the same as calling
@@ -285,7 +291,7 @@ class WorkStealingScheduling:
return return
# Collections are identical, create the index of pending items. # Collections are identical, create the index of pending items.
self.collection = list(self.node2collection.values())[0] self.collection = next(iter(self.node2collection.values()))
self.pending[:] = range(len(self.collection)) self.pending[:] = range(len(self.collection))
if not self.collection: if not self.collection:
return return

View File

@@ -1,17 +1,23 @@
import fnmatch import fnmatch
import os import os
from pathlib import Path
import re import re
import sys import sys
from typing import Any
from typing import List
from typing import Optional
from typing import Sequence
from typing import Set
from typing import Tuple
from typing import Union
import uuid import uuid
from pathlib import Path
from typing import List, Union, Sequence, Optional, Any, Tuple, Set
import pytest
import execnet import execnet
import pytest
from xdist.plugin import _sys_path
import xdist.remote import xdist.remote
from xdist.remote import Producer from xdist.remote import Producer
from xdist.plugin import _sys_path
def parse_spec_config(config): def parse_spec_config(config):
@@ -89,8 +95,8 @@ class NodeManager:
break break
else: else:
return [] return []
import pytest
import _pytest import _pytest
import pytest
def get_dir(p): def get_dir(p):
"""Return the directory path if p is a package or the path to the .py file otherwise.""" """Return the directory path if p is a package or the path to the .py file otherwise."""
@@ -160,7 +166,7 @@ class NodeManager:
class HostRSync(execnet.RSync): class HostRSync(execnet.RSync):
"""RSyncer that filters out common files""" """RSyncer that filters out common files."""
PathLike = Union[str, "os.PathLike[str]"] PathLike = Union[str, "os.PathLike[str]"]
@@ -169,7 +175,7 @@ class HostRSync(execnet.RSync):
sourcedir: PathLike, sourcedir: PathLike,
*, *,
ignores: Optional[Sequence[PathLike]] = None, ignores: Optional[Sequence[PathLike]] = None,
**kwargs: object **kwargs: object,
) -> None: ) -> None:
if ignores is None: if ignores is None:
ignores = [] ignores = []
@@ -308,7 +314,7 @@ class WorkerController:
self._shutdown_sent = True self._shutdown_sent = True
def sendcommand(self, name, **kwargs): def sendcommand(self, name, **kwargs):
"""send a named parametrized command to the other side.""" """Send a named parametrized command to the other side."""
self.log(f"sending command {name}(**{kwargs})") self.log(f"sending command {name}(**{kwargs})")
self.channel.send((name, kwargs)) self.channel.send((name, kwargs))
@@ -316,8 +322,8 @@ class WorkerController:
self.log(f"queuing {eventname}(**{kwargs})") self.log(f"queuing {eventname}(**{kwargs})")
self.putevent((eventname, kwargs)) self.putevent((eventname, kwargs))
def process_from_remote(self, eventcall): # noqa too complex def process_from_remote(self, eventcall):
"""this gets called for each object we receive from """This gets called for each object we receive from
the other side and if the channel closes. the other side and if the channel closes.
Note that channel callbacks run in the receiver Note that channel callbacks run in the receiver
@@ -394,7 +400,7 @@ class WorkerController:
except KeyboardInterrupt: except KeyboardInterrupt:
# should not land in receiver-thread # should not land in receiver-thread
raise raise
except: # noqa except BaseException:
from _pytest._code import ExceptionInfo from _pytest._code import ExceptionInfo
excinfo = ExceptionInfo.from_current() excinfo = ExceptionInfo.from_current()
@@ -405,8 +411,8 @@ class WorkerController:
def unserialize_warning_message(data): def unserialize_warning_message(data):
import warnings
import importlib import importlib
import warnings
if data["message_module"]: if data["message_module"]:
mod = importlib.import_module(data["message_module"]) mod = importlib.import_module(data["message_module"])

View File

@@ -6,6 +6,7 @@ from typing import List
from typing import Tuple from typing import Tuple
import pytest import pytest
import xdist import xdist
@@ -265,8 +266,8 @@ class TestDistribution:
"-pfoobarplugin", "-pfoobarplugin",
"--foobar=123", "--foobar=123",
"--dist=load", "--dist=load",
"--rsyncdir=%(subdir)s" % locals(), f"--rsyncdir={subdir}",
"--tx=popen//chdir=%(dest)s" % locals(), f"--tx=popen//chdir={dest}",
p, p,
) )
assert result.ret == 0 assert result.ret == 0
@@ -481,7 +482,7 @@ class TestTerminalReporting:
) )
def test_logfinish_hook(self, pytester: pytest.Pytester) -> None: def test_logfinish_hook(self, pytester: pytest.Pytester) -> None:
"""Ensure the pytest_runtest_logfinish hook is being properly handled""" """Ensure the pytest_runtest_logfinish hook is being properly handled."""
pytester.makeconftest( pytester.makeconftest(
""" """
def pytest_runtest_logfinish(): def pytest_runtest_logfinish():
@@ -612,7 +613,7 @@ def test_fixture_teardown_failure(pytester: pytest.Pytester) -> None:
def test_config_initialization( def test_config_initialization(
pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, pytestconfig pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch, pytestconfig
) -> None: ) -> None:
"""Ensure workers and controller are initialized consistently. Integration test for #445""" """Ensure workers and controller are initialized consistently. Integration test for #445."""
pytester.makepyfile( pytester.makepyfile(
**{ **{
"dir_a/test_foo.py": """ "dir_a/test_foo.py": """
@@ -636,7 +637,7 @@ def test_config_initialization(
@pytest.mark.parametrize("when", ["setup", "call", "teardown"]) @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
def test_crashing_item(pytester, when) -> None: def test_crashing_item(pytester, when) -> None:
"""Ensure crashing item is correctly reported during all testing stages""" """Ensure crashing item is correctly reported during all testing stages."""
code = dict(setup="", call="", teardown="") code = dict(setup="", call="", teardown="")
code[when] = "os._exit(1)" code[when] = "os._exit(1)"
p = pytester.makepyfile( p = pytester.makepyfile(
@@ -656,9 +657,7 @@ def test_crashing_item(pytester, when) -> None:
def test_ok(): def test_ok():
pass pass
""".format( """.format(**code)
**code
)
) )
passes = 2 if when == "teardown" else 1 passes = 2 if when == "teardown" else 1
result = pytester.runpytest("-n2", p) result = pytester.runpytest("-n2", p)
@@ -769,7 +768,7 @@ def test_tmpdir_disabled(pytester: pytest.Pytester) -> None:
@pytest.mark.parametrize("plugin", ["xdist.looponfail"]) @pytest.mark.parametrize("plugin", ["xdist.looponfail"])
def test_sub_plugins_disabled(pytester, plugin) -> None: def test_sub_plugins_disabled(pytester, plugin) -> None:
"""Test that xdist doesn't break if we disable any of its sub-plugins. (#32)""" """Test that xdist doesn't break if we disable any of its sub-plugins (#32)."""
p1 = pytester.makepyfile( p1 = pytester.makepyfile(
""" """
def test_ok(): def test_ok():
@@ -799,9 +798,7 @@ class TestWarnings:
def test_warning_captured_deprecated_in_pytest_6( def test_warning_captured_deprecated_in_pytest_6(
self, pytester: pytest.Pytester self, pytester: pytest.Pytester
) -> None: ) -> None:
""" """Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)."""
Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
"""
from _pytest import hookspec from _pytest import hookspec
if not hasattr(hookspec, "pytest_warning_captured"): if not hasattr(hookspec, "pytest_warning_captured"):
@@ -833,7 +830,7 @@ class TestWarnings:
@pytest.mark.parametrize("n", ["-n0", "-n1"]) @pytest.mark.parametrize("n", ["-n0", "-n1"])
def test_custom_subclass(self, pytester, n) -> None: def test_custom_subclass(self, pytester, n) -> None:
"""Check that warning subclasses that don't honor the args attribute don't break """Check that warning subclasses that don't honor the args attribute don't break
pytest-xdist (#344) pytest-xdist (#344).
""" """
pytester.makepyfile( pytester.makepyfile(
""" """
@@ -1116,7 +1113,7 @@ def test_error_report_styles(pytester, tb) -> None:
def test_color_yes_collection_on_non_atty(pytester, request) -> None: def test_color_yes_collection_on_non_atty(pytester, request) -> None:
"""skip collect progress report when working on non-terminals. """Skip collect progress report when working on non-terminals.
Similar to pytest-dev/pytest#1397 Similar to pytest-dev/pytest#1397
""" """
@@ -1141,9 +1138,7 @@ def test_color_yes_collection_on_non_atty(pytester, request) -> None:
def test_without_terminal_plugin(pytester, request) -> None: def test_without_terminal_plugin(pytester, request) -> None:
""" """No output when terminal plugin is disabled."""
No output when terminal plugin is disabled
"""
pytester.makepyfile( pytester.makepyfile(
""" """
def test_1(): def test_1():
@@ -1157,9 +1152,7 @@ def test_without_terminal_plugin(pytester, request) -> None:
def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None: def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None:
""" """Internal error when using --maxfail option (#62, #65)."""
Internal error when using --maxfail option (#62, #65).
"""
pytester.makepyfile( pytester.makepyfile(
""" """
import pytest import pytest
@@ -1180,9 +1173,7 @@ def test_internal_error_with_maxfail(pytester: pytest.Pytester) -> None:
def test_maxfail_causes_early_termination(pytester: pytest.Pytester) -> None: def test_maxfail_causes_early_termination(pytester: pytest.Pytester) -> None:
""" """Ensure subsequent tests on a worker aren't run when using --maxfail (#1024)."""
Ensure subsequent tests on a worker aren't run when using --maxfail (#1024).
"""
pytester.makepyfile( pytester.makepyfile(
""" """
def test1(): def test1():
@@ -1520,9 +1511,7 @@ class TestLocking:
FILE_LOCK = filelock.FileLock("test.lock") FILE_LOCK = filelock.FileLock("test.lock")
""" + ( """ + ((_test_content * 4) % ("A", "B", "C", "D"))
(_test_content * 4) % ("A", "B", "C", "D")
)
@pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"]) @pytest.mark.parametrize("scope", ["each", "load", "loadscope", "loadfile", "no"])
def test_single_file(self, pytester, scope) -> None: def test_single_file(self, pytester, scope) -> None:

View File

@@ -1,8 +1,10 @@
import execnet
import pytest
import shutil import shutil
from typing import List from typing import List
import execnet
import pytest
pytest_plugins = "pytester" pytest_plugins = "pytester"

View File

@@ -1,16 +1,18 @@
from __future__ import annotations from __future__ import annotations
from xdist.dsession import (
DSession,
get_default_max_worker_restart,
get_workers_status_line,
WorkerStatus,
)
from xdist.report import report_collection_diff
from xdist.scheduler import EachScheduling, LoadScheduling, WorkStealingScheduling
from typing import Sequence from typing import Sequence
import pytest
import execnet import execnet
import pytest
from xdist.dsession import DSession
from xdist.dsession import get_default_max_worker_restart
from xdist.dsession import get_workers_status_line
from xdist.dsession import WorkerStatus
from xdist.report import report_collection_diff
from xdist.scheduler import EachScheduling
from xdist.scheduler import LoadScheduling
from xdist.scheduler import WorkStealingScheduling
class MockGateway: class MockGateway:
@@ -163,7 +165,7 @@ class TestLoadScheduling:
for i in range(7, 16): for i in range(7, 16):
sched.mark_test_complete(node1, i - 3) sched.mark_test_complete(node1, i - 3)
assert node1.sent == [0, 1] + list(range(4, i)) assert node1.sent == [0, 1, *range(4, i)]
assert node2.sent == [2, 3] assert node2.sent == [2, 3]
assert sched.pending == list(range(i, 16)) assert sched.pending == list(range(i, 16))
@@ -185,7 +187,7 @@ class TestLoadScheduling:
for complete_index, first_pending in enumerate(range(5, 16)): for complete_index, first_pending in enumerate(range(5, 16)):
sched.mark_test_complete(node1, node1.sent[complete_index]) sched.mark_test_complete(node1, node1.sent[complete_index])
assert node1.sent == [0, 1] + list(range(4, first_pending)) assert node1.sent == [0, 1, *range(4, first_pending)]
assert node2.sent == [2, 3] assert node2.sent == [2, 3]
assert sched.pending == list(range(first_pending, 16)) assert sched.pending == list(range(first_pending, 16))
@@ -249,9 +251,7 @@ class TestLoadScheduling:
""" """
class CollectHook: class CollectHook:
""" """Dummy hook that stores collection reports."""
Dummy hook that stores collection reports.
"""
def __init__(self): def __init__(self):
self.reports = [] self.reports = []
@@ -293,7 +293,7 @@ class TestWorkStealingScheduling:
sched.schedule() sched.schedule()
assert not sched.pending assert not sched.pending
assert not sched.tests_finished assert not sched.tests_finished
assert node1.sent == list(range(0, 8)) assert node1.sent == list(range(8))
assert node2.sent == list(range(8, 16)) assert node2.sent == list(range(8, 16))
for i in range(8): for i in range(8):
sched.mark_test_complete(node1, node1.sent[i]) sched.mark_test_complete(node1, node1.sent[i])
@@ -313,7 +313,7 @@ class TestWorkStealingScheduling:
sched.add_node_collection(node2, collection) sched.add_node_collection(node2, collection)
assert sched.collection_is_completed assert sched.collection_is_completed
sched.schedule() sched.schedule()
assert node1.sent == list(range(0, 8)) assert node1.sent == list(range(8))
assert node2.sent == list(range(8, 16)) assert node2.sent == list(range(8, 16))
for i in range(8): for i in range(8):
sched.mark_test_complete(node1, node1.sent[i]) sched.mark_test_complete(node1, node1.sent[i])

View File

@@ -1,12 +1,12 @@
import pathlib import pathlib
from pathlib import Path
import shutil
import tempfile import tempfile
import unittest.mock import textwrap
from typing import List from typing import List
import unittest.mock
import pytest import pytest
import shutil
import textwrap
from pathlib import Path
from xdist.looponfail import RemoteControl from xdist.looponfail import RemoteControl
from xdist.looponfail import StatRecorder from xdist.looponfail import StatRecorder

View File

@@ -14,9 +14,9 @@ class TestHooks:
) )
def test_runtest_logreport(self, pytester: pytest.Pytester) -> None: def test_runtest_logreport(self, pytester: pytest.Pytester) -> None:
"""Test that log reports from pytest_runtest_logreport when running """Test that log reports from pytest_runtest_logreport when running with
with xdist contain "node", "nodeid", "worker_id", and "testrun_uid" attributes. (#8) xdist contain "node", "nodeid", "worker_id", and "testrun_uid"
""" attributes (#8)."""
pytester.makeconftest( pytester.makeconftest(
""" """
def pytest_runtest_logreport(report): def pytest_runtest_logreport(report):

View File

@@ -1,19 +1,19 @@
from contextlib import suppress from contextlib import suppress
import os
from pathlib import Path from pathlib import Path
import sys import sys
import os
import execnet import execnet
from xdist.workermanage import NodeManager
import pytest import pytest
from xdist.workermanage import NodeManager
@pytest.fixture @pytest.fixture
def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch): def monkeypatch_3_cpus(monkeypatch: pytest.MonkeyPatch):
"""Make pytest-xdist believe the system has 3 CPUs""" """Make pytest-xdist believe the system has 3 CPUs."""
# block import # block import
monkeypatch.setitem(sys.modules, "psutil", None) # type: ignore monkeypatch.setitem(sys.modules, "psutil", None)
monkeypatch.delattr(os, "sched_getaffinity", raising=False) monkeypatch.delattr(os, "sched_getaffinity", raising=False)
monkeypatch.setattr(os, "cpu_count", lambda: 3) monkeypatch.setattr(os, "cpu_count", lambda: 3)

View File

@@ -1,13 +1,14 @@
import marshal
import pprint import pprint
import pytest from queue import Queue
import sys import sys
import uuid import uuid
from xdist.workermanage import WorkerController
import execnet import execnet
import marshal import pytest
from xdist.workermanage import WorkerController
from queue import Queue
WAIT_TIMEOUT = 10.0 WAIT_TIMEOUT = 10.0
@@ -15,9 +16,9 @@ WAIT_TIMEOUT = 10.0
def check_marshallable(d): def check_marshallable(d):
try: try:
marshal.dumps(d) marshal.dumps(d)
except ValueError: except ValueError as e:
pprint.pprint(d) pprint.pprint(d)
raise ValueError("not marshallable") raise ValueError("not marshallable") from e
class EventCall: class EventCall:
@@ -163,7 +164,7 @@ class TestWorkerInteractor:
worker.sendcommand("runtests_all") worker.sendcommand("runtests_all")
worker.sendcommand("shutdown") worker.sendcommand("shutdown")
for func in "::test_func", "::test_func2": for func in "::test_func", "::test_func2":
for i in range(3): # setup/call/teardown for _ in range(3): # setup/call/teardown
ev = worker.popevent("testreport") ev = worker.popevent("testreport")
assert ev.name == "testreport" assert ev.name == "testreport"
rep = unserialize_report(ev.kwargs["data"]) rep = unserialize_report(ev.kwargs["data"])
@@ -328,12 +329,10 @@ def test_remote_mainargv(pytester: pytest.Pytester) -> None:
outer_argv = sys.argv outer_argv = sys.argv
pytester.makepyfile( pytester.makepyfile(
""" f"""
def test_mainargv(request): def test_mainargv(request):
assert request.config.workerinput["mainargv"] == {!r} assert request.config.workerinput["mainargv"] == {outer_argv!r}
""".format( """
outer_argv
)
) )
result = pytester.runpytest("-n1") result = pytester.runpytest("-n1")
assert result.ret == 0 assert result.ret == 0

View File

@@ -1,14 +1,19 @@
import execnet from pathlib import Path
import pytest
import shutil import shutil
import textwrap import textwrap
import warnings import warnings
from pathlib import Path
import execnet
import pytest
from util import generate_warning from util import generate_warning
from xdist import workermanage from xdist import workermanage
from xdist._path import visit_path from xdist._path import visit_path
from xdist.remote import serialize_warning_message from xdist.remote import serialize_warning_message
from xdist.workermanage import HostRSync, NodeManager, unserialize_warning_message from xdist.workermanage import HostRSync
from xdist.workermanage import NodeManager
from xdist.workermanage import unserialize_warning_message
pytest_plugins = "pytester" pytest_plugins = "pytester"
@@ -345,8 +350,7 @@ class MyWarning(UserWarning):
], ],
) )
def test_unserialize_warning_msg(w_cls): def test_unserialize_warning_msg(w_cls):
"""Test that warning serialization process works well""" """Test that warning serialization process works well."""
# Create a test warning message # Create a test warning message
with pytest.warns(UserWarning) as w: with pytest.warns(UserWarning) as w:
if not isinstance(w_cls, str): if not isinstance(w_cls, str):
@@ -387,8 +391,7 @@ class MyWarningUnknown(UserWarning):
def test_warning_serialization_tweaked_module(): def test_warning_serialization_tweaked_module():
"""Test for GH#404""" """Test for GH#404."""
# Create a test warning message # Create a test warning message
with pytest.warns(UserWarning) as w: with pytest.warns(UserWarning) as w:
warnings.warn("hello", MyWarningUnknown) warnings.warn("hello", MyWarningUnknown)

View File

@@ -63,7 +63,3 @@ commands =
# it so they don't conflict with each other (#611). # it so they don't conflict with each other (#611).
addopts = -ra -p no:pytest-services addopts = -ra -p no:pytest-services
testpaths = testing testpaths = testing
[flake8]
max-line-length = 120
ignore = E203,W503