IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve typing

Browse Source 
Fix #1057
This commit is contained in:
Ran Benita
2023-04-18 23:06:34 +03:00
parent 5dfc590a8c
commit b78cf1c0ce
24 changed files with 858 additions and 477 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
src/xdist/remote.py
View File

@@ -6,16 +6,22 @@ on the rest of the xdist code. This means that the xdist-plugin
needs not to be installed in remote environments.
"""
from __future__ import annotations
import contextlib
import enum
import os
import sys
import time
from typing import Any
from typing import Generator
from typing import Literal
from typing import Sequence
from typing import TypedDict
import warnings
from _pytest.config import _prepareconfig
from execnet.gateway_base import DumpError
from execnet.gateway_base import dumps
import execnet
import pytest
@@ -23,7 +29,7 @@ try:
from setproctitle import setproctitle
except ImportError:
def setproctitle(title):
def setproctitle(title: str) -> None:
pass
@@ -35,7 +41,7 @@ class Producer:
to have the other way around.
"""
def __init__(self, name: str, *, enabled: bool = True):
def __init__(self, name: str, *, enabled: bool = True) -> None:
self.name = name
self.enabled = enabled
@@ -46,11 +52,11 @@ class Producer:
if self.enabled:
print(f"[{self.name}]", *a, **k, file=sys.stderr)
def __getattr__(self, name: str) -> "Producer":
def __getattr__(self, name: str) -> Producer:
return type(self)(name, enabled=self.enabled)
def worker_title(title):
def worker_title(title: str) -> None:
try:
setproctitle(title)
except Exception:
@@ -64,59 +70,63 @@ class Marker(enum.Enum):
class WorkerInteractor:
def __init__(self, config, channel):
def __init__(self, config: pytest.Config, channel: execnet.Channel) -> None:
self.config = config
self.workerid = config.workerinput.get("workerid", "?")
self.testrunuid = config.workerinput["testrunuid"]
workerinput: dict[str, Any] = config.workerinput # type: ignore[attr-defined]
self.workerid = workerinput.get("workerid", "?")
self.testrunuid = workerinput["testrunuid"]
self.log = Producer(f"worker-{self.workerid}", enabled=config.option.debug)
self.channel = channel
self.torun = self._make_queue()
self.nextitem_index = None
self.nextitem_index: int | None | Literal[Marker.SHUTDOWN] = None
config.pluginmanager.register(self)
def _make_queue(self):
def _make_queue(self) -> Any:
return self.channel.gateway.execmodel.queue.Queue()
def _get_next_item_index(self):
def _get_next_item_index(self) -> int | Literal[Marker.SHUTDOWN]:
"""Gets the next item from test queue. Handles the case when the queue
is replaced concurrently in another thread.
"""
result = self.torun.get()
while result is Marker.QUEUE_REPLACED:
result = self.torun.get()
return result
return result # type: ignore[no-any-return]
def sendevent(self, name, **kwargs):
def sendevent(self, name: str, **kwargs: object) -> None:
self.log("sending", name, kwargs)
self.channel.send((name, kwargs))
@pytest.hookimpl
def pytest_internalerror(self, excrepr):
def pytest_internalerror(self, excrepr: object) -> None:
formatted_error = str(excrepr)
for line in formatted_error.split("\n"):
self.log("IERROR>", line)
interactor.sendevent("internal_error", formatted_error=formatted_error)
@pytest.hookimpl
def pytest_sessionstart(self, session):
def pytest_sessionstart(self, session: pytest.Session) -> None:
self.session = session
workerinfo = getinfodict()
self.sendevent("workerready", workerinfo=workerinfo)
@pytest.hookimpl(hookwrapper=True)
def pytest_sessionfinish(self, exitstatus):
def pytest_sessionfinish(self, exitstatus: int) -> Generator[None, object, None]:
workeroutput: dict[str, Any] = self.config.workeroutput # type: ignore[attr-defined]
# in pytest 5.0+, exitstatus is an IntEnum object
self.config.workeroutput["exitstatus"] = int(exitstatus)
self.config.workeroutput["shouldfail"] = self.session.shouldfail
self.config.workeroutput["shouldstop"] = self.session.shouldstop
workeroutput["exitstatus"] = int(exitstatus)
workeroutput["shouldfail"] = self.session.shouldfail
workeroutput["shouldstop"] = self.session.shouldstop
yield
self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
self.sendevent("workerfinished", workeroutput=workeroutput)
@pytest.hookimpl
def pytest_collection(self, session):
def pytest_collection(self) -> None:
self.sendevent("collectionstart")
def handle_command(self, command):
def handle_command(
self, command: tuple[str, dict[str, Any]] | Literal[Marker.SHUTDOWN]
) -> None:
if command is Marker.SHUTDOWN:
self.torun.put(Marker.SHUTDOWN)
return
@@ -135,18 +145,19 @@ class WorkerInteractor:
elif name == "steal":
self.steal(kwargs["indices"])
def steal(self, indices):
indices = set(indices)
def steal(self, indices: Sequence[int]) -> None:
indices_set = set(indices)
stolen = []
old_queue, self.torun = self.torun, self._make_queue()
def old_queue_get_nowait_noraise():
def old_queue_get_nowait_noraise() -> int | None:
with contextlib.suppress(self.channel.gateway.execmodel.queue.Empty):
return old_queue.get_nowait()
return old_queue.get_nowait() # type: ignore[no-any-return]
return None
for i in iter(old_queue_get_nowait_noraise, None):
if i in indices:
if i in indices_set:
stolen.append(i)
else:
self.torun.put(i)
@@ -155,7 +166,7 @@ class WorkerInteractor:
old_queue.put(Marker.QUEUE_REPLACED)
@pytest.hookimpl
def pytest_runtestloop(self, session):
def pytest_runtestloop(self, session: pytest.Session) -> bool:
self.log("entering main loop")
self.channel.setcallback(self.handle_command, endmarker=Marker.SHUTDOWN)
self.nextitem_index = self._get_next_item_index()
@@ -165,7 +176,8 @@ class WorkerInteractor:
break
return True
def run_one_test(self):
def run_one_test(self) -> None:
assert isinstance(self.nextitem_index, int)
self.item_index = self.nextitem_index
self.nextitem_index = self._get_next_item_index()
@@ -174,6 +186,7 @@ class WorkerInteractor:
if self.nextitem_index is Marker.SHUTDOWN:
nextitem = None
else:
assert self.nextitem_index is not None
nextitem = items[self.nextitem_index]
worker_title("[pytest-xdist running] %s" % item.nodeid)
@@ -188,7 +201,11 @@ class WorkerInteractor:
"runtest_protocol_complete", item_index=self.item_index, duration=duration
)
def pytest_collection_modifyitems(self, session, config, items):
def pytest_collection_modifyitems(
self,
config: pytest.Config,
items: list[pytest.Item],
) -> None:
# add the group name to nodeid as suffix if --dist=loadgroup
if config.getvalue("loadgroup"):
for item in items:
@@ -203,7 +220,7 @@ class WorkerInteractor:
item._nodeid = f"{item.nodeid}@{gname}"
@pytest.hookimpl
def pytest_collection_finish(self, session):
def pytest_collection_finish(self, session: pytest.Session) -> None:
self.sendevent(
"collectionfinish",
topdir=str(self.config.rootpath),
@@ -211,15 +228,23 @@ class WorkerInteractor:
)
@pytest.hookimpl
def pytest_runtest_logstart(self, nodeid, location):
def pytest_runtest_logstart(
self,
nodeid: str,
location: tuple[str, int | None, str],
) -> None:
self.sendevent("logstart", nodeid=nodeid, location=location)
@pytest.hookimpl
def pytest_runtest_logfinish(self, nodeid, location):
def pytest_runtest_logfinish(
self,
nodeid: str,
location: tuple[str, int | None, str],
) -> None:
self.sendevent("logfinish", nodeid=nodeid, location=location)
@pytest.hookimpl
def pytest_runtest_logreport(self, report):
def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
data = self.config.hook.pytest_report_to_serializable(
config=self.config, report=report
)
@@ -230,7 +255,7 @@ class WorkerInteractor:
self.sendevent("testreport", data=data)
@pytest.hookimpl
def pytest_collectreport(self, report):
def pytest_collectreport(self, report: pytest.CollectReport) -> None:
# send only reports that have not passed to controller as optimization (#330)
if not report.passed:
data = self.config.hook.pytest_report_to_serializable(
@@ -239,7 +264,13 @@ class WorkerInteractor:
self.sendevent("collectreport", data=data)
@pytest.hookimpl
def pytest_warning_recorded(self, warning_message, when, nodeid, location):
def pytest_warning_recorded(
self,
warning_message: warnings.WarningMessage,
when: str,
nodeid: str,
location: tuple[str, int, str] | None,
) -> None:
self.sendevent(
"warning_recorded",
warning_message_data=serialize_warning_message(warning_message),
@@ -249,7 +280,9 @@ class WorkerInteractor:
)
def serialize_warning_message(warning_message):
def serialize_warning_message(
warning_message: warnings.WarningMessage,
) -> dict[str, Any]:
if isinstance(warning_message.message, Warning):
message_module = type(warning_message.message).__module__
message_class_name = type(warning_message.message).__name__
@@ -257,8 +290,8 @@ def serialize_warning_message(warning_message):
# check now if we can serialize the warning arguments (#349)
# if not, we will just use the exception message on the controller node
try:
dumps(warning_message.message.args)
except DumpError:
execnet.dumps(warning_message.message.args)
except execnet.DumpError:
message_args = None
else:
message_args = warning_message.message.args
@@ -283,27 +316,38 @@ def serialize_warning_message(warning_message):
"category_class_name": category_class_name,
}
# access private _WARNING_DETAILS because the attributes vary between Python versions
for attr_name in warning_message._WARNING_DETAILS:
for attr_name in warning_message._WARNING_DETAILS: # type: ignore[attr-defined]
if attr_name in ("message", "category"):
continue
attr = getattr(warning_message, attr_name)
# Check if we can serialize the warning detail, marking `None` otherwise
# Note that we need to define the attr (even as `None`) to allow deserializing
try:
dumps(attr)
except DumpError:
execnet.dumps(attr)
except execnet.DumpError:
result[attr_name] = repr(attr)
else:
result[attr_name] = attr
return result
def getinfodict():
class WorkerInfo(TypedDict):
version: str
version_info: tuple[int, int, int, str, int]
sysplatform: str
platform: str
executable: str
cwd: str
id: str
spec: execnet.XSpec
def getinfodict() -> WorkerInfo:
import platform
return dict(
version=sys.version,
version_info=tuple(sys.version_info),
version_info=tuple(sys.version_info), # type: ignore[typeddict-item]
sysplatform=sys.platform,
platform=platform.platform(),
executable=sys.executable,
@@ -311,7 +355,7 @@ def getinfodict():
)
def setup_config(config, basetemp):
def setup_config(config: pytest.Config, basetemp: str | None) -> None:
config.option.loadgroup = config.getvalue("dist") == "loadgroup"
config.option.looponfail = False
config.option.usepdb = False
@@ -323,7 +367,7 @@ def setup_config(config, basetemp):
if __name__ == "__channelexec__":
channel = channel # type: ignore[name-defined] # noqa: F821, PLW0127
channel: execnet.Channel = channel # type: ignore[name-defined] # noqa: F821, PLW0127
workerinput, args, option_dict, change_sys_path = channel.receive() # type: ignore[name-defined]
if change_sys_path is None: