From 7fb7bd9d044cb9dd58e94ce42366768d5e401c75 Mon Sep 17 00:00:00 2001 From: bwilson Date: Mon, 12 Nov 2018 13:29:39 -0600 Subject: [PATCH 01/10] Add a hook to allow overriding the remote module that's executed by execnet --- xdist/newhooks.py | 5 +++++ xdist/workermanage.py | 9 ++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 7d8007f..f389192 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -30,6 +30,11 @@ def pytest_xdist_rsyncfinish(source, gateways): """ called after rsyncing a directory to remote gateways takes place. """ +@pytest.mark.firstresult +def pytest_xdist_getremotemodule(): + """ called when creating remote node""" + + def pytest_configure_node(node): """ configure node information before it gets instantiated. """ diff --git a/xdist/workermanage.py b/xdist/workermanage.py index 3b35c14..a25243a 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -203,7 +203,13 @@ def make_reltoroot(roots, args): class WorkerController(object): ENDMARK = -1 + class RemoteHook: + @pytest.mark.trylast + def pytest_xdist_getremotemodule(self): + return xdist.remote + def __init__(self, nodemanager, gateway, config, putevent): + config.pluginmanager.register(self.RemoteHook()) self.nodemanager = nodemanager self.putevent = putevent self.gateway = gateway @@ -242,7 +248,8 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - self.channel = self.gateway.remote_exec(xdist.remote) + remote_module = self.config.hook.pytest_xdist_getremotemodule() + self.channel = self.gateway.remote_exec(remote_module) self.channel.send((self.workerinput, args, option_dict)) if self.putevent: self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK) From 47d271ad9764cf3c2f35416a16a83b403cc093d4 Mon Sep 17 00:00:00 2001 From: bwilson Date: Mon, 12 Nov 2018 13:41:18 -0600 Subject: [PATCH 02/10] Adding news file --- changelog/374.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/374.feature.rst diff --git a/changelog/374.feature.rst b/changelog/374.feature.rst new file mode 100644 index 0000000..fadd4ca --- /dev/null +++ b/changelog/374.feature.rst @@ -0,0 +1 @@ +The new pytest_xdist_getremotemodule hook allows overriding the module run on remote nodes. \ No newline at end of file From 5bb35b0f26d7a90c8d8e0051ab65c95bcbb77759 Mon Sep 17 00:00:00 2001 From: bwilson Date: Mon, 12 Nov 2018 13:53:35 -0600 Subject: [PATCH 03/10] fixing linting problem with changelog --- changelog/374.feature.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/374.feature.rst b/changelog/374.feature.rst index fadd4ca..81eb85e 100644 --- a/changelog/374.feature.rst +++ b/changelog/374.feature.rst @@ -1 +1 @@ -The new pytest_xdist_getremotemodule hook allows overriding the module run on remote nodes. \ No newline at end of file +The new pytest_xdist_getremotemodule hook allows overriding the module run on remote nodes. From 311554324aed1f2db0eb4beee716a32c13e8ca17 Mon Sep 17 00:00:00 2001 From: bwilson Date: Tue, 13 Nov 2018 10:37:37 -0600 Subject: [PATCH 04/10] Adding RemoteTarget class to wrap module/function --- xdist/newhooks.py | 7 +++++-- xdist/workermanage.py | 26 ++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index f389192..b10049c 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -31,8 +31,11 @@ def pytest_xdist_rsyncfinish(source, gateways): @pytest.mark.firstresult -def pytest_xdist_getremotemodule(): - """ called when creating remote node""" +def pytest_xdist_getremotetarget(): + """ + called to get the remote target. + :return: a workermanager.RemoteTarget + """ def pytest_configure_node(node): diff --git a/xdist/workermanage.py b/xdist/workermanage.py index a25243a..a3fc382 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -3,6 +3,7 @@ import fnmatch import os import re import threading +import types import py import pytest @@ -205,8 +206,8 @@ class WorkerController(object): class RemoteHook: @pytest.mark.trylast - def pytest_xdist_getremotemodule(self): - return xdist.remote + def pytest_xdist_getremotetarget(self): + return RemoteTarget(xdist.remote) def __init__(self, nodemanager, gateway, config, putevent): config.pluginmanager.register(self.RemoteHook()) @@ -248,8 +249,8 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - remote_module = self.config.hook.pytest_xdist_getremotemodule() - self.channel = self.gateway.remote_exec(remote_module) + target, _, target_kwargs = self.config.hook.pytest_xdist_getremotetarget().pack() + self.channel = self.gateway.remote_exec(target, **target_kwargs) self.channel.send((self.workerinput, args, option_dict)) if self.putevent: self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK) @@ -359,6 +360,23 @@ class WorkerController(object): self.notify_inproc("errordown", node=self, error=excinfo) +class RemoteTarget: + def __init__(self, target, *args, **kwargs): + self._target = target + self._args = args + self._kwargs = kwargs + self.validate() + + def validate(self): + # this is based on execnet criteria + assert not self._args, "Positional arguments are not yet supported" + if isinstance(self._target, types.ModuleType): + assert not (self._args or self._kwargs), "Arguments are not used with module targets" + + def pack(self): + return self._target, self._args, self._kwargs + + def unserialize_report(name, reportdict): def assembled_report(reportdict): from _pytest._code.code import ( From b94dcfe4cd72fdf4b9803a8148900144789e9ee7 Mon Sep 17 00:00:00 2001 From: bwilson Date: Tue, 13 Nov 2018 10:45:40 -0600 Subject: [PATCH 05/10] linting --- xdist/workermanage.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xdist/workermanage.py b/xdist/workermanage.py index a3fc382..99f74e8 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -249,7 +249,9 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - target, _, target_kwargs = self.config.hook.pytest_xdist_getremotetarget().pack() + target, _, target_kwargs = ( + self.config.hook.pytest_xdist_getremotetarget().pack() + ) self.channel = self.gateway.remote_exec(target, **target_kwargs) self.channel.send((self.workerinput, args, option_dict)) if self.putevent: @@ -371,7 +373,9 @@ class RemoteTarget: # this is based on execnet criteria assert not self._args, "Positional arguments are not yet supported" if isinstance(self._target, types.ModuleType): - assert not (self._args or self._kwargs), "Arguments are not used with module targets" + assert not ( + self._args or self._kwargs + ), "Arguments are not used with module targets" def pack(self): return self._target, self._args, self._kwargs From b28c3d98c593279d754007831e35d826efeafc5f Mon Sep 17 00:00:00 2001 From: bwilson Date: Mon, 26 Nov 2018 09:49:10 -0600 Subject: [PATCH 06/10] Revert "linting" This reverts commit b94dcfe4cd72fdf4b9803a8148900144789e9ee7. --- xdist/workermanage.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xdist/workermanage.py b/xdist/workermanage.py index 99f74e8..a3fc382 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -249,9 +249,7 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - target, _, target_kwargs = ( - self.config.hook.pytest_xdist_getremotetarget().pack() - ) + target, _, target_kwargs = self.config.hook.pytest_xdist_getremotetarget().pack() self.channel = self.gateway.remote_exec(target, **target_kwargs) self.channel.send((self.workerinput, args, option_dict)) if self.putevent: @@ -373,9 +371,7 @@ class RemoteTarget: # this is based on execnet criteria assert not self._args, "Positional arguments are not yet supported" if isinstance(self._target, types.ModuleType): - assert not ( - self._args or self._kwargs - ), "Arguments are not used with module targets" + assert not (self._args or self._kwargs), "Arguments are not used with module targets" def pack(self): return self._target, self._args, self._kwargs From ab5f12f92e749bc9f0b804f47fd0ed64e85cb3da Mon Sep 17 00:00:00 2001 From: bwilson Date: Mon, 26 Nov 2018 09:49:11 -0600 Subject: [PATCH 07/10] Revert "Adding RemoteTarget class to wrap module/function" This reverts commit 311554324aed1f2db0eb4beee716a32c13e8ca17. --- xdist/newhooks.py | 7 ++----- xdist/workermanage.py | 26 ++++---------------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index b10049c..f389192 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -31,11 +31,8 @@ def pytest_xdist_rsyncfinish(source, gateways): @pytest.mark.firstresult -def pytest_xdist_getremotetarget(): - """ - called to get the remote target. - :return: a workermanager.RemoteTarget - """ +def pytest_xdist_getremotemodule(): + """ called when creating remote node""" def pytest_configure_node(node): diff --git a/xdist/workermanage.py b/xdist/workermanage.py index a3fc382..a25243a 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -3,7 +3,6 @@ import fnmatch import os import re import threading -import types import py import pytest @@ -206,8 +205,8 @@ class WorkerController(object): class RemoteHook: @pytest.mark.trylast - def pytest_xdist_getremotetarget(self): - return RemoteTarget(xdist.remote) + def pytest_xdist_getremotemodule(self): + return xdist.remote def __init__(self, nodemanager, gateway, config, putevent): config.pluginmanager.register(self.RemoteHook()) @@ -249,8 +248,8 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - target, _, target_kwargs = self.config.hook.pytest_xdist_getremotetarget().pack() - self.channel = self.gateway.remote_exec(target, **target_kwargs) + remote_module = self.config.hook.pytest_xdist_getremotemodule() + self.channel = self.gateway.remote_exec(remote_module) self.channel.send((self.workerinput, args, option_dict)) if self.putevent: self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK) @@ -360,23 +359,6 @@ class WorkerController(object): self.notify_inproc("errordown", node=self, error=excinfo) -class RemoteTarget: - def __init__(self, target, *args, **kwargs): - self._target = target - self._args = args - self._kwargs = kwargs - self.validate() - - def validate(self): - # this is based on execnet criteria - assert not self._args, "Positional arguments are not yet supported" - if isinstance(self._target, types.ModuleType): - assert not (self._args or self._kwargs), "Arguments are not used with module targets" - - def pack(self): - return self._target, self._args, self._kwargs - - def unserialize_report(name, reportdict): def assembled_report(reportdict): from _pytest._code.code import ( From ad5e35213d7cad0c22eae79bdd5c432fe772ca30 Mon Sep 17 00:00:00 2001 From: bwilson Date: Wed, 9 Jan 2019 16:17:57 -0600 Subject: [PATCH 08/10] moving py import --- xdist/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdist/remote.py b/xdist/remote.py index a0afa49..0aadff9 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -10,6 +10,7 @@ import sys import os import time +import py import _pytest.hookspec import pytest from execnet.gateway_base import dumps, DumpError @@ -258,7 +259,6 @@ if __name__ == "__channelexec__": os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"] os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"]) # os.environ['PYTHONPATH'] = importpath - import py config = remote_initconfig(option_dict, args) config.workerinput = workerinput From 26d0fcabf635cdbd7365192ee17ccfa578280eed Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 15 Feb 2019 09:31:14 -0200 Subject: [PATCH 09/10] Fix linting --- xdist/workermanage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xdist/workermanage.py b/xdist/workermanage.py index 72665a6..d08478a 100644 --- a/xdist/workermanage.py +++ b/xdist/workermanage.py @@ -250,13 +250,13 @@ class WorkerController(object): basetemp = self.config._tmpdirhandler.getbasetemp() option_dict["basetemp"] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) - + remote_module = self.config.hook.pytest_xdist_getremotemodule() - self.channel = self.gateway.remote_exec(remote_module) + self.channel = self.gateway.remote_exec(remote_module) # change sys.path only for remote workers change_sys_path = not self.gateway.spec.popen self.channel.send((self.workerinput, args, option_dict, change_sys_path)) - + if self.putevent: self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK) From 81309e8d69d711f6c3f00414e4abf27e75f4e2a6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 15 Feb 2019 09:34:41 -0200 Subject: [PATCH 10/10] Adust linting and CHANGELOG --- changelog/374.feature.rst | 2 +- changelog/{415.feature => 415.feature.rst} | 0 xdist/remote.py | 2 -- 3 files changed, 1 insertion(+), 3 deletions(-) rename changelog/{415.feature => 415.feature.rst} (100%) diff --git a/changelog/374.feature.rst b/changelog/374.feature.rst index 81eb85e..d0c6c88 100644 --- a/changelog/374.feature.rst +++ b/changelog/374.feature.rst @@ -1 +1 @@ -The new pytest_xdist_getremotemodule hook allows overriding the module run on remote nodes. +The new ``pytest_xdist_getremotemodule`` hook allows overriding the module run on remote nodes. diff --git a/changelog/415.feature b/changelog/415.feature.rst similarity index 100% rename from changelog/415.feature rename to changelog/415.feature.rst diff --git a/xdist/remote.py b/xdist/remote.py index 120eadd..c492f9e 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -262,8 +262,6 @@ def remote_initconfig(option_dict, args): if __name__ == "__channelexec__": - import py - channel = channel # noqa workerinput, args, option_dict, change_sys_path = channel.receive()