From 7d099b65c246844036575a1d3802d0d244bf3ea3 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Sun, 2 Oct 2016 15:46:34 +0500 Subject: [PATCH 01/14] Allow custom scheduler class --- xdist/dsession.py | 24 +++++++++++++++--------- xdist/plugin.py | 4 ++++ xdist/remote.py | 1 + 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 46f76a9..8e87fa4 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -1,4 +1,5 @@ import difflib +import importlib import itertools from _pytest.runner import CollectReport @@ -24,7 +25,7 @@ class EachScheduling: assigned the remaining items from the removed node. """ - def __init__(self, numnodes, log=None): + def __init__(self, numnodes, log=None, config=None): self.numnodes = numnodes self.node2collection = {} self.node2pending = {} @@ -485,6 +486,17 @@ class DSession: self.trdist = TerminalDistReporter(config) config.pluginmanager.register(self.trdist, "terminaldistreporter") + self.sched = None + dist = self.config.getvalue("dist") + if dist == "load": + self.ScheduleCls = LoadScheduling + elif dist == "each": + self.ScheduleCls = EachScheduling + else: + mod_name, cls_name = dist.rsplit('.', 1) + mod = importlib.import_module(mod_name) + self.ScheduleCls = getattr(mod, cls_name) + @property def session_finished(self): """Return True if the distributed session has finished @@ -523,15 +535,9 @@ class DSession: def pytest_runtestloop(self): numnodes = len(self.nodemanager.specs) - dist = self.config.getvalue("dist") - if dist == "load": - self.sched = LoadScheduling(numnodes, log=self.log, - config=self.config) - elif dist == "each": - self.sched = EachScheduling(numnodes, log=self.log) - else: - assert 0, dist self.shouldstop = False + self.sched = self.ScheduleCls(numnodes, log=self.log, + config=self.config) while not self.session_finished: self.loop_once() if self.shouldstop: diff --git a/xdist/plugin.py b/xdist/plugin.py index dc721da..77c81e8 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -37,6 +37,8 @@ def pytest_addoption(parser): "each: send each test to each available environment.\n\n" "load: send each test to available environment.\n\n" "(default) no: run tests inprocess, don't distribute.")) + group._addoption('--dc', dest='distcustom', + help="Custom scheduler implementation") group._addoption( '--tx', dest="tx", action="append", default=[], metavar="xspec", @@ -99,6 +101,8 @@ def pytest_cmdline_main(config): config.option.tx = ['popen'] * config.option.numprocesses if config.option.distload: config.option.dist = "load" + if config.option.distcustom: + config.option.dist = config.option.distcustom val = config.getvalue if not val("collectonly"): usepdb = config.getoption('usepdb') # a core option diff --git a/xdist/remote.py b/xdist/remote.py index 40bbeac..31dcc19 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -131,6 +131,7 @@ def remote_initconfig(option_dict, args): config.option.usepdb = False config.option.dist = "no" config.option.distload = False + config.option.distcustom = None config.option.numprocesses = None config.args = args return config From d5ba3804bbe9bf62fef89572188b7cd9dd0d6dbb Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 3 Oct 2016 15:05:29 +0500 Subject: [PATCH 02/14] Use __import__ instead of importlib for py26 support --- xdist/dsession.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 8e87fa4..4a998c0 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -1,5 +1,4 @@ import difflib -import importlib import itertools from _pytest.runner import CollectReport @@ -494,7 +493,7 @@ class DSession: self.ScheduleCls = EachScheduling else: mod_name, cls_name = dist.rsplit('.', 1) - mod = importlib.import_module(mod_name) + mod = __import__(mod_name) self.ScheduleCls = getattr(mod, cls_name) @property From ffcbe4468505a588d51aa32e1318a5c3bd662070 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 3 Oct 2016 15:29:08 +0500 Subject: [PATCH 03/14] Fix codestyle --- xdist/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdist/plugin.py b/xdist/plugin.py index 77c81e8..a144f20 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -38,7 +38,7 @@ def pytest_addoption(parser): "load: send each test to available environment.\n\n" "(default) no: run tests inprocess, don't distribute.")) group._addoption('--dc', dest='distcustom', - help="Custom scheduler implementation") + help="Custom scheduler implementation") group._addoption( '--tx', dest="tx", action="append", default=[], metavar="xspec", From 3165b78936135f76c6eccd91d17fd3f260b84734 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 3 Oct 2016 15:13:01 +0500 Subject: [PATCH 04/14] Test custom scheduler plugin config --- testing/test_plugin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/testing/test_plugin.py b/testing/test_plugin.py index 189a1bf..b1614ab 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -25,6 +25,13 @@ def test_dist_options(testdir): assert config.option.dist == "load" +def test_dist_custom_options(testdir): + from xdist.plugin import pytest_cmdline_main as check_options + config = testdir.parseconfigure("-n2", "--dc=foo.bar.Class") + check_options(config) + assert config.option.dist == "foo.bar.Class" + + def test_auto_detect_cpus(testdir, monkeypatch): import os if hasattr(os, 'cpu_count'): From 2ff2772521c1c443fd70c22d1ce5f3219d7c8c7e Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 10 Oct 2016 19:33:14 +0500 Subject: [PATCH 05/14] Use pytest_xdist_make_scheduler hook to set up scheduller instance --- xdist/dsession.py | 26 +++++++++++++------------- xdist/newhooks.py | 7 +++++++ xdist/plugin.py | 4 ---- xdist/remote.py | 1 - 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 4a998c0..7f03326 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -485,17 +485,6 @@ class DSession: self.trdist = TerminalDistReporter(config) config.pluginmanager.register(self.trdist, "terminaldistreporter") - self.sched = None - dist = self.config.getvalue("dist") - if dist == "load": - self.ScheduleCls = LoadScheduling - elif dist == "each": - self.ScheduleCls = EachScheduling - else: - mod_name, cls_name = dist.rsplit('.', 1) - mod = __import__(mod_name) - self.ScheduleCls = getattr(mod, cls_name) - @property def session_finished(self): """Return True if the distributed session has finished @@ -532,11 +521,22 @@ class DSession: # prohibit collection of test items in master process return True + @pytest.hookimpl(trylast=True) + def pytest_xdist_make_scheduler(self, numnodes, log, config): + dist = config.getvalue("dist") + if dist == "load": + return LoadScheduling(numnodes, log=log, config=config) + elif dist == "each": + return EachScheduling(numnodes, log=log, config=config) + def pytest_runtestloop(self): numnodes = len(self.nodemanager.specs) + self.sched = self.config.hook.pytest_xdist_make_scheduler(numnodes=numnodes, + log=self.log, + config=self.config) + assert self.sched is not None + self.shouldstop = False - self.sched = self.ScheduleCls(numnodes, log=self.log, - config=self.config) while not self.session_finished: self.loop_once() if self.shouldstop: diff --git a/xdist/newhooks.py b/xdist/newhooks.py index d31aed8..5942f29 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -11,6 +11,7 @@ must be taken in plugins in case ``xdist`` is not installed. Please see: http://pytest.org/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins """ +import pytest def pytest_xdist_setupnodes(config, specs): @@ -44,3 +45,9 @@ def pytest_testnodedown(node, error): def pytest_xdist_node_collection_finished(node, ids): """called by the master node when a node finishes collecting. """ + + +@pytest.hookspec(firstresult=True) +def pytest_xdist_make_scheduler(numnodes, log, config): + """Return a node scheduler implementation, or None. + """ diff --git a/xdist/plugin.py b/xdist/plugin.py index a144f20..dc721da 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -37,8 +37,6 @@ def pytest_addoption(parser): "each: send each test to each available environment.\n\n" "load: send each test to available environment.\n\n" "(default) no: run tests inprocess, don't distribute.")) - group._addoption('--dc', dest='distcustom', - help="Custom scheduler implementation") group._addoption( '--tx', dest="tx", action="append", default=[], metavar="xspec", @@ -101,8 +99,6 @@ def pytest_cmdline_main(config): config.option.tx = ['popen'] * config.option.numprocesses if config.option.distload: config.option.dist = "load" - if config.option.distcustom: - config.option.dist = config.option.distcustom val = config.getvalue if not val("collectonly"): usepdb = config.getoption('usepdb') # a core option diff --git a/xdist/remote.py b/xdist/remote.py index 31dcc19..40bbeac 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -131,7 +131,6 @@ def remote_initconfig(option_dict, args): config.option.usepdb = False config.option.dist = "no" config.option.distload = False - config.option.distcustom = None config.option.numprocesses = None config.args = args return config From cf863fd38abd5d76fe1ad780e3f7d70541ee5dc9 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 10 Oct 2016 19:41:45 +0500 Subject: [PATCH 06/14] Remove unnecessary test --- testing/test_plugin.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/testing/test_plugin.py b/testing/test_plugin.py index b1614ab..189a1bf 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -25,13 +25,6 @@ def test_dist_options(testdir): assert config.option.dist == "load" -def test_dist_custom_options(testdir): - from xdist.plugin import pytest_cmdline_main as check_options - config = testdir.parseconfigure("-n2", "--dc=foo.bar.Class") - check_options(config) - assert config.option.dist == "foo.bar.Class" - - def test_auto_detect_cpus(testdir, monkeypatch): import os if hasattr(os, 'cpu_count'): From 753f9241a760495117876a9affece7a555cc27bb Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Mon, 10 Oct 2016 21:07:31 +0500 Subject: [PATCH 07/14] Support older pytest versions --- xdist/dsession.py | 2 +- xdist/newhooks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 7f03326..3ae2e03 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -521,7 +521,7 @@ class DSession: # prohibit collection of test items in master process return True - @pytest.hookimpl(trylast=True) + @pytest.mark.trylast def pytest_xdist_make_scheduler(self, numnodes, log, config): dist = config.getvalue("dist") if dist == "load": diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 5942f29..5bffa7b 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -47,7 +47,7 @@ def pytest_xdist_node_collection_finished(node, ids): """ -@pytest.hookspec(firstresult=True) +@pytest.mark.firstresult def pytest_xdist_make_scheduler(numnodes, log, config): """Return a node scheduler implementation, or None. """ From 81003df660fd8e3fdf81f41b5deb61d14a933d40 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Tue, 18 Oct 2016 19:52:00 +0500 Subject: [PATCH 08/14] Fix codestyle --- xdist/dsession.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 3ae2e03..c930213 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -531,9 +531,11 @@ class DSession: def pytest_runtestloop(self): numnodes = len(self.nodemanager.specs) - self.sched = self.config.hook.pytest_xdist_make_scheduler(numnodes=numnodes, - log=self.log, - config=self.config) + self.sched = self.config.hook.pytest_xdist_make_scheduler( + numnodes=numnodes, + log=self.log, + config=self.config + ) assert self.sched is not None self.shouldstop = False From 27ffbbb4f7ba5614af12d9acdf044e2e484ec74c Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Thu, 20 Oct 2016 01:07:39 +0500 Subject: [PATCH 09/14] Unified scheduler class interface --- testing/test_dsession.py | 37 ++++++++++++++++++++++--------------- xdist/dsession.py | 16 +++++++++------- xdist/newhooks.py | 5 ++--- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index f33fc41..2abd976 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -49,10 +49,11 @@ def dumpqueue(queue): class TestEachScheduling: - def test_schedule_load_simple(self): + def test_schedule_load_simple(self, testdir): node1 = MockNode() node2 = MockNode() - sched = EachScheduling(2) + config = testdir.parseconfig() + sched = EachScheduling(2, config) sched.addnode(node1) sched.addnode(node2) collection = ["a.py::test_1", ] @@ -72,9 +73,10 @@ class TestEachScheduling: sched.remove_item(node2, 0) assert sched.tests_finished() - def test_schedule_remove_node(self): + def test_schedule_remove_node(self, testdir): node1 = MockNode() - sched = EachScheduling(1) + config = testdir.parseconfig() + sched = EachScheduling(1, config) sched.addnode(node1) collection = ["a.py::test_1", ] assert not sched.collection_is_completed @@ -90,8 +92,9 @@ class TestEachScheduling: class TestLoadScheduling: - def test_schedule_load_simple(self): - sched = LoadScheduling(2) + def test_schedule_load_simple(self, testdir): + config = testdir.parseconfig() + sched = LoadScheduling(2, config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -113,8 +116,9 @@ class TestLoadScheduling: sched.remove_item(node1, node1.sent[0]) assert sched.tests_finished() - def test_init_distribute_batch_size(self): - sched = LoadScheduling(2) + def test_init_distribute_batch_size(self, testdir): + config = testdir.parseconfig() + sched = LoadScheduling(2, config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -139,8 +143,9 @@ class TestLoadScheduling: assert node1.sent == [0, 2, 4, 5] assert not sched.pending - def test_init_distribute_fewer_tests_than_nodes(self): - sched = LoadScheduling(2) + def test_init_distribute_fewer_tests_than_nodes(self, testdir): + config = testdir.parseconfig() + sched = LoadScheduling(2, config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -158,8 +163,9 @@ class TestLoadScheduling: assert sent3 == [] assert not sched.pending - def test_init_distribute_fewer_than_two_tests_per_node(self): - sched = LoadScheduling(2) + def test_init_distribute_fewer_than_two_tests_per_node(self, testdir): + config = testdir.parseconfig() + sched = LoadScheduling(2, config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -177,9 +183,10 @@ class TestLoadScheduling: assert sent3 == [2] assert not sched.pending - def test_add_remove_node(self): + def test_add_remove_node(self, testdir): node = MockNode() - sched = LoadScheduling(1) + config = testdir.parseconfig() + sched = LoadScheduling(1, config) sched.addnode(node) collection = ["test_file.py::test_func"] sched.addnode_collection(node, collection) @@ -211,7 +218,7 @@ class TestLoadScheduling: config.pluginmanager.register(collect_hook, "collect_hook") node1 = MockNode() node2 = MockNode() - sched = LoadScheduling(2, config=config) + sched = LoadScheduling(2, config) sched.addnode(node1) sched.addnode(node2) sched.addnode_collection(node1, ["a.py::test_1"]) diff --git a/xdist/dsession.py b/xdist/dsession.py index c930213..f33f649 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -24,7 +24,7 @@ class EachScheduling: assigned the remaining items from the removed node. """ - def __init__(self, numnodes, log=None, config=None): + def __init__(self, numnodes, config, log=None): self.numnodes = numnodes self.node2collection = {} self.node2pending = {} @@ -181,7 +181,7 @@ class LoadScheduling: :config: Config object, used for handling hooks. """ - def __init__(self, numnodes, log=None, config=None): + def __init__(self, numnodes, config, log=None): self.numnodes = numnodes self.node2collection = {} self.node2pending = {} @@ -466,6 +466,8 @@ class DSession: self.log = py.log.Producer("dsession") if not config.option.debug: py.log.setconsumer(self.log._keywords, None) + self.nodemanager = None + self.sched = None self.shuttingdown = False self.countfailures = 0 self.maxfail = config.getvalue("maxfail") @@ -522,19 +524,19 @@ class DSession: return True @pytest.mark.trylast - def pytest_xdist_make_scheduler(self, numnodes, log, config): + def pytest_xdist_make_scheduler(self, numnodes, config, log): dist = config.getvalue("dist") if dist == "load": - return LoadScheduling(numnodes, log=log, config=config) + return LoadScheduling(numnodes, config, log) elif dist == "each": - return EachScheduling(numnodes, log=log, config=config) + return EachScheduling(numnodes, config, log) def pytest_runtestloop(self): numnodes = len(self.nodemanager.specs) self.sched = self.config.hook.pytest_xdist_make_scheduler( numnodes=numnodes, - log=self.log, - config=self.config + config=self.config, + log=self.log ) assert self.sched is not None diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 5bffa7b..e98b0e2 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -48,6 +48,5 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult -def pytest_xdist_make_scheduler(numnodes, log, config): - """Return a node scheduler implementation, or None. - """ +def pytest_xdist_make_scheduler(numnodes, config, log): + """ return a node scheduler implementation """ From a2aaf9bf04a694ab4dee9d2daf8f6c573b8892eb Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Thu, 20 Oct 2016 09:32:59 +0500 Subject: [PATCH 10/14] Rename init_distribute() to more clear schedule() --- testing/test_dsession.py | 22 +++++++++++----------- xdist/dsession.py | 14 ++++++-------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 2abd976..dbc96a1 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -64,7 +64,7 @@ class TestEachScheduling: assert sched.collection_is_completed assert sched.node2collection[node1] == collection assert sched.node2collection[node2] == collection - sched.init_distribute() + sched.schedule() assert sched.tests_finished() assert node1.sent == ['ALL'] assert node2.sent == ['ALL'] @@ -83,7 +83,7 @@ class TestEachScheduling: sched.addnode_collection(node1, collection) assert sched.collection_is_completed assert sched.node2collection[node1] == collection - sched.init_distribute() + sched.schedule() assert sched.tests_finished() crashitem = sched.remove_node(node1) assert crashitem @@ -106,7 +106,7 @@ class TestLoadScheduling: assert sched.collection_is_completed assert sched.node2collection[node1] == collection assert sched.node2collection[node2] == collection - sched.init_distribute() + sched.schedule() assert not sched.pending assert sched.tests_finished() assert len(node1.sent) == 1 @@ -116,7 +116,7 @@ class TestLoadScheduling: sched.remove_item(node1, node1.sent[0]) assert sched.tests_finished() - def test_init_distribute_batch_size(self, testdir): + def test_schedule_batch_size(self, testdir): config = testdir.parseconfig() sched = LoadScheduling(2, config) sched.addnode(MockNode()) @@ -125,7 +125,7 @@ class TestLoadScheduling: col = ["xyz"] * (6) sched.addnode_collection(node1, col) sched.addnode_collection(node2, col) - sched.init_distribute() + sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent sent2 = node2.sent @@ -143,7 +143,7 @@ class TestLoadScheduling: assert node1.sent == [0, 2, 4, 5] assert not sched.pending - def test_init_distribute_fewer_tests_than_nodes(self, testdir): + def test_schedule_fewer_tests_than_nodes(self, testdir): config = testdir.parseconfig() sched = LoadScheduling(2, config) sched.addnode(MockNode()) @@ -153,7 +153,7 @@ class TestLoadScheduling: col = ["xyz"] * 2 sched.addnode_collection(node1, col) sched.addnode_collection(node2, col) - sched.init_distribute() + sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent sent2 = node2.sent @@ -163,7 +163,7 @@ class TestLoadScheduling: assert sent3 == [] assert not sched.pending - def test_init_distribute_fewer_than_two_tests_per_node(self, testdir): + def test_schedule_fewer_than_two_tests_per_node(self, testdir): config = testdir.parseconfig() sched = LoadScheduling(2, config) sched.addnode(MockNode()) @@ -173,7 +173,7 @@ class TestLoadScheduling: col = ["xyz"] * 5 sched.addnode_collection(node1, col) sched.addnode_collection(node2, col) - sched.init_distribute() + sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent sent2 = node2.sent @@ -191,7 +191,7 @@ class TestLoadScheduling: collection = ["test_file.py::test_func"] sched.addnode_collection(node, collection) assert sched.collection_is_completed - sched.init_distribute() + sched.schedule() assert not sched.pending crashitem = sched.remove_node(node) assert crashitem == collection[0] @@ -223,7 +223,7 @@ class TestLoadScheduling: sched.addnode(node2) sched.addnode_collection(node1, ["a.py::test_1"]) sched.addnode_collection(node2, ["a.py::test_2"]) - sched.init_distribute() + sched.schedule() assert len(collect_hook.reports) == 1 rep = collect_hook.reports[0] assert 'Different tests were collected between' in rep.longrepr diff --git a/xdist/dsession.py b/xdist/dsession.py index f33f649..3032e11 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -78,7 +78,7 @@ class EachScheduling: list. When the collection is already completed this submission is from a node which was restarted to replace a dead node. In this case we already assign the pending items - here. In either case ``.init_distribute()`` will instruct the + here. In either case ``.schedule()`` will instruct the node to start running the required tests. """ assert node in self.node2pending @@ -115,7 +115,7 @@ class EachScheduling: self._removed2pending[node] = pending return crashitem - def init_distribute(self): + def schedule(self): """Schedule the test items on the nodes If the node's pending list is empty it is a new node which @@ -170,7 +170,7 @@ class LoadScheduling: :collection: The one collection once it is validated to be identical between all the nodes. It is initialised to None - until ``.init_distribute()`` is called. + until ``.schedule()`` is called. :pending: List of indices of globally pending tests. These are tests which have not yet been allocated to a chunk for a node @@ -258,7 +258,7 @@ class LoadScheduling: assert node in self.node2pending if self.collection_is_completed: # A new node has been added later, perhaps an original one died. - # .init_distribute() should have + # .schedule() should have # been called by now assert self.collection if collection != self.collection: @@ -335,7 +335,7 @@ class LoadScheduling: self.check_schedule(node) return crashitem - def init_distribute(self): + def schedule(self): """Initiate distribution of the test collection Initiate scheduling of the items across the nodes. If this @@ -345,8 +345,6 @@ class LoadScheduling: This is called by the ``DSession.slave_collectionfinish`` hook if ``.collection_is_completed`` is True. - - XXX Perhaps this method should have been called ".schedule()". """ assert self.collection_is_completed @@ -653,7 +651,7 @@ class DSession: self.terminal.write_line("") self.terminal.write_line("scheduling tests via %s" % ( self.sched.__class__.__name__)) - self.sched.init_distribute() + self.sched.schedule() def slave_logstart(self, node, nodeid, location): """Emitted when a node calls the pytest_runtest_logstart hook.""" From 6200123fac00f2a33db816a9b5cd5d497078e395 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Tue, 14 Feb 2017 20:19:41 +0500 Subject: [PATCH 11/14] Made EachScheduling.numnodes countable from config --- testing/test_dsession.py | 32 ++++++++++++++++---------------- xdist/dsession.py | 19 +++++++++---------- xdist/newhooks.py | 2 +- xdist/slavemanage.py | 30 +++++++++++++++++------------- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index dbc96a1..81b35e6 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -52,8 +52,8 @@ class TestEachScheduling: def test_schedule_load_simple(self, testdir): node1 = MockNode() node2 = MockNode() - config = testdir.parseconfig() - sched = EachScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = EachScheduling(config) sched.addnode(node1) sched.addnode(node2) collection = ["a.py::test_1", ] @@ -75,8 +75,8 @@ class TestEachScheduling: def test_schedule_remove_node(self, testdir): node1 = MockNode() - config = testdir.parseconfig() - sched = EachScheduling(1, config) + config = testdir.parseconfig("--tx=popen") + sched = EachScheduling(config) sched.addnode(node1) collection = ["a.py::test_1", ] assert not sched.collection_is_completed @@ -93,8 +93,8 @@ class TestEachScheduling: class TestLoadScheduling: def test_schedule_load_simple(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -117,8 +117,8 @@ class TestLoadScheduling: assert sched.tests_finished() def test_schedule_batch_size(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -144,8 +144,8 @@ class TestLoadScheduling: assert not sched.pending def test_schedule_fewer_tests_than_nodes(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -164,8 +164,8 @@ class TestLoadScheduling: assert not sched.pending def test_schedule_fewer_than_two_tests_per_node(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -185,8 +185,8 @@ class TestLoadScheduling: def test_add_remove_node(self, testdir): node = MockNode() - config = testdir.parseconfig() - sched = LoadScheduling(1, config) + config = testdir.parseconfig("--tx=popen") + sched = LoadScheduling(config) sched.addnode(node) collection = ["test_file.py::test_func"] sched.addnode_collection(node, collection) @@ -214,11 +214,11 @@ class TestLoadScheduling: self.reports.append(report) collect_hook = CollectHook() - config = testdir.parseconfig() + config = testdir.parseconfig("--tx=2*popen") config.pluginmanager.register(collect_hook, "collect_hook") node1 = MockNode() node2 = MockNode() - sched = LoadScheduling(2, config) + sched = LoadScheduling(config) sched.addnode(node1) sched.addnode(node2) sched.addnode_collection(node1, ["a.py::test_1"]) diff --git a/xdist/dsession.py b/xdist/dsession.py index 3032e11..c22c865 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -4,7 +4,7 @@ from _pytest.runner import CollectReport import pytest import py -from xdist.slavemanage import NodeManager +from xdist.slavemanage import NodeManager, parse_spec_config queue = py.builtin._tryimport('queue', 'Queue') @@ -24,8 +24,9 @@ class EachScheduling: assigned the remaining items from the removed node. """ - def __init__(self, numnodes, config, log=None): - self.numnodes = numnodes + def __init__(self, config, log=None): + self.config = config + self.numnodes = len(parse_spec_config(config)) self.node2collection = {} self.node2pending = {} self._started = [] @@ -181,8 +182,8 @@ class LoadScheduling: :config: Config object, used for handling hooks. """ - def __init__(self, numnodes, config, log=None): - self.numnodes = numnodes + def __init__(self, config, log=None): + self.numnodes = len(parse_spec_config(config)) self.node2collection = {} self.node2pending = {} self.pending = [] @@ -522,17 +523,15 @@ class DSession: return True @pytest.mark.trylast - def pytest_xdist_make_scheduler(self, numnodes, config, log): + def pytest_xdist_make_scheduler(self, config, log): dist = config.getvalue("dist") if dist == "load": - return LoadScheduling(numnodes, config, log) + return LoadScheduling(config, log) elif dist == "each": - return EachScheduling(numnodes, config, log) + return EachScheduling(config, log) def pytest_runtestloop(self): - numnodes = len(self.nodemanager.specs) self.sched = self.config.hook.pytest_xdist_make_scheduler( - numnodes=numnodes, config=self.config, log=self.log ) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index e98b0e2..364fb36 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -48,5 +48,5 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult -def pytest_xdist_make_scheduler(numnodes, config, log): +def pytest_xdist_make_scheduler(config, log): """ return a node scheduler implementation """ diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index ffac07e..bb44828 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -10,6 +10,22 @@ import xdist.remote from _pytest import runner # XXX load dynamically +def parse_spec_config(config): + xspeclist = [] + for xspec in config.getvalue("tx"): + i = xspec.find("*") + try: + num = int(xspec[:i]) + except ValueError: + xspeclist.append(xspec) + else: + xspeclist.extend([xspec[i + 1:]] * num) + if not xspeclist: + raise pytest.UsageError( + "MISSING test execution (tx) nodes: please specify --tx") + return xspeclist + + class NodeManager(object): EXIT_TIMEOUT = 10 DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~'] @@ -62,19 +78,7 @@ class NodeManager(object): self.group.terminate(self.EXIT_TIMEOUT) def _getxspecs(self): - xspeclist = [] - for xspec in self.config.getvalue("tx"): - i = xspec.find("*") - try: - num = int(xspec[:i]) - except ValueError: - xspeclist.append(xspec) - else: - xspeclist.extend([xspec[i+1:]] * num) - if not xspeclist: - raise pytest.UsageError( - "MISSING test execution (tx) nodes: please specify --tx") - return [execnet.XSpec(x) for x in xspeclist] + return [execnet.XSpec(x) for x in parse_spec_config(self.config)] def _getrsyncdirs(self): for spec in self.specs: From a99e6e7ca84e56269364bf43f11adb2906afb480 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Wed, 15 Feb 2017 00:21:26 +0500 Subject: [PATCH 12/14] Clean up scheduller interfaces --- testing/test_dsession.py | 62 ++++++++++++++++++++-------------------- xdist/dsession.py | 26 ++++++++--------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 81b35e6..28f1586 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -54,13 +54,13 @@ class TestEachScheduling: node2 = MockNode() config = testdir.parseconfig("--tx=2*popen") sched = EachScheduling(config) - sched.addnode(node1) - sched.addnode(node2) + sched.add_node(node1) + sched.add_node(node2) collection = ["a.py::test_1", ] assert not sched.collection_is_completed - sched.addnode_collection(node1, collection) + sched.add_node_collection(node1, collection) assert not sched.collection_is_completed - sched.addnode_collection(node2, collection) + sched.add_node_collection(node2, collection) assert sched.collection_is_completed assert sched.node2collection[node1] == collection assert sched.node2collection[node2] == collection @@ -77,10 +77,10 @@ class TestEachScheduling: node1 = MockNode() config = testdir.parseconfig("--tx=popen") sched = EachScheduling(config) - sched.addnode(node1) + sched.add_node(node1) collection = ["a.py::test_1", ] assert not sched.collection_is_completed - sched.addnode_collection(node1, collection) + sched.add_node_collection(node1, collection) assert sched.collection_is_completed assert sched.node2collection[node1] == collection sched.schedule() @@ -88,21 +88,21 @@ class TestEachScheduling: crashitem = sched.remove_node(node1) assert crashitem assert sched.tests_finished() - assert not sched.hasnodes() + assert not sched.has_nodes() class TestLoadScheduling: def test_schedule_load_simple(self, testdir): config = testdir.parseconfig("--tx=2*popen") sched = LoadScheduling(config) - sched.addnode(MockNode()) - sched.addnode(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) node1, node2 = sched.nodes collection = ["a.py::test_1", "a.py::test_2"] assert not sched.collection_is_completed - sched.addnode_collection(node1, collection) + sched.add_node_collection(node1, collection) assert not sched.collection_is_completed - sched.addnode_collection(node2, collection) + sched.add_node_collection(node2, collection) assert sched.collection_is_completed assert sched.node2collection[node1] == collection assert sched.node2collection[node2] == collection @@ -119,12 +119,12 @@ class TestLoadScheduling: def test_schedule_batch_size(self, testdir): config = testdir.parseconfig("--tx=2*popen") sched = LoadScheduling(config) - sched.addnode(MockNode()) - sched.addnode(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) node1, node2 = sched.nodes col = ["xyz"] * (6) - sched.addnode_collection(node1, col) - sched.addnode_collection(node2, col) + sched.add_node_collection(node1, col) + sched.add_node_collection(node2, col) sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent @@ -146,13 +146,13 @@ class TestLoadScheduling: def test_schedule_fewer_tests_than_nodes(self, testdir): config = testdir.parseconfig("--tx=2*popen") sched = LoadScheduling(config) - sched.addnode(MockNode()) - sched.addnode(MockNode()) - sched.addnode(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) node1, node2, node3 = sched.nodes col = ["xyz"] * 2 - sched.addnode_collection(node1, col) - sched.addnode_collection(node2, col) + sched.add_node_collection(node1, col) + sched.add_node_collection(node2, col) sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent @@ -166,13 +166,13 @@ class TestLoadScheduling: def test_schedule_fewer_than_two_tests_per_node(self, testdir): config = testdir.parseconfig("--tx=2*popen") sched = LoadScheduling(config) - sched.addnode(MockNode()) - sched.addnode(MockNode()) - sched.addnode(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) + sched.add_node(MockNode()) node1, node2, node3 = sched.nodes col = ["xyz"] * 5 - sched.addnode_collection(node1, col) - sched.addnode_collection(node2, col) + sched.add_node_collection(node1, col) + sched.add_node_collection(node2, col) sched.schedule() # assert not sched.tests_finished() sent1 = node1.sent @@ -187,9 +187,9 @@ class TestLoadScheduling: node = MockNode() config = testdir.parseconfig("--tx=popen") sched = LoadScheduling(config) - sched.addnode(node) + sched.add_node(node) collection = ["test_file.py::test_func"] - sched.addnode_collection(node, collection) + sched.add_node_collection(node, collection) assert sched.collection_is_completed sched.schedule() assert not sched.pending @@ -219,10 +219,10 @@ class TestLoadScheduling: node1 = MockNode() node2 = MockNode() sched = LoadScheduling(config) - sched.addnode(node1) - sched.addnode(node2) - sched.addnode_collection(node1, ["a.py::test_1"]) - sched.addnode_collection(node2, ["a.py::test_2"]) + sched.add_node(node1) + sched.add_node(node2) + sched.add_node_collection(node1, ["a.py::test_1"]) + sched.add_node_collection(node2, ["a.py::test_2"]) sched.schedule() assert len(collect_hook.reports) == 1 rep = collect_hook.reports[0] diff --git a/xdist/dsession.py b/xdist/dsession.py index c22c865..4b5eb4f 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -42,10 +42,10 @@ class EachScheduling: """A list of all nodes in the scheduler.""" return list(self.node2pending.keys()) - def hasnodes(self): + def has_nodes(self): return bool(self.node2pending) - def haspending(self): + def has_pending(self): """Return True if there are pending test items This indicates that collection has finished and nodes are @@ -57,7 +57,7 @@ class EachScheduling: return True return False - def addnode(self, node): + def add_node(self, node): assert node not in self.node2pending self.node2pending[node] = [] @@ -71,7 +71,7 @@ class EachScheduling: return False return True - def addnode_collection(self, node, collection): + def add_node_collection(self, node, collection): """Add the collected test items from a node Collection is complete once all nodes have submitted their @@ -107,7 +107,7 @@ class EachScheduling: self.node2pending[node].remove(item_index) def remove_node(self, node): - # KeyError if we didn't get an addnode() yet + # KeyError if we didn't get an add_node() yet pending = self.node2pending.pop(node) if not pending: return @@ -121,7 +121,7 @@ class EachScheduling: If the node's pending list is empty it is a new node which needs to run all the tests. If the pending list is already - populated (by ``.addnode_collection()``) then it replaces a + populated (by ``.add_node_collection()``) then it replaces a dead node and we only need to run those tests. """ assert self.collection_is_completed @@ -209,7 +209,7 @@ class LoadScheduling: """ return len(self.node2collection) >= self.numnodes - def haspending(self): + def has_pending(self): """Return True if there are pending test items This indicates that collection has finished and nodes are @@ -223,11 +223,11 @@ class LoadScheduling: return True return False - def hasnodes(self): + def has_nodes(self): """Return True if nodes exist in the scheduler.""" return bool(self.node2pending) - def addnode(self, node): + def add_node(self, node): """Add a new node to the scheduler. From now on the node will be allocated chunks of tests to @@ -250,7 +250,7 @@ class LoadScheduling: return False return True - def addnode_collection(self, node, collection): + def add_node_collection(self, node, collection): """Add the collected test items from a node The collection is stored in the ``.node2collection`` map. @@ -579,7 +579,7 @@ class DSession: if self.shuttingdown: node.shutdown() else: - self.sched.addnode(node) + self.sched.add_node(node) def slave_slavefinished(self, node): """Emitted when node executes its pytest_sessionfinish hook. @@ -641,11 +641,11 @@ class DSession: # tell session which items were effectively collected otherwise # the master node will finish the session with EXIT_NOTESTSCOLLECTED self._session.testscollected = len(ids) - self.sched.addnode_collection(node, ids) + self.sched.add_node_collection(node, ids) if self.terminal: self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids))) if self.sched.collection_is_completed: - if self.terminal and not self.sched.haspending(): + if self.terminal and not self.sched.has_pending(): self.trdist.ensure_show_status() self.terminal.write_line("") self.terminal.write_line("scheduling tests via %s" % ( From c809406ef6e8e7560f3b09ae7e84c508d7a5c9d1 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Wed, 15 Feb 2017 11:43:56 +0500 Subject: [PATCH 13/14] Refactor scheduller interface, make more properties instead of methods --- testing/test_dsession.py | 22 ++++++++-------- xdist/dsession.py | 55 +++++++++++++++++++--------------------- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 28f1586..a5ec76f 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -65,13 +65,13 @@ class TestEachScheduling: assert sched.node2collection[node1] == collection assert sched.node2collection[node2] == collection sched.schedule() - assert sched.tests_finished() + assert sched.tests_finished assert node1.sent == ['ALL'] assert node2.sent == ['ALL'] sched.remove_item(node1, 0) - assert sched.tests_finished() + assert sched.tests_finished sched.remove_item(node2, 0) - assert sched.tests_finished() + assert sched.tests_finished def test_schedule_remove_node(self, testdir): node1 = MockNode() @@ -84,11 +84,11 @@ class TestEachScheduling: assert sched.collection_is_completed assert sched.node2collection[node1] == collection sched.schedule() - assert sched.tests_finished() + assert sched.tests_finished crashitem = sched.remove_node(node1) assert crashitem - assert sched.tests_finished() - assert not sched.has_nodes() + assert sched.tests_finished + assert not sched.nodes class TestLoadScheduling: @@ -108,13 +108,13 @@ class TestLoadScheduling: assert sched.node2collection[node2] == collection sched.schedule() assert not sched.pending - assert sched.tests_finished() + assert sched.tests_finished assert len(node1.sent) == 1 assert len(node2.sent) == 1 assert node1.sent == [0] assert node2.sent == [1] sched.remove_item(node1, node1.sent[0]) - assert sched.tests_finished() + assert sched.tests_finished def test_schedule_batch_size(self, testdir): config = testdir.parseconfig("--tx=2*popen") @@ -126,7 +126,7 @@ class TestLoadScheduling: sched.add_node_collection(node1, col) sched.add_node_collection(node2, col) sched.schedule() - # assert not sched.tests_finished() + # assert not sched.tests_finished sent1 = node1.sent sent2 = node2.sent assert sent1 == [0, 2] @@ -154,7 +154,7 @@ class TestLoadScheduling: sched.add_node_collection(node1, col) sched.add_node_collection(node2, col) sched.schedule() - # assert not sched.tests_finished() + # assert not sched.tests_finished sent1 = node1.sent sent2 = node2.sent sent3 = node3.sent @@ -174,7 +174,7 @@ class TestLoadScheduling: sched.add_node_collection(node1, col) sched.add_node_collection(node2, col) sched.schedule() - # assert not sched.tests_finished() + # assert not sched.tests_finished sent1 = node1.sent sent2 = node2.sent sent3 = node3.sent diff --git a/xdist/dsession.py b/xdist/dsession.py index 4b5eb4f..6c3dd70 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -42,9 +42,18 @@ class EachScheduling: """A list of all nodes in the scheduler.""" return list(self.node2pending.keys()) - def has_nodes(self): - return bool(self.node2pending) + @property + def tests_finished(self): + if not self.collection_is_completed: + return False + if self._removed2pending: + return False + for pending in self.node2pending.values(): + if len(pending) >= 2: + return False + return True + @property def has_pending(self): """Return True if there are pending test items @@ -61,16 +70,6 @@ class EachScheduling: assert node not in self.node2pending self.node2pending[node] = [] - def tests_finished(self): - if not self.collection_is_completed: - return False - if self._removed2pending: - return False - for pending in self.node2pending.values(): - if len(pending) >= 2: - return False - return True - def add_node_collection(self, node, collection): """Add the collected test items from a node @@ -209,6 +208,19 @@ class LoadScheduling: """ return len(self.node2collection) >= self.numnodes + @property + def tests_finished(self): + """Return True if all tests have been executed by the nodes.""" + if not self.collection_is_completed: + return False + if self.pending: + return False + for pending in self.node2pending.values(): + if len(pending) >= 2: + return False + return True + + @property def has_pending(self): """Return True if there are pending test items @@ -223,10 +235,6 @@ class LoadScheduling: return True return False - def has_nodes(self): - """Return True if nodes exist in the scheduler.""" - return bool(self.node2pending) - def add_node(self, node): """Add a new node to the scheduler. @@ -239,17 +247,6 @@ class LoadScheduling: assert node not in self.node2pending self.node2pending[node] = [] - def tests_finished(self): - """Return True if all tests have been executed by the nodes.""" - if not self.collection_is_completed: - return False - if self.pending: - return False - for pending in self.node2pending.values(): - if len(pending) >= 2: - return False - return True - def add_node_collection(self, node, collection): """Add the collected test items from a node @@ -559,7 +556,7 @@ class DSession: call = getattr(self, method) self.log("calling method", method, kwargs) call(**kwargs) - if self.sched.tests_finished(): + if self.sched.tests_finished: self.triggershutdown() # @@ -645,7 +642,7 @@ class DSession: if self.terminal: self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids))) if self.sched.collection_is_completed: - if self.terminal and not self.sched.has_pending(): + if self.terminal and not self.sched.has_pending: self.trdist.ensure_show_status() self.terminal.write_line("") self.terminal.write_line("scheduling tests via %s" % ( From 4ed8161ef808f04d1b16ab65066fe556acbbff1a Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Wed, 15 Feb 2017 23:54:23 +0500 Subject: [PATCH 14/14] Renamed scheduller remove_item method to mark_test_complete --- testing/test_dsession.py | 10 +++++----- xdist/dsession.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index a5ec76f..5de6c50 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -68,9 +68,9 @@ class TestEachScheduling: assert sched.tests_finished assert node1.sent == ['ALL'] assert node2.sent == ['ALL'] - sched.remove_item(node1, 0) + sched.mark_test_complete(node1, 0) assert sched.tests_finished - sched.remove_item(node2, 0) + sched.mark_test_complete(node2, 0) assert sched.tests_finished def test_schedule_remove_node(self, testdir): @@ -113,7 +113,7 @@ class TestLoadScheduling: assert len(node2.sent) == 1 assert node1.sent == [0] assert node2.sent == [1] - sched.remove_item(node1, node1.sent[0]) + sched.mark_test_complete(node1, node1.sent[0]) assert sched.tests_finished def test_schedule_batch_size(self, testdir): @@ -135,11 +135,11 @@ class TestLoadScheduling: assert sched.node2pending[node1] == sent1 assert sched.node2pending[node2] == sent2 assert len(sched.pending) == 2 - sched.remove_item(node1, 0) + sched.mark_test_complete(node1, 0) assert node1.sent == [0, 2, 4] assert sched.pending == [5] assert node2.sent == [1, 3] - sched.remove_item(node1, 2) + sched.mark_test_complete(node1, 2) assert node1.sent == [0, 2, 4, 5] assert not sched.pending diff --git a/xdist/dsession.py b/xdist/dsession.py index 6c3dd70..9ddbc2d 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -102,7 +102,7 @@ class EachScheduling: self.node2pending[node] = pending break - def remove_item(self, node, item_index, duration=0): + def mark_test_complete(self, node, item_index, duration=0): self.node2pending[node].remove(item_index) def remove_node(self, node): @@ -143,7 +143,7 @@ class LoadScheduling: when all collections are received it is verified they are identical collections. Then the collection gets divided up in chunks and chunks get submitted to nodes. Whenever a node finishes - an item, it calls ``.remove_item()`` which will trigger the + an item, it calls ``.mark_test_complete()`` which will trigger the scheduler to assign more tests if the number of pending tests for the node falls below a low-watermark. @@ -269,7 +269,7 @@ class LoadScheduling: return self.node2collection[node] = list(collection) - def remove_item(self, node, item_index, duration=0): + def mark_test_complete(self, node, item_index, duration=0): """Mark test item as completed by node The duration it took to execute the item is used as a hint to @@ -661,7 +661,7 @@ class DSession: the item from the pending list in the scheduler. """ if rep.when == "call" or (rep.when == "setup" and not rep.passed): - self.sched.remove_item(node, rep.item_index, rep.duration) + self.sched.mark_test_complete(node, rep.item_index, rep.duration) # self.report_line("testreport %s: %s" %(rep.id, rep.status)) rep.node = node self.config.hook.pytest_runtest_logreport(report=rep)