From 6200123fac00f2a33db816a9b5cd5d497078e395 Mon Sep 17 00:00:00 2001 From: Michael Elovskikh Date: Tue, 14 Feb 2017 20:19:41 +0500 Subject: [PATCH] Made EachScheduling.numnodes countable from config --- testing/test_dsession.py | 32 ++++++++++++++++---------------- xdist/dsession.py | 19 +++++++++---------- xdist/newhooks.py | 2 +- xdist/slavemanage.py | 30 +++++++++++++++++------------- 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index dbc96a1..81b35e6 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -52,8 +52,8 @@ class TestEachScheduling: def test_schedule_load_simple(self, testdir): node1 = MockNode() node2 = MockNode() - config = testdir.parseconfig() - sched = EachScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = EachScheduling(config) sched.addnode(node1) sched.addnode(node2) collection = ["a.py::test_1", ] @@ -75,8 +75,8 @@ class TestEachScheduling: def test_schedule_remove_node(self, testdir): node1 = MockNode() - config = testdir.parseconfig() - sched = EachScheduling(1, config) + config = testdir.parseconfig("--tx=popen") + sched = EachScheduling(config) sched.addnode(node1) collection = ["a.py::test_1", ] assert not sched.collection_is_completed @@ -93,8 +93,8 @@ class TestEachScheduling: class TestLoadScheduling: def test_schedule_load_simple(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -117,8 +117,8 @@ class TestLoadScheduling: assert sched.tests_finished() def test_schedule_batch_size(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) node1, node2 = sched.nodes @@ -144,8 +144,8 @@ class TestLoadScheduling: assert not sched.pending def test_schedule_fewer_tests_than_nodes(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -164,8 +164,8 @@ class TestLoadScheduling: assert not sched.pending def test_schedule_fewer_than_two_tests_per_node(self, testdir): - config = testdir.parseconfig() - sched = LoadScheduling(2, config) + config = testdir.parseconfig("--tx=2*popen") + sched = LoadScheduling(config) sched.addnode(MockNode()) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -185,8 +185,8 @@ class TestLoadScheduling: def test_add_remove_node(self, testdir): node = MockNode() - config = testdir.parseconfig() - sched = LoadScheduling(1, config) + config = testdir.parseconfig("--tx=popen") + sched = LoadScheduling(config) sched.addnode(node) collection = ["test_file.py::test_func"] sched.addnode_collection(node, collection) @@ -214,11 +214,11 @@ class TestLoadScheduling: self.reports.append(report) collect_hook = CollectHook() - config = testdir.parseconfig() + config = testdir.parseconfig("--tx=2*popen") config.pluginmanager.register(collect_hook, "collect_hook") node1 = MockNode() node2 = MockNode() - sched = LoadScheduling(2, config) + sched = LoadScheduling(config) sched.addnode(node1) sched.addnode(node2) sched.addnode_collection(node1, ["a.py::test_1"]) diff --git a/xdist/dsession.py b/xdist/dsession.py index 3032e11..c22c865 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -4,7 +4,7 @@ from _pytest.runner import CollectReport import pytest import py -from xdist.slavemanage import NodeManager +from xdist.slavemanage import NodeManager, parse_spec_config queue = py.builtin._tryimport('queue', 'Queue') @@ -24,8 +24,9 @@ class EachScheduling: assigned the remaining items from the removed node. """ - def __init__(self, numnodes, config, log=None): - self.numnodes = numnodes + def __init__(self, config, log=None): + self.config = config + self.numnodes = len(parse_spec_config(config)) self.node2collection = {} self.node2pending = {} self._started = [] @@ -181,8 +182,8 @@ class LoadScheduling: :config: Config object, used for handling hooks. """ - def __init__(self, numnodes, config, log=None): - self.numnodes = numnodes + def __init__(self, config, log=None): + self.numnodes = len(parse_spec_config(config)) self.node2collection = {} self.node2pending = {} self.pending = [] @@ -522,17 +523,15 @@ class DSession: return True @pytest.mark.trylast - def pytest_xdist_make_scheduler(self, numnodes, config, log): + def pytest_xdist_make_scheduler(self, config, log): dist = config.getvalue("dist") if dist == "load": - return LoadScheduling(numnodes, config, log) + return LoadScheduling(config, log) elif dist == "each": - return EachScheduling(numnodes, config, log) + return EachScheduling(config, log) def pytest_runtestloop(self): - numnodes = len(self.nodemanager.specs) self.sched = self.config.hook.pytest_xdist_make_scheduler( - numnodes=numnodes, config=self.config, log=self.log ) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index e98b0e2..364fb36 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -48,5 +48,5 @@ def pytest_xdist_node_collection_finished(node, ids): @pytest.mark.firstresult -def pytest_xdist_make_scheduler(numnodes, config, log): +def pytest_xdist_make_scheduler(config, log): """ return a node scheduler implementation """ diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index ffac07e..bb44828 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -10,6 +10,22 @@ import xdist.remote from _pytest import runner # XXX load dynamically +def parse_spec_config(config): + xspeclist = [] + for xspec in config.getvalue("tx"): + i = xspec.find("*") + try: + num = int(xspec[:i]) + except ValueError: + xspeclist.append(xspec) + else: + xspeclist.extend([xspec[i + 1:]] * num) + if not xspeclist: + raise pytest.UsageError( + "MISSING test execution (tx) nodes: please specify --tx") + return xspeclist + + class NodeManager(object): EXIT_TIMEOUT = 10 DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~'] @@ -62,19 +78,7 @@ class NodeManager(object): self.group.terminate(self.EXIT_TIMEOUT) def _getxspecs(self): - xspeclist = [] - for xspec in self.config.getvalue("tx"): - i = xspec.find("*") - try: - num = int(xspec[:i]) - except ValueError: - xspeclist.append(xspec) - else: - xspeclist.extend([xspec[i+1:]] * num) - if not xspeclist: - raise pytest.UsageError( - "MISSING test execution (tx) nodes: please specify --tx") - return [execnet.XSpec(x) for x in xspeclist] + return [execnet.XSpec(x) for x in parse_spec_config(self.config)] def _getrsyncdirs(self): for spec in self.specs: