IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up scheduller interfaces

Browse Source
This commit is contained in:
Michael Elovskikh
2017-02-15 00:21:26 +05:00
parent 6200123fac
commit a99e6e7ca8
2 changed files with 44 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
testing/test_dsession.py
View File

@@ -54,13 +54,13 @@ class TestEachScheduling:
node2 = MockNode() node2 = MockNode()
config = testdir.parseconfig("--tx=2*popen") config = testdir.parseconfig("--tx=2*popen")
sched = EachScheduling(config) sched = EachScheduling(config)
sched.addnode(node1) sched.add_node(node1)
sched.addnode(node2) sched.add_node(node2)
collection = ["a.py::test_1", ] collection = ["a.py::test_1", ]
assert not sched.collection_is_completed assert not sched.collection_is_completed
sched.addnode_collection(node1, collection) sched.add_node_collection(node1, collection)
assert not sched.collection_is_completed assert not sched.collection_is_completed
sched.addnode_collection(node2, collection) sched.add_node_collection(node2, collection)
assert sched.collection_is_completed assert sched.collection_is_completed
assert sched.node2collection[node1] == collection assert sched.node2collection[node1] == collection
assert sched.node2collection[node2] == collection assert sched.node2collection[node2] == collection
@@ -77,10 +77,10 @@ class TestEachScheduling:
node1 = MockNode() node1 = MockNode()
config = testdir.parseconfig("--tx=popen") config = testdir.parseconfig("--tx=popen")
sched = EachScheduling(config) sched = EachScheduling(config)
sched.addnode(node1) sched.add_node(node1)
collection = ["a.py::test_1", ] collection = ["a.py::test_1", ]
assert not sched.collection_is_completed assert not sched.collection_is_completed
sched.addnode_collection(node1, collection) sched.add_node_collection(node1, collection)
assert sched.collection_is_completed assert sched.collection_is_completed
assert sched.node2collection[node1] == collection assert sched.node2collection[node1] == collection
sched.schedule() sched.schedule()
@@ -88,21 +88,21 @@ class TestEachScheduling:
crashitem = sched.remove_node(node1) crashitem = sched.remove_node(node1)
assert crashitem assert crashitem
assert sched.tests_finished() assert sched.tests_finished()
assert not sched.hasnodes() assert not sched.has_nodes()
class TestLoadScheduling: class TestLoadScheduling:
def test_schedule_load_simple(self, testdir): def test_schedule_load_simple(self, testdir):
config = testdir.parseconfig("--tx=2*popen") config = testdir.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
node1, node2 = sched.nodes node1, node2 = sched.nodes
collection = ["a.py::test_1", "a.py::test_2"] collection = ["a.py::test_1", "a.py::test_2"]
assert not sched.collection_is_completed assert not sched.collection_is_completed
sched.addnode_collection(node1, collection) sched.add_node_collection(node1, collection)
assert not sched.collection_is_completed assert not sched.collection_is_completed
sched.addnode_collection(node2, collection) sched.add_node_collection(node2, collection)
assert sched.collection_is_completed assert sched.collection_is_completed
assert sched.node2collection[node1] == collection assert sched.node2collection[node1] == collection
assert sched.node2collection[node2] == collection assert sched.node2collection[node2] == collection
@@ -119,12 +119,12 @@ class TestLoadScheduling:
def test_schedule_batch_size(self, testdir): def test_schedule_batch_size(self, testdir):
config = testdir.parseconfig("--tx=2*popen") config = testdir.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
node1, node2 = sched.nodes node1, node2 = sched.nodes
col = ["xyz"] * (6) col = ["xyz"] * (6)
sched.addnode_collection(node1, col) sched.add_node_collection(node1, col)
sched.addnode_collection(node2, col) sched.add_node_collection(node2, col)
sched.schedule() sched.schedule()
# assert not sched.tests_finished() # assert not sched.tests_finished()
sent1 = node1.sent sent1 = node1.sent
@@ -146,13 +146,13 @@ class TestLoadScheduling:
def test_schedule_fewer_tests_than_nodes(self, testdir): def test_schedule_fewer_tests_than_nodes(self, testdir):
config = testdir.parseconfig("--tx=2*popen") config = testdir.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
node1, node2, node3 = sched.nodes node1, node2, node3 = sched.nodes
col = ["xyz"] * 2 col = ["xyz"] * 2
sched.addnode_collection(node1, col) sched.add_node_collection(node1, col)
sched.addnode_collection(node2, col) sched.add_node_collection(node2, col)
sched.schedule() sched.schedule()
# assert not sched.tests_finished() # assert not sched.tests_finished()
sent1 = node1.sent sent1 = node1.sent
@@ -166,13 +166,13 @@ class TestLoadScheduling:
def test_schedule_fewer_than_two_tests_per_node(self, testdir): def test_schedule_fewer_than_two_tests_per_node(self, testdir):
config = testdir.parseconfig("--tx=2*popen") config = testdir.parseconfig("--tx=2*popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
sched.addnode(MockNode()) sched.add_node(MockNode())
node1, node2, node3 = sched.nodes node1, node2, node3 = sched.nodes
col = ["xyz"] * 5 col = ["xyz"] * 5
sched.addnode_collection(node1, col) sched.add_node_collection(node1, col)
sched.addnode_collection(node2, col) sched.add_node_collection(node2, col)
sched.schedule() sched.schedule()
# assert not sched.tests_finished() # assert not sched.tests_finished()
sent1 = node1.sent sent1 = node1.sent
@@ -187,9 +187,9 @@ class TestLoadScheduling:
node = MockNode() node = MockNode()
config = testdir.parseconfig("--tx=popen") config = testdir.parseconfig("--tx=popen")
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(node) sched.add_node(node)
collection = ["test_file.py::test_func"] collection = ["test_file.py::test_func"]
sched.addnode_collection(node, collection) sched.add_node_collection(node, collection)
assert sched.collection_is_completed assert sched.collection_is_completed
sched.schedule() sched.schedule()
assert not sched.pending assert not sched.pending
@@ -219,10 +219,10 @@ class TestLoadScheduling:
node1 = MockNode() node1 = MockNode()
node2 = MockNode() node2 = MockNode()
sched = LoadScheduling(config) sched = LoadScheduling(config)
sched.addnode(node1) sched.add_node(node1)
sched.addnode(node2) sched.add_node(node2)
sched.addnode_collection(node1, ["a.py::test_1"]) sched.add_node_collection(node1, ["a.py::test_1"])
sched.addnode_collection(node2, ["a.py::test_2"]) sched.add_node_collection(node2, ["a.py::test_2"])
sched.schedule() sched.schedule()
assert len(collect_hook.reports) == 1 assert len(collect_hook.reports) == 1
rep = collect_hook.reports[0] rep = collect_hook.reports[0]

26
xdist/dsession.py
View File

@@ -42,10 +42,10 @@ class EachScheduling:
"""A list of all nodes in the scheduler.""" """A list of all nodes in the scheduler."""
return list(self.node2pending.keys()) return list(self.node2pending.keys())
def hasnodes(self): def has_nodes(self):
return bool(self.node2pending) return bool(self.node2pending)
def haspending(self): def has_pending(self):
"""Return True if there are pending test items """Return True if there are pending test items
This indicates that collection has finished and nodes are This indicates that collection has finished and nodes are
@@ -57,7 +57,7 @@ class EachScheduling:
return True return True
return False return False
def addnode(self, node): def add_node(self, node):
assert node not in self.node2pending assert node not in self.node2pending
self.node2pending[node] = [] self.node2pending[node] = []
@@ -71,7 +71,7 @@ class EachScheduling:
return False return False
return True return True
def addnode_collection(self, node, collection): def add_node_collection(self, node, collection):
"""Add the collected test items from a node """Add the collected test items from a node
Collection is complete once all nodes have submitted their Collection is complete once all nodes have submitted their
@@ -107,7 +107,7 @@ class EachScheduling:
self.node2pending[node].remove(item_index) self.node2pending[node].remove(item_index)
def remove_node(self, node): def remove_node(self, node):
# KeyError if we didn't get an addnode() yet # KeyError if we didn't get an add_node() yet
pending = self.node2pending.pop(node) pending = self.node2pending.pop(node)
if not pending: if not pending:
return return
@@ -121,7 +121,7 @@ class EachScheduling:
If the node's pending list is empty it is a new node which If the node's pending list is empty it is a new node which
needs to run all the tests. If the pending list is already needs to run all the tests. If the pending list is already
populated (by ``.addnode_collection()``) then it replaces a populated (by ``.add_node_collection()``) then it replaces a
dead node and we only need to run those tests. dead node and we only need to run those tests.
""" """
assert self.collection_is_completed assert self.collection_is_completed
@@ -209,7 +209,7 @@ class LoadScheduling:
""" """
return len(self.node2collection) >= self.numnodes return len(self.node2collection) >= self.numnodes
def haspending(self): def has_pending(self):
"""Return True if there are pending test items """Return True if there are pending test items
This indicates that collection has finished and nodes are This indicates that collection has finished and nodes are
@@ -223,11 +223,11 @@ class LoadScheduling:
return True return True
return False return False
def hasnodes(self): def has_nodes(self):
"""Return True if nodes exist in the scheduler.""" """Return True if nodes exist in the scheduler."""
return bool(self.node2pending) return bool(self.node2pending)
def addnode(self, node): def add_node(self, node):
"""Add a new node to the scheduler. """Add a new node to the scheduler.
From now on the node will be allocated chunks of tests to From now on the node will be allocated chunks of tests to
@@ -250,7 +250,7 @@ class LoadScheduling:
return False return False
return True return True
def addnode_collection(self, node, collection): def add_node_collection(self, node, collection):
"""Add the collected test items from a node """Add the collected test items from a node
The collection is stored in the ``.node2collection`` map. The collection is stored in the ``.node2collection`` map.
@@ -579,7 +579,7 @@ class DSession:
if self.shuttingdown: if self.shuttingdown:
node.shutdown() node.shutdown()
else: else:
self.sched.addnode(node) self.sched.add_node(node)
def slave_slavefinished(self, node): def slave_slavefinished(self, node):
"""Emitted when node executes its pytest_sessionfinish hook. """Emitted when node executes its pytest_sessionfinish hook.
@@ -641,11 +641,11 @@ class DSession:
# tell session which items were effectively collected otherwise # tell session which items were effectively collected otherwise
# the master node will finish the session with EXIT_NOTESTSCOLLECTED # the master node will finish the session with EXIT_NOTESTSCOLLECTED
self._session.testscollected = len(ids) self._session.testscollected = len(ids)
self.sched.addnode_collection(node, ids) self.sched.add_node_collection(node, ids)
if self.terminal: if self.terminal:
self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids))) self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids)))
if self.sched.collection_is_completed: if self.sched.collection_is_completed:
if self.terminal and not self.sched.haspending(): if self.terminal and not self.sched.has_pending():
self.trdist.ensure_show_status() self.trdist.ensure_show_status()
self.terminal.write_line("") self.terminal.write_line("")
self.terminal.write_line("scheduling tests via %s" % ( self.terminal.write_line("scheduling tests via %s" % (