From 09d79ace356ebf14d221cecd3fd8002d075a2034 Mon Sep 17 00:00:00 2001 From: Steven Hazel Date: Wed, 18 Nov 2015 13:06:04 -0800 Subject: [PATCH 01/40] Do a better job parallelizing the inital batch of tests when the number of nodes is more than half the number of tests. This makes it possible, for example, to run all tests in parallel, where previous the maximum parallelization was half of all tests. --- testing/test_dsession.py | 68 ++++++++++++++++++++++++++++++++-------- xdist/dsession.py | 26 ++++++++++----- xdist/slavemanage.py | 6 ++++ 3 files changed, 80 insertions(+), 20 deletions(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index 234f94f..be638e4 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -27,6 +27,7 @@ class MockNode: def __init__(self): self.sent = [] self.gateway = MockGateway() + self._shutdown = False def send_runtest_some(self, indices): self.sent.extend(indices) @@ -37,6 +38,10 @@ class MockNode: def shutdown(self): self._shutdown = True + @property + def shutting_down(self): + return self._shutdown + def dumpqueue(queue): while queue.qsize(): @@ -100,16 +105,15 @@ class TestLoadScheduling: assert sched.node2collection[node2] == collection sched.init_distribute() assert not sched.pending - assert not sched.tests_finished() - assert len(node1.sent) == 2 - assert len(node2.sent) == 0 - assert node1.sent == [0, 1] + assert sched.tests_finished() + assert len(node1.sent) == 1 + assert len(node2.sent) == 1 + assert node1.sent == [0] + assert node2.sent == [1] sched.remove_item(node1, node1.sent[0]) assert sched.tests_finished() - sched.remove_item(node1, node1.sent[1]) - assert sched.tests_finished() - def test_init_distribute_chunksize(self): + def test_init_distribute_batch_size(self): sched = LoadScheduling(2) sched.addnode(MockNode()) sched.addnode(MockNode()) @@ -121,18 +125,56 @@ class TestLoadScheduling: # assert not sched.tests_finished() sent1 = node1.sent sent2 = node2.sent - assert sent1 == [0, 1] - assert sent2 == [2, 3] + assert sent1 == [0, 2] + assert sent2 == [1, 3] assert sched.pending == [4, 5] assert sched.node2pending[node1] == sent1 assert sched.node2pending[node2] == sent2 assert len(sched.pending) == 2 sched.remove_item(node1, 0) - assert node1.sent == [0, 1, 4] + assert node1.sent == [0, 2, 4] assert sched.pending == [5] - assert node2.sent == [2, 3] - sched.remove_item(node1, 1) - assert node1.sent == [0, 1, 4, 5] + assert node2.sent == [1, 3] + sched.remove_item(node1, 2) + assert node1.sent == [0, 2, 4, 5] + assert not sched.pending + + def test_init_distribute_fewer_tests_than_nodes(self): + sched = LoadScheduling(2) + sched.addnode(MockNode()) + sched.addnode(MockNode()) + sched.addnode(MockNode()) + node1, node2, node3 = sched.nodes + col = ["xyz"] * 2 + sched.addnode_collection(node1, col) + sched.addnode_collection(node2, col) + sched.init_distribute() + # assert not sched.tests_finished() + sent1 = node1.sent + sent2 = node2.sent + sent3 = node3.sent + assert sent1 == [0] + assert sent2 == [1] + assert sent3 == [] + assert not sched.pending + + def test_init_distribute_fewer_than_two_tests_per_node(self): + sched = LoadScheduling(2) + sched.addnode(MockNode()) + sched.addnode(MockNode()) + sched.addnode(MockNode()) + node1, node2, node3 = sched.nodes + col = ["xyz"] * 5 + sched.addnode_collection(node1, col) + sched.addnode_collection(node2, col) + sched.init_distribute() + # assert not sched.tests_finished() + sent1 = node1.sent + sent2 = node2.sent + sent3 = node3.sent + assert sent1 == [0, 3] + assert sent2 == [1, 4] + assert sent3 == [2] assert not sched.pending def test_add_remove_node(self): diff --git a/xdist/dsession.py b/xdist/dsession.py index a0438de..85a16a3 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -1,4 +1,5 @@ import difflib +import itertools from _pytest.runner import CollectReport import pytest @@ -289,6 +290,9 @@ class LoadScheduling: ``duration`` of the last test is optionally used as a heuristic to influence how many tests the node is assigned. """ + if node.shutting_down: + return + if self.pending: # how many nodes do we have? num_nodes = len(self.node2pending) @@ -363,13 +367,21 @@ class LoadScheduling: if not self.collection: return - # how many items per node do we have about? - items_per_node = len(self.collection) // len(self.node2pending) - # take a fraction of tests for initial distribution - node_chunksize = max(items_per_node // 4, 2) - # and initialize each node with a chunk of tests - for node in self.nodes: - self._send_tests(node, node_chunksize) + # Send a batch of tests to run. If we don't have at least two + # tests per node, we have to send them all so that we can send + # shutdown signals and get all nodes working. + initial_batch = max(len(self.pending) // 4, + 2 * len(self.nodes)) + + # distribute tests round-robin up to the batch size (or until we run out) + nodes = itertools.cycle(self.nodes) + for i in xrange(initial_batch): + self._send_tests(nodes.next(), 1) + + if not self.pending: + # initial distribution sent all tests, start node shutdown + for node in self.nodes: + node.shutdown() def _send_tests(self, node, num): tests_per_node = self.pending[:num] diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 076bc7f..b50dab5 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -207,6 +207,7 @@ class SlaveController(object): self.config = config self.slaveinput = {'slaveid': gateway.id} self._down = False + self._shutdown_sent = False self.log = py.log.Producer("slavectl-%s" % gateway.id) if not self.config.option.debug: py.log.setconsumer(self.log._keywords, None) @@ -214,6 +215,10 @@ class SlaveController(object): def __repr__(self): return "<%s %s>" % (self.__class__.__name__, self.gateway.id,) + @property + def shutting_down(self): + return self._down or self._shutdown_sent + def setup(self): self.log("setting up slave session") spec = self.gateway.spec @@ -256,6 +261,7 @@ class SlaveController(object): self.sendcommand("shutdown") except IOError: pass + self._shutdown_sent = True def sendcommand(self, name, **kwargs): """ send a named parametrized command to the other side. """ From 3140873f0a2610bd8490407ba39df95de742463a Mon Sep 17 00:00:00 2001 From: Steven Hazel Date: Wed, 18 Nov 2015 13:34:39 -0800 Subject: [PATCH 02/40] bugfix: iterate in a python 3 compatible way --- xdist/dsession.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 85a16a3..679baf8 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -375,7 +375,7 @@ class LoadScheduling: # distribute tests round-robin up to the batch size (or until we run out) nodes = itertools.cycle(self.nodes) - for i in xrange(initial_batch): + for i in range(initial_batch): self._send_tests(nodes.next(), 1) if not self.pending: From 412febd89838f454c9458b5873ca9f1535834cf0 Mon Sep 17 00:00:00 2001 From: Steven Hazel Date: Wed, 18 Nov 2015 13:39:53 -0800 Subject: [PATCH 03/40] pep8 and python 3 compatibility fixes --- xdist/dsession.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 679baf8..6f88afc 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -373,10 +373,11 @@ class LoadScheduling: initial_batch = max(len(self.pending) // 4, 2 * len(self.nodes)) - # distribute tests round-robin up to the batch size (or until we run out) + # distribute tests round-robin up to the batch size + # (or until we run out) nodes = itertools.cycle(self.nodes) for i in range(initial_batch): - self._send_tests(nodes.next(), 1) + self._send_tests(next(nodes), 1) if not self.pending: # initial distribution sent all tests, start node shutdown From 4c22653fd4044eb52c6173f4afbcb4f11b1ae34b Mon Sep 17 00:00:00 2001 From: Nicholas Chammas Date: Tue, 1 Dec 2015 14:45:03 -0500 Subject: [PATCH 04/40] Typo fixes to docstrings --- xdist/dsession.py | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/xdist/dsession.py b/xdist/dsession.py index 6f88afc..25b7405 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -15,12 +15,12 @@ class EachScheduling: If a node gets added after the test run is started then it is assumed to replace a node which got removed before it finished - it's collection. In this case it will only be used if a a node + its collection. In this case it will only be used if a node with the same spec got removed earlier. Any nodes added after the run is started will only get items - assigned if a node with matching spec was removed before it - finished all it's pending items. The new node will then be + assigned if a node with a matching spec was removed before it + finished all its pending items. The new node will then be assigned the remaining items from the removed node. """ @@ -48,8 +48,8 @@ class EachScheduling: """Return True if there are pending test items This indicates that collection has finished and nodes are - still processing test items, so can be thought of as "the - scheduler is active". + still processing test items, so this can be thought of as + "the scheduler is active". """ for pending in self.node2pending.values(): if pending: @@ -74,10 +74,10 @@ class EachScheduling: """Add the collected test items from a node Collection is complete once all nodes have submitted their - collection. In this case it's peding list is set to an empty + collection. In this case its pending list is set to an empty list. When the collection is already completed this submission is from a node which was restarted to replace a - dead node. In this case we already assing the pending items + dead node. In this case we already assign the pending items here. In either case ``.init_distribute()`` will instruct the node to start running the required tests. """ @@ -121,7 +121,7 @@ class EachScheduling: If the node's pending list is empty it is a new node which needs to run all the tests. If the pending list is already populated (by ``.addnode_collection()``) then it replaces a - died node and we only need to run those tests. + dead node and we only need to run those tests. """ assert self.collection_is_completed for node, pending in self.node2pending.items(): @@ -139,16 +139,16 @@ class LoadScheduling: """Implement load scheduling accross nodes. This distributes the tests collected across all nodes so each test - is run just once. All nodes collect and submit the test suit and + is run just once. All nodes collect and submit the test suite and when all collections are received it is verified they are - identical collections. Then the collection gets devided up in - chunks and chunks get submitted to nodes. Whenver a node finishes - an item they call ``.remove_item()`` which will trigger the + identical collections. Then the collection gets divided up in + chunks and chunks get submitted to nodes. Whenever a node finishes + an item, it calls ``.remove_item()`` which will trigger the scheduler to assign more tests if the number of pending tests for the node falls below a low-watermark. - When created ``numnodes`` defines how many nodes are expected to - submit a collection, this is used to know when all nodes have + When created, ``numnodes`` defines how many nodes are expected to + submit a collection. This is used to know when all nodes have finished collection or how large the chunks need to be created. Attributes: @@ -156,7 +156,7 @@ class LoadScheduling: :numnodes: The expected number of nodes taking part. The actual number of nodes will vary during the scheduler's lifetime as nodes are added by the DSession as they are brought up and - removed either because of a died node or normal shutdown. This + removed either because of a dead node or normal shutdown. This number is primarily used to know when the initial collection is completed. @@ -212,8 +212,8 @@ class LoadScheduling: """Return True if there are pending test items This indicates that collection has finished and nodes are - still processing test items, so can be thought of as "the - scheduler is active". + still processing test items, so this can be thought of as + "the scheduler is active". """ if self.pending: return True @@ -227,13 +227,13 @@ class LoadScheduling: return bool(self.node2pending) def addnode(self, node): - """Add a new node in the scheduler. + """Add a new node to the scheduler. From now on the node will be allocated chunks of tests to execute. Called by the ``DSession.slave_slaveready`` hook when it - sucessfully bootstrapped a new node. + sucessfully bootstraps a new node. """ assert node not in self.node2pending self.node2pending[node] = [] @@ -312,7 +312,7 @@ class LoadScheduling: self.log("num items waiting for node:", len(self.pending)) def remove_node(self, node): - """Remove an node from the scheduler + """Remove a node from the scheduler This should be called either when the node crashed or at shutdown time. In the former case any pending items assigned @@ -490,7 +490,7 @@ class DSession: """Return True if the distributed session has finished This means all nodes have executed all test items. This is - used to by pytest_runtestloop to break out of it's loop. + used by pytest_runtestloop to break out of its loop. """ return bool(self.shuttingdown and not self._active_nodes) @@ -579,7 +579,7 @@ class DSession: Removes the node from the scheduler. - The node might not be the scheduler if it had not emitted + The node might not be in the scheduler if it had not emitted slaveready before shutdown was triggered. """ self.config.hook.pytest_testnodedown(node=node, error=None) @@ -623,7 +623,7 @@ class DSession: This adds the collection for this node to the scheduler. If the scheduler indicates collection is finished (i.e. all - initial nodes have submitted their collection), then tells the + initial nodes have submitted their collections), then tells the scheduler to schedule the collected items. When initiating scheduling the first time it logs which scheduler is in use. """ @@ -651,7 +651,7 @@ class DSession: def slave_testreport(self, node, rep): """Emitted when a node calls the pytest_runtest_logreport hook. - If the node indicates it is finished with a test item remove + If the node indicates it is finished with a test item, remove the item from the pending list in the scheduler. """ if rep.when == "call" or (rep.when == "setup" and not rep.passed): @@ -669,9 +669,9 @@ class DSession: def _clone_node(self, node): """Return new node based on an existing one. - This is normally for when a node died, this will copy the spec + This is normally for when a node dies, this will copy the spec of the existing node and create a new one with a new id. The - new node will have been setup so will start calling the + new node will have been setup so it will start calling the "slave_*" hooks and do work soon. """ spec = node.gateway.spec From 7609e9376a18f49b9aa5bb2fa7a1c993508d06b8 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 3 Dec 2015 18:29:15 -0200 Subject: [PATCH 05/40] Fix call to report_collection_diff in test "from" and "to" of the diff algorithm expect strings only --- testing/test_dsession.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_dsession.py b/testing/test_dsession.py index be638e4..f33fc41 100644 --- a/testing/test_dsession.py +++ b/testing/test_dsession.py @@ -277,7 +277,7 @@ def test_report_collection_diff_different(): ' ccc\n' '-YYY') - msg = report_collection_diff(from_collection, to_collection, 1, 2) + msg = report_collection_diff(from_collection, to_collection, '1', '2') assert msg == error_message From ce464aab39edc3c9bebdcccae7ee0aaa0ce9a598 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 3 Dec 2015 18:32:48 -0200 Subject: [PATCH 06/40] Update travis to use more recent python and pytest versions - Fix current "python" environment on travis to 3.5 - Updated build matrix to give preference of testing all pytest versions in py27 and py35 --- .travis.yml | 33 ++++++++++++++++++++------------- tox.ini | 4 ++-- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3443209..ab6e4f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python python: - - '3.5.0b3' + - '3.5' # command to install dependencies install: "pip install -U tox setuptools_scm" # # command to run tests @@ -10,23 +10,30 @@ env: - TESTENV=flakes - TESTENV=readme # matrix was trimmed to skip - # some builds that are unnecessary/perceived redundant - - TESTENV=py26-pytest24 - - TESTENV=py26-pytest25 - - TESTENV=py26-pytest26 + # some builds that are unnecessary/perceived redundant: + # py27 and py35 are tested with all pytest versions we declare to support; + # py26 and py34 only with latest pytest versions. - TESTENV=py26-pytest27 - - TESTENV=py33-pytest27 - - TESTENV=py34-pytest24 - - TESTENV=py34-pytest25 - - TESTENV=py34-pytest26 + - TESTENV=py26-pytest28 + - TESTENV=py27-pytest24 - TESTENV=py27-pytest25 - TESTENV=py27-pytest26 + - TESTENV=py27-pytest27 + - TESTENV=py27-pytest28 - - TESTENV=py27-pytest27-pexpect - - TESTENV=py34-pytest27-pexpect -# - TESTENV=py35-pytest27 - - TESTENV=pypy-pytest27 + - TESTENV=py34-pytest27 + - TESTENV=py34-pytest28 + + - TESTENV=py35-pytest24 + - TESTENV=py35-pytest25 + - TESTENV=py35-pytest26 + - TESTENV=py35-pytest27 + - TESTENV=py35-pytest28 + + - TESTENV=py27-pytest28-pexpect + - TESTENV=py34-pytest28-pexpect + - TESTENV=pypy-pytest28 script: tox --recreate -e $TESTENV diff --git a/tox.ini b/tox.ini index de603e2..189f3f8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist= - py{26,33,34,27}-pytest2{4,5,6,7},py{27,34}-pytest27-pexpect,flakes,readme + py{26,27,34,35}-pytest2{4,5,6,7,8},py{27,34}-pytest28-pexpect,flakes,readme [testenv] @@ -10,9 +10,9 @@ deps = setuptools_scm # to avoid .eggs pytest24: pytest~=2.4.0 pytest25: pytest~=2.5.0 - pytest26: pytest~=2.6.1 pytest27: pytest~=2.7.2 + pytest28: pytest~=2.8.3 pexpect: pexpect commands= # always clean to avoid code unmarshal mismatch on old python/pytest From c39e53e6b091905dd4d6b2c8f061ed54d2b6337d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 3 Dec 2015 18:41:48 -0200 Subject: [PATCH 07/40] pytest24,25,26 don't work on py35 due to changes in ast module so drop them from build matrix --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index ab6e4f6..9f98e43 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,7 @@ env: # some builds that are unnecessary/perceived redundant: # py27 and py35 are tested with all pytest versions we declare to support; # py26 and py34 only with latest pytest versions. + # pytest24,25,26 don't work on py35 due to changes in ast module - TESTENV=py26-pytest27 - TESTENV=py26-pytest28 @@ -25,9 +26,6 @@ env: - TESTENV=py34-pytest27 - TESTENV=py34-pytest28 - - TESTENV=py35-pytest24 - - TESTENV=py35-pytest25 - - TESTENV=py35-pytest26 - TESTENV=py35-pytest27 - TESTENV=py35-pytest28 From 757c0806c993d88707f9b147ec718b4f1b8aa489 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 1 Sep 2015 20:19:12 -0300 Subject: [PATCH 08/40] Add appveyor badge --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index 9494a2b..54c72e4 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,9 @@ .. image:: http://img.shields.io/pypi/v/pytest-xdist.svg :target: https://pypi.python.org/pypi/pytest-xdist +.. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true + :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist + xdist: pytest distributed testing plugin ========================================= From d59a4fb52be76ec74364cf21d98972a05b5f06b7 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 3 Dec 2015 18:52:11 -0200 Subject: [PATCH 09/40] Simplify AppVeyor build and add badge There's no point in creating a huge matrix because AppVeyor does not run the builds in parallel; in fact it will greatly increase build times as each matrix will boot a separate VM to execute on. Also removed install.ps1 and run_with_env.cmd, which are only really required if you have C extensions. --- appveyor.yml | 76 +--------------- appveyor/install.ps1 | 180 -------------------------------------- appveyor/run_with_env.cmd | 47 ---------- 3 files changed, 4 insertions(+), 299 deletions(-) delete mode 100644 appveyor/install.ps1 delete mode 100644 appveyor/run_with_env.cmd diff --git a/appveyor.yml b/appveyor.yml index 559f53b..f7dd1d1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,85 +1,17 @@ environment: - global: - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" - matrix: - - # Pre-installed Python versions, which Appveyor may upgrade to - # a later point release. + - PYTHON: "C:\\Python35" + TESTENV: "py35" - PYTHON: "C:\\Python27" - PYTHON_VERSION: "2.7.x" # currently 2.7.9 - PYTHON_ARCH: "32" TESTENV: "py27" - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7.x" # currently 2.7.9 - PYTHON_ARCH: "64" - TESTENV: "py27" - - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.x" # currently 3.3.5 - PYTHON_ARCH: "32" - TESTENV: "py33" - - - PYTHON: "C:\\Python33-x64" - PYTHON_VERSION: "3.3.x" # currently 3.3.5 - PYTHON_ARCH: "64" - TESTENV: "py33" - - - PYTHON: "C:\\Python34" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "32" - TESTENV: "py34" - - - PYTHON: "C:\\Python34-x64" - PYTHON_VERSION: "3.4.x" # currently 3.4.3 - PYTHON_ARCH: "64" - TESTENV: "py34" - - # Also test a Python version not pre-installed - # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 - - - PYTHON: "C:\\Python266" - PYTHON_VERSION: "2.6.6" - PYTHON_ARCH: "32" - TESTENV: "py26" - - install: - - ECHO "Filesystem root:" - - ps: "ls \"C:/\"" - - - ECHO "Installed SDKs:" - - ps: "ls \"C:/Program Files/Microsoft SDKs/Windows\"" - - # Install Python (from the official .msi of http://python.org) and pip when - # not already installed. - - ps: if (-not(Test-Path($env:PYTHON))) { & appveyor\install.ps1 } - - # Prepend newly installed Python to the PATH of this build (this cannot be - # done from inside the powershell script as it would require to restart - # the parent CMD process). - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - # Check that we have the expected version and architecture for Python - - "python --version" - - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" - - # Install the build dependencies of the project. If some dependencies contain - # compiled extensions and are not provided as pre-built wheel packages, - # pip will build them from source using the MSVC compiler matching the - # target Python version and architecture - - "%CMD_IN_ENV% pip install tox setuptools_scm" + - pip install tox setuptools_scm" build: false # Not a C# project, build stuff at the test step instead. test_script: # Build the compiled extension and run the project tests - - "%CMD_IN_ENV% tox -e %TESTENV%-pytest24" - - "%CMD_IN_ENV% tox -e %TESTENV%-pytest25" - - "%CMD_IN_ENV% tox -e %TESTENV%-pytest26" - - "%CMD_IN_ENV% tox -e %TESTENV%-pytest27,readme,flakes" + - tox -e %TESTENV%-pytest27,%TESTENV%-pytest28,readme,flakes diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 deleted file mode 100644 index 0f165d8..0000000 --- a/appveyor/install.ps1 +++ /dev/null @@ -1,180 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$MINICONDA_URL = "http://repo.continuum.io/miniconda/" -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - - -function DownloadPython ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "python-" + $python_version + $platform_suffix + ".msi" - $url = $BASE_URL + $python_version + "/" + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = ".amd64" - } - $msipath = DownloadPython $python_version $platform_suffix - Write-Host "Installing" $msipath "to" $python_home - $install_log = $python_home + ".log" - $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" - $uninstall_args = "/qn /x $msipath" - RunCommand "msiexec.exe" $install_args - if (-not(Test-Path $python_home)) { - Write-Host "Python seems to be installed else-where, reinstalling." - RunCommand "msiexec.exe" $uninstall_args - RunCommand "msiexec.exe" $install_args - } - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - -function RunCommand ($command, $command_args) { - Write-Host $command $command_args - Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "\Scripts\pip.exe" - $python_path = $python_home + "\python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - - -function DownloadMiniconda ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - if ($python_version -eq "3.4") { - $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" - } else { - $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" - } - $url = $MINICONDA_URL + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 3 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 2 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - if (Test-Path $filepath) { - Write-Host "File saved at" $filepath - } else { - # Retry once to get the error message if any at the last try - $webclient.DownloadFile($url, $filepath) - } - return $filepath -} - - -function InstallMiniconda ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "x86" - } else { - $platform_suffix = "x86_64" - } - $filepath = DownloadMiniconda $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $install_log = $python_home + ".log" - $args = "/S /D=$python_home" - Write-Host $filepath $args - Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru - if (Test-Path $python_home) { - Write-Host "Python $python_version ($architecture) installation complete" - } else { - Write-Host "Failed to install Python in $python_home" - Get-Content -Path $install_log - Exit 1 - } -} - - -function InstallMinicondaPip ($python_home) { - $pip_path = $python_home + "\Scripts\pip.exe" - $conda_path = $python_home + "\Scripts\conda.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $args = "install --yes pip" - Write-Host $conda_path $args - Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON -} - -main diff --git a/appveyor/run_with_env.cmd b/appveyor/run_with_env.cmd deleted file mode 100644 index 3a472bc..0000000 --- a/appveyor/run_with_env.cmd +++ /dev/null @@ -1,47 +0,0 @@ -:: To build extensions for 64 bit Python 3, we need to configure environment -:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) -:: -:: To build extensions for 64 bit Python 2, we need to configure environment -:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: -:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) -:: -:: 32 bit builds do not require specific environment configurations. -:: -:: Note: this script needs to be run with the /E:ON and /V:ON flags for the -:: cmd interpreter, at least for (SDK v7.0) -:: -:: More details at: -:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows -:: http://stackoverflow.com/a/13751649/163740 -:: -:: Author: Olivier Grisel -:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ -@ECHO OFF - -SET COMMAND_TO_RUN=%* -SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows - -SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" -IF %MAJOR_PYTHON_VERSION% == "2" ( - SET WINDOWS_SDK_VERSION="v7.0" -) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( - SET WINDOWS_SDK_VERSION="v7.1" -) ELSE ( - ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" - EXIT 1 -) - -IF "%PYTHON_ARCH%"=="64" ( - ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture - SET DISTUTILS_USE_SDK=1 - SET MSSdk=1 - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% - "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) ELSE ( - ECHO Using default MSVC build environment for 32 bit architecture - ECHO Executing: %COMMAND_TO_RUN% - call %COMMAND_TO_RUN% || EXIT 1 -) From 55d9886efd9be8c381167667ea325c3e137ce250 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 3 Dec 2015 19:12:59 -0200 Subject: [PATCH 10/40] Pass some environment variables so tmpdir can get the username in pytest27 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 189f3f8..66d2b4e 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,7 @@ envlist= [testenv] changedir=testing +passenv = USER USERNAME deps = pycmd setuptools_scm # to avoid .eggs From 318080a49dcf5ea744271c0decf8d28630590268 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Fri, 4 Dec 2015 10:03:57 -0200 Subject: [PATCH 11/40] Mark test_each_multiple as xfail Related to #20 --- testing/acceptance_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 9297bfb..70052c3 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -564,6 +564,7 @@ class TestNodeFailure: "*1 failed*1 passed*", ]) + @pytest.mark.xfail(reason='#20: xdist race condition on node restart') def test_each_multiple(self, testdir): f = testdir.makepyfile(""" import os From 5ddef63085725d2d5f3e6d87cad3d23cb99b557a Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 5 Dec 2015 12:03:30 -0200 Subject: [PATCH 12/40] xdist now works if the internal tmpdir plugin is disabled Fix #22 --- CHANGELOG | 1 + testing/acceptance_test.py | 12 ++++++++++++ xdist/slavemanage.py | 5 +++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ac74bc9..d12c29b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,7 @@ ------- - fix readme display on pypi +- fix #22: xdist now works if the internal tmpdir plugin is disabled. 1.13.1 diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 70052c3..bec87f1 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -522,6 +522,18 @@ def test_issue_594_random_parametrize(testdir): ]) +def test_tmpdir_disabled(testdir): + """Test xdist doesn't break if internal tmpdir plugin is disabled (#22). + """ + p1 = testdir.makepyfile(""" + def test_ok(): + pass + """) + result = testdir.runpytest(p1, "-n1", '-p', 'no:tmpdir') + assert result.ret == 0 + result.stdout.fnmatch_lines("*1 passed*") + + class TestNodeFailure: def test_load_single(self, testdir): f = testdir.makepyfile(""" diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index b50dab5..7632137 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -228,8 +228,9 @@ class SlaveController(object): option_dict = vars(self.config.option) if spec.popen: name = "popen-%s" % self.gateway.id - basetemp = self.config._tmpdirhandler.getbasetemp() - option_dict['basetemp'] = str(basetemp.join(name)) + if hasattr(self.config, '_tmpdirhandler'): + basetemp = self.config._tmpdirhandler.getbasetemp() + option_dict['basetemp'] = str(basetemp.join(name)) self.config.hook.pytest_configure_node(node=self) self.channel = self.gateway.remote_exec(xdist.remote) self.channel.send((self.slaveinput, args, option_dict)) From 10bc4b6e9aa46c54d2485aca523b62ebcc3a0ebc Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 8 Dec 2015 20:08:46 -0200 Subject: [PATCH 13/40] First version of architecture overview doc --- OVERVIEW.md | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 OVERVIEW.md diff --git a/OVERVIEW.md b/OVERVIEW.md new file mode 100644 index 0000000..552fb33 --- /dev/null +++ b/OVERVIEW.md @@ -0,0 +1,66 @@ +# Overview # + +Here it is described a brief overview of xdist's internal architecture. + + +`xdist` works by spawning one or more **worker nodes**, which are controlled +by the **master node**. Each **worker node** is responsible for performing +a full test collection and afterwards running tests as dictated by the **master node**. + +The execution flow is: + +1. **master node** spawns one or more **worker nodes** at the begginning of + the test session. The communication between **master** and **worker** nodes makes use of + [execnet](http://codespeak.net/execnet/) and its [gateways](http://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters). + The actual interpreters executing the code for the **worker nodes** might + be remote or local. + +1. Each **worker node** itself is a mini pytest runner. **workers** at this + point perform a full test collection, sending back the collected + test-ids back to the **master node** which does not + perform any collection itself. + +1. The **master node** receives the result of the collection from all nodes. + At this point the **master node** performs some sanity check to ensure that + all **worker nodes** collected the same tests (including order), bailing out otherwise. + If all is well, it converts the list of test-ids into a list of simple + indexes, where each index corresponds to the position of that test in the + original collection list. This works because all nodes have the same + collection list, and saves bandwidth because the **master** can now tell + one of the workers to just *execute test index 3* index of passing the + full test id. + +1. If **dist-mode** is **each**: the **master node** just sends the full list + of test indexes to each node at this moment. + +1. If **dist-mode** is **load**: the **master node** takes around 25% of the + tests and sends them one by one to each **worker node** in a round robin + fashion. The rest of the tests will be distributed later as **worker nodes** + finish tests (see below). + +1. **worker nodes** re-implement `pytest_runtestloop`: pytest's default implementation + basically loops over all collected items in the `session` object and executes + the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly + waiting for **master node** to send tests for execution. As tests are + received by **workers**, `pytest_runtest_protocol` is executed for each test. + Here it worth noting an implementation detail: at least one + test is kept always in **worker nodes** must they comply with + `pytest_runtest_protocol` in that it needs to know which will be the + `nextitem` in the hook call: either a new test in case the **master node** sends + a new test, or `None` if the **worker** receives a "shutdown" request. + +1. As tests are started and completed at the **workers**, the results are sent + back to the **master node**, which then just forwards the results to + the appropriate pytest hooks: `pytest_runtest_logstart` and + `pytest_runtest_logreport`. This way other plugins (for example `junitxml`) + can work normally. The **master node** (when in dist-mode **load**) + decides to send more tests to a node when a test completes, using + some heuristics such as test durations and how many tests each **worker node** + still has to run. + +1. When the **master node** has no more pending tests it will + send a "shutdown" signal to all **workers**, which will then run their + remaining tests to completion and shut down. At this point the + **master node** will sit waiting for **workers** to shut down, still + processing events such as `pytest_runtest_logreport`. + From 4ff6be21a85e4d2e8dbfc0d1a98abe3608c4c063 Mon Sep 17 00:00:00 2001 From: Hyunjun Kim Date: Wed, 9 Dec 2015 13:23:43 +0900 Subject: [PATCH 14/40] Fix typos --- example/boxed.txt | 2 +- xdist/dsession.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/boxed.txt b/example/boxed.txt index af3b025..00cec2f 100644 --- a/example/boxed.txt +++ b/example/boxed.txt @@ -2,7 +2,7 @@ If your testing involves C or C++ libraries you might have to deal with crashing processes. The xdist-plugin provides the ``--boxed`` option -to run each test in a controled subprocess. Here is a basic example:: +to run each test in a controlled subprocess. Here is a basic example:: # content of test_module.py diff --git a/xdist/dsession.py b/xdist/dsession.py index 25b7405..451a308 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -136,7 +136,7 @@ class EachScheduling: class LoadScheduling: - """Implement load scheduling accross nodes. + """Implement load scheduling across nodes. This distributes the tests collected across all nodes so each test is run just once. All nodes collect and submit the test suite and @@ -233,7 +233,7 @@ class LoadScheduling: execute. Called by the ``DSession.slave_slaveready`` hook when it - sucessfully bootstraps a new node. + successfully bootstraps a new node. """ assert node not in self.node2pending self.node2pending[node] = [] @@ -350,7 +350,7 @@ class LoadScheduling: """ assert self.collection_is_completed - # Initial distribution already happend, reschedule on all nodes + # Initial distribution already happened, reschedule on all nodes if self.collection is not None: for node in self.nodes: self.check_schedule(node) From 7da064f0fa5733ccf7bee24e83b6257495a9124c Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 9 Dec 2015 08:51:42 -0200 Subject: [PATCH 15/40] Mark test_auto_detect_cpus as xfail until #30 is fixed --- testing/acceptance_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index bec87f1..93479a1 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -353,6 +353,7 @@ def test_terminate_on_hangingnode(testdir): result.stdout.fnmatch_lines(["*killed*my*", ]) +@pytest.mark.xfail(reason='see #30') def test_auto_detect_cpus(testdir, monkeypatch): import multiprocessing monkeypatch.setattr(multiprocessing, 'cpu_count', lambda: 3) From b29ff737811a93f62b3d1e3f7bdf0af5818d47b5 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 9 Dec 2015 19:00:20 -0200 Subject: [PATCH 16/40] Apply small review requests * Fixed typo * Removed superfluous introduction --- OVERVIEW.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 552fb33..2562695 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -1,15 +1,12 @@ # Overview # -Here it is described a brief overview of xdist's internal architecture. - - `xdist` works by spawning one or more **worker nodes**, which are controlled by the **master node**. Each **worker node** is responsible for performing a full test collection and afterwards running tests as dictated by the **master node**. The execution flow is: -1. **master node** spawns one or more **worker nodes** at the begginning of +1. **master node** spawns one or more **worker nodes** at the beginning of the test session. The communication between **master** and **worker** nodes makes use of [execnet](http://codespeak.net/execnet/) and its [gateways](http://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters). The actual interpreters executing the code for the **worker nodes** might From c62effdd793133afa5b12109677701ae60f7928f Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 10 Dec 2015 19:55:31 -0200 Subject: [PATCH 17/40] Reword reason why workers must keep a single test on queue always --- OVERVIEW.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 2562695..62cb4d1 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -40,11 +40,12 @@ The execution flow is: the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly waiting for **master node** to send tests for execution. As tests are received by **workers**, `pytest_runtest_protocol` is executed for each test. - Here it worth noting an implementation detail: at least one - test is kept always in **worker nodes** must they comply with - `pytest_runtest_protocol` in that it needs to know which will be the - `nextitem` in the hook call: either a new test in case the **master node** sends - a new test, or `None` if the **worker** receives a "shutdown" request. + Here it worth noting an implementation detail: **workers** always must keep at + least one test item on their queue due to how the `pytest_runtest_protocol(item, nextitem)` + hook is defined: in order to pass the `nextitem` to the hook, the worker must wait for more + instructions from master before executing that remaining test. If it receives more tests, + then it can safely call `pytest_runtest_protocol` because it knows what the `nextitem` parameter will be. + If it receives a "shutdown" signal, then it can execute the hook passing `nextitem` as `None`. 1. As tests are started and completed at the **workers**, the results are sent back to the **master node**, which then just forwards the results to From 1716767a1c49dc12e1c42cfdc49ecf5d04aca3b0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 10 Dec 2015 20:00:43 -0200 Subject: [PATCH 18/40] Drop "node" from "workers" and "master" --- OVERVIEW.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index 62cb4d1..15fc1f6 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -1,25 +1,25 @@ # Overview # -`xdist` works by spawning one or more **worker nodes**, which are controlled -by the **master node**. Each **worker node** is responsible for performing -a full test collection and afterwards running tests as dictated by the **master node**. +`xdist` works by spawning one or more **workers**, which are controlled +by the **master**. Each **worker** is responsible for performing +a full test collection and afterwards running tests as dictated by the **master**. The execution flow is: -1. **master node** spawns one or more **worker nodes** at the beginning of +1. **master** spawns one or more **workers** at the beginning of the test session. The communication between **master** and **worker** nodes makes use of [execnet](http://codespeak.net/execnet/) and its [gateways](http://codespeak.net/execnet/basics.html#gateways-bootstrapping-python-interpreters). - The actual interpreters executing the code for the **worker nodes** might + The actual interpreters executing the code for the **workers** might be remote or local. -1. Each **worker node** itself is a mini pytest runner. **workers** at this +1. Each **worker** itself is a mini pytest runner. **workers** at this point perform a full test collection, sending back the collected - test-ids back to the **master node** which does not + test-ids back to the **master** which does not perform any collection itself. -1. The **master node** receives the result of the collection from all nodes. - At this point the **master node** performs some sanity check to ensure that - all **worker nodes** collected the same tests (including order), bailing out otherwise. +1. The **master** receives the result of the collection from all nodes. + At this point the **master** performs some sanity check to ensure that + all **workers** collected the same tests (including order), bailing out otherwise. If all is well, it converts the list of test-ids into a list of simple indexes, where each index corresponds to the position of that test in the original collection list. This works because all nodes have the same @@ -27,18 +27,18 @@ The execution flow is: one of the workers to just *execute test index 3* index of passing the full test id. -1. If **dist-mode** is **each**: the **master node** just sends the full list +1. If **dist-mode** is **each**: the **master** just sends the full list of test indexes to each node at this moment. -1. If **dist-mode** is **load**: the **master node** takes around 25% of the - tests and sends them one by one to each **worker node** in a round robin - fashion. The rest of the tests will be distributed later as **worker nodes** +1. If **dist-mode** is **load**: the **master** takes around 25% of the + tests and sends them one by one to each **worker** in a round robin + fashion. The rest of the tests will be distributed later as **workers** finish tests (see below). -1. **worker nodes** re-implement `pytest_runtestloop`: pytest's default implementation +1. **workers** re-implement `pytest_runtestloop`: pytest's default implementation basically loops over all collected items in the `session` object and executes the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly - waiting for **master node** to send tests for execution. As tests are + waiting for **master** to send tests for execution. As tests are received by **workers**, `pytest_runtest_protocol` is executed for each test. Here it worth noting an implementation detail: **workers** always must keep at least one test item on their queue due to how the `pytest_runtest_protocol(item, nextitem)` @@ -48,17 +48,17 @@ The execution flow is: If it receives a "shutdown" signal, then it can execute the hook passing `nextitem` as `None`. 1. As tests are started and completed at the **workers**, the results are sent - back to the **master node**, which then just forwards the results to + back to the **master**, which then just forwards the results to the appropriate pytest hooks: `pytest_runtest_logstart` and `pytest_runtest_logreport`. This way other plugins (for example `junitxml`) - can work normally. The **master node** (when in dist-mode **load**) + can work normally. The **master** (when in dist-mode **load**) decides to send more tests to a node when a test completes, using - some heuristics such as test durations and how many tests each **worker node** + some heuristics such as test durations and how many tests each **worker** still has to run. -1. When the **master node** has no more pending tests it will +1. When the **master** has no more pending tests it will send a "shutdown" signal to all **workers**, which will then run their remaining tests to completion and shut down. At this point the - **master node** will sit waiting for **workers** to shut down, still + **master** will sit waiting for **workers** to shut down, still processing events such as `pytest_runtest_logreport`. From 0a5bdfcda52b04cd4a8fa9cbaa3f685b7832124d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 10 Dec 2015 20:01:06 -0200 Subject: [PATCH 19/40] Add a FAQ section --- OVERVIEW.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/OVERVIEW.md b/OVERVIEW.md index 15fc1f6..25bb76f 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -62,3 +62,15 @@ The execution flow is: **master** will sit waiting for **workers** to shut down, still processing events such as `pytest_runtest_logreport`. +## FAQ ## + +> Why does each worker do its own collection, as opposed to having +the master collect once and distribute from that collection to the workers? + +If collection was performed by master then it would have to +serialize collected items to send them through the wire, as workers live in another process. +The problem is that test items are not easily (impossible?) to serialize, as they contain references to +the test functions, fixture managers, config objects, etc. Even if one manages to serialize it, +it seems it would be very hard to get it right and easy to break by any small change in pytest. + + From c58d22f2797f4261f625f13e4785e928b37ac8f6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 12 Dec 2015 10:26:31 -0200 Subject: [PATCH 20/40] Make xdist work even if looponfail or boxed are disabled Fix #32 --- CHANGELOG | 11 ++++++++--- testing/acceptance_test.py | 13 +++++++++++++ xdist/looponfail.py | 4 ++++ xdist/plugin.py | 8 ++------ 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index d12c29b..85302a2 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,13 @@ -1.13.2 -------- +1.13.2.dev +---------- + +- fix README display on pypi -- fix readme display on pypi - fix #22: xdist now works if the internal tmpdir plugin is disabled. + Thanks Bruno Oliveira for the PR. + +- fix #32: xdist now works if looponfail or boxed are disabled. + Thanks Bruno Oliveira for the PR. 1.13.1 diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 93479a1..c074674 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -535,6 +535,19 @@ def test_tmpdir_disabled(testdir): result.stdout.fnmatch_lines("*1 passed*") +@pytest.mark.parametrize('plugin', ['xdist.looponfail', 'xdist.boxed']) +def test_sub_plugins_disabled(testdir, plugin): + """Test that xdist doesn't break if we disable any of its sub-plugins. (#32) + """ + p1 = testdir.makepyfile(""" + def test_ok(): + pass + """) + result = testdir.runpytest(p1, "-n1", '-p', 'no:%s' % plugin) + assert result.ret == 0 + result.stdout.fnmatch_lines("*1 passed*") + + class TestNodeFailure: def test_load_single(self, testdir): f = testdir.makepyfile(""" diff --git a/xdist/looponfail.py b/xdist/looponfail.py index 99604bb..6f76b43 100644 --- a/xdist/looponfail.py +++ b/xdist/looponfail.py @@ -25,6 +25,10 @@ def pytest_addoption(parser): def pytest_cmdline_main(config): if config.getoption("looponfail"): + usepdb = config.getoption('usepdb') # a core option + if usepdb: + raise pytest.UsageError( + "--pdb incompatible with --looponfail.") looponfail_main(config) return 2 # looponfail only can get stop with ctrl-C anyway diff --git a/xdist/plugin.py b/xdist/plugin.py index dc3635e..69c3a9b 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -94,12 +94,8 @@ def pytest_cmdline_main(config): config.option.dist = "load" val = config.getvalue if not val("collectonly"): - usepdb = config.option.usepdb # a core option - if val("looponfail"): - if usepdb: - raise pytest.UsageError( - "--pdb incompatible with --looponfail.") - elif val("dist") != "no": + usepdb = config.getoption('usepdb') # a core option + if val("dist") != "no": if usepdb: raise pytest.UsageError( "--pdb incompatible with distributing tests.") From 2ae9bd7f4316a0c82f71f0039ebfbd684fbd1998 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 12 Dec 2015 12:38:45 -0200 Subject: [PATCH 21/40] Add new pytest_xdist_node_collection_finished hook Fix #8 --- CHANGELOG | 7 ++++++ testing/test_newhooks.py | 50 ++++++++++++++++++++++++++++++++++++++++ xdist/dsession.py | 2 ++ xdist/newhooks.py | 19 +++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 testing/test_newhooks.py diff --git a/CHANGELOG b/CHANGELOG index 85302a2..62717c7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +1.14.0.dev +---------- + +- new hook: ``pytest_xdist_node_collection_finished(node, ids)``, called when + a worker has finished collection. Thanks Omer Katz for the request and + Bruno Oliveira for the PR. + 1.13.2.dev ---------- diff --git a/testing/test_newhooks.py b/testing/test_newhooks.py new file mode 100644 index 0000000..a6737ea --- /dev/null +++ b/testing/test_newhooks.py @@ -0,0 +1,50 @@ +import pytest + + +class TestHooks: + + @pytest.fixture(autouse=True) + def create_test_file(self, testdir): + testdir.makepyfile(""" + import os + def test_a(): pass + def test_b(): pass + def test_c(): pass + """) + + def test_runtest_logreport(self, testdir): + """Test that log reports from pytest_runtest_logreport when running + with xdist contain a "node" attribute. (#8) + """ + testdir.makeconftest(""" + def pytest_runtest_logreport(report): + if hasattr(report, 'node'): + slaveid = report.node.slaveinput['slaveid'] + if report.when == "call": + print("HOOK: %s %s" % (report.nodeid, slaveid)) + """) + res = testdir.runpytest('-n1', '-s') + res.stdout.fnmatch_lines([ + '*HOOK: test_runtest_logreport.py::test_a gw0*', + '*HOOK: test_runtest_logreport.py::test_b gw0*', + '*HOOK: test_runtest_logreport.py::test_c gw0*', + '*3 passed*', + ]) + + def test_node_collection_finished(self, testdir): + """Test pytest_xdist_node_collection_finished hook (#8). + """ + testdir.makeconftest(""" + def pytest_xdist_node_collection_finished(node, ids): + slaveid = node.slaveinput['slaveid'] + stripped_ids = [x.split('::')[1] for x in ids] + print("HOOK: %s %s" % (slaveid, ', '.join(stripped_ids))) + """) + res = testdir.runpytest('-n2', '-s') + res.stdout.fnmatch_lines_random([ + '*HOOK: gw0 test_a, test_b, test_c', + '*HOOK: gw1 test_a, test_b, test_c', + ]) + res.stdout.fnmatch_lines([ + '*3 passed*', + ]) diff --git a/xdist/dsession.py b/xdist/dsession.py index 451a308..fa66f1d 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -629,6 +629,8 @@ class DSession: """ if self.shuttingdown: return + self.config.hook.pytest_xdist_node_collection_finished(node=node, + ids=ids) # tell session which items were effectively collected otherwise # the master node will finish the session with EXIT_NOTESTSCOLLECTED self._session.testscollected = len(ids) diff --git a/xdist/newhooks.py b/xdist/newhooks.py index 0207935..d31aed8 100644 --- a/xdist/newhooks.py +++ b/xdist/newhooks.py @@ -1,3 +1,17 @@ +""" +xdist hooks. + +Additionally, pytest-xdist will also decorate a few other hooks +with the worker instance that executed the hook originally: + +``pytest_runtest_logreport``: ``rep`` parameter has a ``node`` attribute. + +You can use this hooks just as you would use normal pytest hooks, but some care +must be taken in plugins in case ``xdist`` is not installed. Please see: + + http://pytest.org/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins +""" + def pytest_xdist_setupnodes(config, specs): """ called before any remote node is set up. """ @@ -25,3 +39,8 @@ def pytest_testnodeready(node): def pytest_testnodedown(node, error): """ Test Node is down. """ + + +def pytest_xdist_node_collection_finished(node, ids): + """called by the master node when a node finishes collecting. + """ From 462a3eb9e205442fd2f436c4f6a9a5d9d66dceab Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 12 Dec 2015 12:55:41 -0200 Subject: [PATCH 22/40] Fix test_auto_detect_cpus flakyness Fix #30 --- testing/acceptance_test.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index c074674..22f61c8 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -353,18 +353,15 @@ def test_terminate_on_hangingnode(testdir): result.stdout.fnmatch_lines(["*killed*my*", ]) -@pytest.mark.xfail(reason='see #30') def test_auto_detect_cpus(testdir, monkeypatch): import multiprocessing - monkeypatch.setattr(multiprocessing, 'cpu_count', lambda: 3) + count = multiprocessing.cpu_count() testdir.makeconftest(""" - def pytest_unconfigure(config): - with open('cpus', 'w') as f: - f.write('cpus = %s' % config.option.numprocesses) + def pytest_configure(config): + print("numprocesses: (%s)" % config.getoption("numprocesses")) """) - testdir.inline_run('-n=auto') - cpus_file = testdir.tmpdir.join('cpus') - assert cpus_file.read() == 'cpus = 3' + result = testdir.runpytest('-n=auto', '-s') + result.stdout.fnmatch_lines(["*numprocesses: (%s)*" % count]) @pytest.mark.xfail(reason="works if run outside test suite", run=False) From 0127cac3f97da5d89d0401028c77e562ffddc6e0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 14 Dec 2015 22:51:41 -0200 Subject: [PATCH 23/40] Unit test "auto" cpu detection Use that instead of an acceptance test: on travis it would result in over 16 slaves being spawned, which caused weird crashes on multiprocessing module during teardown Fix #30 --- testing/acceptance_test.py | 11 ----------- testing/test_plugin.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 22f61c8..c52a6da 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -353,17 +353,6 @@ def test_terminate_on_hangingnode(testdir): result.stdout.fnmatch_lines(["*killed*my*", ]) -def test_auto_detect_cpus(testdir, monkeypatch): - import multiprocessing - count = multiprocessing.cpu_count() - testdir.makeconftest(""" - def pytest_configure(config): - print("numprocesses: (%s)" % config.getoption("numprocesses")) - """) - result = testdir.runpytest('-n=auto', '-s') - result.stdout.fnmatch_lines(["*numprocesses: (%s)*" % count]) - - @pytest.mark.xfail(reason="works if run outside test suite", run=False) def test_session_hooks(testdir): testdir.makeconftest(""" diff --git a/testing/test_plugin.py b/testing/test_plugin.py index c3e87c7..3fbb5b3 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -25,6 +25,17 @@ def test_dist_options(testdir): assert config.option.dist == "load" +def test_auto_detect_cpus(testdir, monkeypatch): + import multiprocessing + monkeypatch.setattr(multiprocessing, 'cpu_count', lambda: 99) + + config = testdir.parseconfigure("-n2") + assert config.getoption('numprocesses') == 2 + + config = testdir.parseconfigure("-nauto") + assert config.getoption('numprocesses') == 99 + + class TestDistOptions: def test_getxspecs(self, testdir): config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz") From eee72557f7e992f8c331363b31f38a74d9d98f11 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 30 Jan 2016 11:27:36 -0200 Subject: [PATCH 24/40] Fix CHANGELOG for 1.14 release --- CHANGELOG | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 62717c7..15b4f10 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,13 +1,10 @@ -1.14.0.dev ----------- +1.14 +---- - new hook: ``pytest_xdist_node_collection_finished(node, ids)``, called when a worker has finished collection. Thanks Omer Katz for the request and Bruno Oliveira for the PR. -1.13.2.dev ----------- - - fix README display on pypi - fix #22: xdist now works if the internal tmpdir plugin is disabled. From 023226840c5dd1acd9927096f3739b87783afeda Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 30 Jan 2016 12:03:55 -0200 Subject: [PATCH 25/40] Prepare CHANGELOG for next release --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 15b4f10..aa7a5a4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +1.14.1.dev +---------- + + 1.14 ---- From 96506867d1acc4fea5671e93180c131f3656d172 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 30 Jan 2016 12:19:52 -0200 Subject: [PATCH 26/40] Refactor tox and ci scripts Make sure a plain "tox" command works in all platforms without failures, which facilitates making releases using a "devpi upload/test" workflow * Fix small flakes failures * Limit py35 testing to pytest>=2.7, as pytest<2.7 does not work on py35 * pexpect environments only work on Linux platforms and should be skipped on Windows * Simplify AppVeyor script by running all tox environments: no need to have a build matrix because AppVeyor does not execute builds in parallel * Add all environments to travis.yml, obtained from "tox --listenvs" --- .travis.yml | 23 ++++++++++------------- appveyor.yml | 14 ++------------ testing/test_remote.py | 3 ++- tox.ini | 25 ++++++++++++++++--------- xdist/__init__.py | 3 ++- 5 files changed, 32 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9f98e43..3f7910e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,31 +7,28 @@ install: "pip install -U tox setuptools_scm" # # command to run tests env: matrix: - - TESTENV=flakes - - TESTENV=readme - # matrix was trimmed to skip - # some builds that are unnecessary/perceived redundant: - # py27 and py35 are tested with all pytest versions we declare to support; - # py26 and py34 only with latest pytest versions. - # pytest24,25,26 don't work on py35 due to changes in ast module + # note: please use "tox --listenvs" to populate the build matrix + - TESTENV=py26-pytest24 + - TESTENV=py26-pytest25 + - TESTENV=py26-pytest26 - TESTENV=py26-pytest27 - TESTENV=py26-pytest28 - - TESTENV=py27-pytest24 - TESTENV=py27-pytest25 - TESTENV=py27-pytest26 - TESTENV=py27-pytest27 - TESTENV=py27-pytest28 - + - TESTENV=py34-pytest24 + - TESTENV=py34-pytest25 + - TESTENV=py34-pytest26 - TESTENV=py34-pytest27 - TESTENV=py34-pytest28 - - TESTENV=py35-pytest27 - TESTENV=py35-pytest28 - - TESTENV=py27-pytest28-pexpect - - TESTENV=py34-pytest28-pexpect - - TESTENV=pypy-pytest28 + - TESTENV=py35-pytest28-pexpect + - TESTENV=flakes + - TESTENV=readme script: tox --recreate -e $TESTENV diff --git a/appveyor.yml b/appveyor.yml index f7dd1d1..6c5f28a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,17 +1,7 @@ -environment: - matrix: - - PYTHON: "C:\\Python35" - TESTENV: "py35" - - - PYTHON: "C:\\Python27" - TESTENV: "py27" - install: - - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - pip install tox setuptools_scm" + - C:\Python35\python -m pip install tox setuptools_scm build: false # Not a C# project, build stuff at the test step instead. test_script: - # Build the compiled extension and run the project tests - - tox -e %TESTENV%-pytest27,%TESTENV%-pytest28,readme,flakes + - C:\Python35\python -m tox diff --git a/testing/test_remote.py b/testing/test_remote.py index 53dc2f2..7bac373 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -2,9 +2,10 @@ import py from xdist.slavemanage import SlaveController, unserialize_report from xdist.remote import serialize_report import execnet -queue = py.builtin._tryimport("queue", "Queue") import marshal +queue = py.builtin._tryimport("queue", "Queue") + WAIT_TIMEOUT = 10.0 diff --git a/tox.ini b/tox.ini index 66d2b4e..54fb438 100644 --- a/tox.ini +++ b/tox.ini @@ -1,20 +1,27 @@ [tox] +# if you change the envlist, please update .travis.yml file as well envlist= - py{26,27,34,35}-pytest2{4,5,6,7,8},py{27,34}-pytest28-pexpect,flakes,readme + py{26,27,34}-pytest2{4,5,6,7,8} + py35-pytest2{7,8} + py{27,35}-pytest28-pexpect + flakes + readme [testenv] changedir=testing passenv = USER USERNAME deps = - pycmd - setuptools_scm # to avoid .eggs - pytest24: pytest~=2.4.0 - pytest25: pytest~=2.5.0 - pytest26: pytest~=2.6.1 - pytest27: pytest~=2.7.2 - pytest28: pytest~=2.8.3 - pexpect: pexpect + pycmd + setuptools_scm # to avoid .eggs + pytest24: pytest~=2.4.0 + pytest25: pytest~=2.5.0 + pytest26: pytest~=2.6.1 + pytest27: pytest~=2.7.2 + pytest28: pytest~=2.8.3 + pexpect: pexpect +platform= + pexpect: linux|darwin commands= # always clean to avoid code unmarshal mismatch on old python/pytest py.cleanup -aq diff --git a/xdist/__init__.py b/xdist/__init__.py index 4e791df..7aa44d0 100644 --- a/xdist/__init__.py +++ b/xdist/__init__.py @@ -1,2 +1,3 @@ -__all__ = ['__version__'] from xdist._version import version as __version__ + +__all__ = ['__version__'] From a11632b5d01a72fce8245ad58f8ac17c73388db6 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Sat, 30 Jan 2016 21:36:44 -0200 Subject: [PATCH 27/40] Remove old .hgtags file --- .hgtags | 21 --------------------- MANIFEST.in | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 .hgtags diff --git a/.hgtags b/.hgtags deleted file mode 100644 index cdb1359..0000000 --- a/.hgtags +++ /dev/null @@ -1,21 +0,0 @@ -42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0 -eca7ce17eabf296983c36812c8b8be901e7055a3 1.1 -56d8e5280be224a0ad3220a9deed55334710bd23 1.2 -e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3 -e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3 -eaf8b1cb7c312883598677231be5bbeea3b5c127 1.3 -a423748bf17ee778a37853225210257699cad9c1 1.4 -cd44a941c833c098e4899fe3d42a96703754d0d5 1.5 -4815040bdad8f182a5487f57a9da385483836e75 1.6 -20875fed94e7f3dff50bdf762df91153b15ceca6 1.7 -20875fed94e7f3dff50bdf762df91153b15ceca6 1.7 -29c38e195526f5f0fdd651fb51f59d6efaaafbb0 1.7 -0d1c00018008433956aa7d93007bab6ea7de96e4 1.8 -0d1c00018008433956aa7d93007bab6ea7de96e4 1.8 -1d27987c267577899350a25ba5828d55d87083ad 1.8 -5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9 -4406fc2a6427fadc021ed7e43e7aa5032b1ea91f 1.10 -220f6e46eb71a6212ccbe6b67b9e6edcf8ee4fa5 1.11 -39ef85dbc893cc63dede11601208098a667b58e9 1.12 -4e25f4c568be2d7cb4d1739638a9e66bbf28f588 v1.13 -67ff3aa4d294f75ade0ebf03267e5739e1bd9473 v1.13.1 diff --git a/MANIFEST.in b/MANIFEST.in index cde85f9..4549c3b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ include README.txt include setup.py include tox.ini graft testing -prune .hg +prune .git From 7f8ae3944c0ad68c460b3fe9c135de664206f096 Mon Sep 17 00:00:00 2001 From: hellmanj Date: Tue, 23 Feb 2016 17:56:41 -0500 Subject: [PATCH 28/40] worker_id fixture as shown in #47 --- README.rst | 10 ++++++++++ testing/acceptance_test.py | 15 +++++++++++++++ xdist/plugin.py | 12 ++++++++++++ 3 files changed, 37 insertions(+) diff --git a/README.rst b/README.rst index 54c72e4..0cbb08b 100644 --- a/README.rst +++ b/README.rst @@ -185,6 +185,16 @@ at once. The specifications strings use the `xspec syntax`_. .. _`execnet`: http://codespeak.net/execnet +Identifying the worker process during a test ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +If you need to determine the identity of a worker process in +a test or fixture, you may use the ``worker_id`` fixture to do so:: + + @pytest.fixture() + def user_account(worker_id): + """ use a different account in each xdist worker """ + return "account_%s" % worker_id + Specifying test exec environments in an ini file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index c52a6da..036d187 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -620,3 +620,18 @@ class TestNodeFailure: "*Slave*crashed while running*", "*1 failed*2 passed*", ]) + +def test_worker_id_fixture(testdir): + f = testdir.makepyfile(""" + import pytest + @pytest.mark.parametrize("run_num", [1,2]) + def test_worker_id1(worker_id, run_num): + with open("worker_id%s" % run_num, "w") as f: + f.write(worker_id) + """) + result = testdir.runpytest(f, "-n2") + worker_ids = [] + for run_num in [1,2]: + worker_id_file_path = testdir.tmpdir.join("worker_id%s" % run_num).strpath + worker_ids.append(open(worker_id_file_path, "r").read()) + assert "gw0" in worker_ids and "gw1" in worker_ids diff --git a/xdist/plugin.py b/xdist/plugin.py index 69c3a9b..c232a5e 100644 --- a/xdist/plugin.py +++ b/xdist/plugin.py @@ -99,3 +99,15 @@ def pytest_cmdline_main(config): if usepdb: raise pytest.UsageError( "--pdb incompatible with distributing tests.") + +# ------------------------------------------------------------------------- +# fixtures +# ------------------------------------------------------------------------- + + +@pytest.fixture(scope="session") +def worker_id(request): + if hasattr(request.config, 'slaveinput'): + return request.config.slaveinput['slaveid'] + else: + return 'master' From 7e6011541fd02fb70f0c3c97eb58099ff2ff65f1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 23 Feb 2016 20:39:41 -0300 Subject: [PATCH 29/40] Fix flakes --- testing/acceptance_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 036d187..f5ad367 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -621,6 +621,7 @@ class TestNodeFailure: "*1 failed*2 passed*", ]) + def test_worker_id_fixture(testdir): f = testdir.makepyfile(""" import pytest @@ -630,8 +631,9 @@ def test_worker_id_fixture(testdir): f.write(worker_id) """) result = testdir.runpytest(f, "-n2") + result.stdout.fnmatch_lines('* 2 passed in *') worker_ids = [] - for run_num in [1,2]: - worker_id_file_path = testdir.tmpdir.join("worker_id%s" % run_num).strpath - worker_ids.append(open(worker_id_file_path, "r").read()) + for run_num in [1, 2]: + worker_id_file_path = testdir.tmpdir.join("worker_id%s" % run_num) + worker_ids.append(open(str(worker_id_file_path), "r").read()) assert "gw0" in worker_ids and "gw1" in worker_ids From b27301e1d929ad97bd96a3aca9f95c0e3b5c4b51 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 23 Feb 2016 20:53:46 -0300 Subject: [PATCH 30/40] Improve test a bit and add CHANGELOG entry --- CHANGELOG | 5 ++++- README.rst | 3 +++ testing/acceptance_test.py | 23 ++++++++++++++--------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index aa7a5a4..1e212b6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,9 @@ -1.14.1.dev +1.15.0.dev ---------- +- new ``worker_id`` fixture, returns the id of the worker in a test or fixture. + Thanks Jared Hellman for the PR. + 1.14 ---- diff --git a/README.rst b/README.rst index 0cbb08b..04452cb 100644 --- a/README.rst +++ b/README.rst @@ -195,6 +195,9 @@ a test or fixture, you may use the ``worker_id`` fixture to do so:: """ use a different account in each xdist worker """ return "account_%s" % worker_id +When ``xdist`` is disabled (running with ``-n0`` for example), then +``worker_id`` will return ``"master"``. + Specifying test exec environments in an ini file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index f5ad367..9ba158b 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -622,18 +622,23 @@ class TestNodeFailure: ]) -def test_worker_id_fixture(testdir): +@pytest.mark.parametrize('n', [0, 2]) +def test_worker_id_fixture(testdir, n): + import glob f = testdir.makepyfile(""" import pytest - @pytest.mark.parametrize("run_num", [1,2]) + @pytest.mark.parametrize("run_num", range(2)) def test_worker_id1(worker_id, run_num): - with open("worker_id%s" % run_num, "w") as f: + with open("worker_id%s.txt" % run_num, "w") as f: f.write(worker_id) """) - result = testdir.runpytest(f, "-n2") + result = testdir.runpytest(f, "-n%d" % n) result.stdout.fnmatch_lines('* 2 passed in *') - worker_ids = [] - for run_num in [1, 2]: - worker_id_file_path = testdir.tmpdir.join("worker_id%s" % run_num) - worker_ids.append(open(str(worker_id_file_path), "r").read()) - assert "gw0" in worker_ids and "gw1" in worker_ids + worker_ids = set() + for fname in glob.glob(str(testdir.tmpdir.join("*.txt"))): + with open(fname) as f: + worker_ids.add(f.read().strip()) + if n == 0: + assert worker_ids == set(['master']) + else: + assert worker_ids == set(['gw0', 'gw1']) From ba35a3da02e78acecbe3e82edaae44f601758aeb Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 23 Feb 2016 22:36:42 -0300 Subject: [PATCH 31/40] Add syntax highlight in README --- README.rst | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 04452cb..29fa5dd 100644 --- a/README.rst +++ b/README.rst @@ -188,7 +188,9 @@ at once. The specifications strings use the `xspec syntax`_. Identifying the worker process during a test +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ If you need to determine the identity of a worker process in -a test or fixture, you may use the ``worker_id`` fixture to do so:: +a test or fixture, you may use the ``worker_id`` fixture to do so: + +.. code-block:: python @pytest.fixture() def user_account(worker_id): @@ -203,12 +205,16 @@ Specifying test exec environments in an ini file pytest (since version 2.0) supports ini-style cofiguration. You can for example make running with three subprocesses -your default like this:: +your default like this: + +.. code-block:: ini [pytest] addopts = -n3 -You can also add default environments like this:: +You can also add default environments like this: + +.. code-block:: ini [pytest] addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6 @@ -223,7 +229,9 @@ Specifying "rsync" dirs in an ini-file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ In a ``tox.ini`` or ``setup.cfg`` file in your root project directory -you may specify directories to include or to exclude in synchronisation:: +you may specify directories to include or to exclude in synchronisation: + +.. code-block:: ini [pytest] rsyncdirs = . mypkg helperpkg From 8954f0d63dd45e9eec1a7f935870ac7c7d2d0bf2 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 22 Mar 2016 20:53:07 -0300 Subject: [PATCH 32/40] Add Framework::Pytest to list of classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7a5a73e..34c91f6 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ setup( setup_requires=['setuptools_scm'], classifiers=[ 'Development Status :: 5 - Production/Stable', + 'Framework :: Pytest', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: POSIX', From 20fe1f987487b53e927fe8e86e427007b29096eb Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 8 Mar 2016 17:20:36 -0300 Subject: [PATCH 33/40] Skip progress display when in non-terminal (pytest >= 2.9) See pytest-dev/pytest#1397 --- CHANGELOG | 3 +++ testing/acceptance_test.py | 22 ++++++++++++++++++++++ xdist/dsession.py | 5 +++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 1e212b6..29c5dd5 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,9 @@ - new ``worker_id`` fixture, returns the id of the worker in a test or fixture. Thanks Jared Hellman for the PR. +- display progress during collection only when in a terminal, similar to pytest #1397 issue. + Thanks Bruno Oliveira for the PR. + 1.14 ---- diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 9ba158b..a736ad9 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -642,3 +642,25 @@ def test_worker_id_fixture(testdir, n): assert worker_ids == set(['master']) else: assert worker_ids == set(['gw0', 'gw1']) + + +def test_color_yes_collection_on_non_atty(testdir, request): + """skip collect progress report when working on non-terminals. + + Similar to pytest-dev/pytest#1397 + """ + tr = request.config.pluginmanager.getplugin("terminalreporter") + if not hasattr(tr, 'isatty'): + pytest.skip('only valid for newer pytest versions') + testdir.makepyfile(""" + import pytest + @pytest.mark.parametrize('i', range(10)) + def test_this(i): + assert 1 + """) + args = ['--color=yes', '-n2'] + result = testdir.runpytest(*args) + assert 'test session starts' in result.stdout.str() + assert '\x1b[1m' in result.stdout.str() + assert 'gw0 [10] / gw1 [10]' in result.stdout.str() + assert 'gw0 C / gw1 C' not in result.stdout.str() diff --git a/xdist/dsession.py b/xdist/dsession.py index fa66f1d..ae71db4 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -722,17 +722,18 @@ class TerminalDistReporter: self.tr = config.pluginmanager.getplugin("terminalreporter") self._status = {} self._lastlen = 0 + self._isatty = getattr(self.tr, 'isatty', self.tr.hasmarkup) def write_line(self, msg): self.tr.write_line(msg) def ensure_show_status(self): - if not self.tr.hasmarkup: + if not self._isatty: self.write_line(self.getstatus()) def setstatus(self, spec, status, show=True): self._status[spec.id] = status - if show and self.tr.hasmarkup: + if show and self._isatty: self.rewrite(self.getstatus()) def getstatus(self): From dd3d180fe3161f9a9a450a83d1b30c0186ecf7e1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 4 May 2016 17:34:45 -0300 Subject: [PATCH 34/40] Add pytest 2.9 and drop pytest 2.4 and 2.5 from build matrix --- .travis.yml | 10 ++++------ tox.ini | 9 ++++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3f7910e..6acf63a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,23 +8,21 @@ install: "pip install -U tox setuptools_scm" env: matrix: # note: please use "tox --listenvs" to populate the build matrix - - TESTENV=py26-pytest24 - - TESTENV=py26-pytest25 - TESTENV=py26-pytest26 - TESTENV=py26-pytest27 - TESTENV=py26-pytest28 - - TESTENV=py27-pytest24 - - TESTENV=py27-pytest25 + - TESTENV=py26-pytest29 - TESTENV=py27-pytest26 - TESTENV=py27-pytest27 - TESTENV=py27-pytest28 - - TESTENV=py34-pytest24 - - TESTENV=py34-pytest25 + - TESTENV=py27-pytest29 - TESTENV=py34-pytest26 - TESTENV=py34-pytest27 - TESTENV=py34-pytest28 + - TESTENV=py34-pytest29 - TESTENV=py35-pytest27 - TESTENV=py35-pytest28 + - TESTENV=py35-pytest29 - TESTENV=py27-pytest28-pexpect - TESTENV=py35-pytest28-pexpect - TESTENV=flakes diff --git a/tox.ini b/tox.ini index 54fb438..4e9273e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,8 @@ [tox] # if you change the envlist, please update .travis.yml file as well envlist= - py{26,27,34}-pytest2{4,5,6,7,8} - py35-pytest2{7,8} + py{26,27,34}-pytest2{6,7,8,9} + py35-pytest2{7,8,9} py{27,35}-pytest28-pexpect flakes readme @@ -14,11 +14,10 @@ passenv = USER USERNAME deps = pycmd setuptools_scm # to avoid .eggs - pytest24: pytest~=2.4.0 - pytest25: pytest~=2.5.0 pytest26: pytest~=2.6.1 pytest27: pytest~=2.7.2 - pytest28: pytest~=2.8.3 + pytest28: pytest~=2.8.7 + pytest29: pytest~=2.9.1 pexpect: pexpect platform= pexpect: linux|darwin From 8ec8d29cbf4f01ef20b88c1cad48ec93ef0fc500 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 11 May 2016 22:25:12 -0300 Subject: [PATCH 35/40] Mark test_remoteinitconfig as xfail #59 --- testing/test_remote.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/test_remote.py b/testing/test_remote.py index 7bac373..8192b7d 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -1,4 +1,5 @@ import py +import pytest from xdist.slavemanage import SlaveController, unserialize_report from xdist.remote import serialize_report import execnet @@ -62,6 +63,7 @@ def pytest_funcarg__slave(request): return SlaveSetup(request) +@pytest.mark.xfail(reason='#59') def test_remoteinitconfig(testdir): from xdist.remote import remote_initconfig config1 = testdir.parseconfig() From 9786e3d7744f5312cf45d13f6471ff29ee11c571 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 11 May 2016 22:28:22 -0300 Subject: [PATCH 36/40] Move comment in tox.ini Latest tox seems to be broken in this regard, it is not stripping the comment from the dependency line --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4e9273e..7095101 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,8 @@ changedir=testing passenv = USER USERNAME deps = pycmd - setuptools_scm # to avoid .eggs + # to avoid .eggs + setuptools_scm pytest26: pytest~=2.6.1 pytest27: pytest~=2.7.2 pytest28: pytest~=2.8.7 From b4a7a1a8a84ddf18462208aa2333cf4f8e1e113e Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Mon, 6 Jun 2016 20:46:25 -0300 Subject: [PATCH 37/40] Fix internal error when using --maxfail option Fix #65 Fix #62 --- CHANGELOG | 3 +++ testing/acceptance_test.py | 21 +++++++++++++++++++++ xdist/remote.py | 5 ++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 29c5dd5..247f3a4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -7,6 +7,9 @@ - display progress during collection only when in a terminal, similar to pytest #1397 issue. Thanks Bruno Oliveira for the PR. +- fix internal error message when ``--maxfail`` is used (#62, #65). + Thanks Collin RM Stocks and Bryan A. Jones for reports and Bruno Oliveira for the PR. + 1.14 ---- diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index a736ad9..cccb8f7 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -664,3 +664,24 @@ def test_color_yes_collection_on_non_atty(testdir, request): assert '\x1b[1m' in result.stdout.str() assert 'gw0 [10] / gw1 [10]' in result.stdout.str() assert 'gw0 C / gw1 C' not in result.stdout.str() + + +def test_internal_error_with_maxfail(testdir): + """ + Internal error when using --maxfail option (#62, #65). + """ + testdir.makepyfile(""" + import pytest + + @pytest.fixture(params=['1', '2']) + def crasher(): + raise RuntimeError + + def test_aaa0(crasher): + pass + def test_aaa1(crasher): + pass + """) + result = testdir.runpytest_subprocess('--maxfail=1', '-n1') + result.stdout.fnmatch_lines(['* 1 error in *']) + assert 'INTERNALERROR' not in result.stderr.str() diff --git a/xdist/remote.py b/xdist/remote.py index 0d6997f..226262a 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -46,7 +46,10 @@ class SlaveInteractor: self.log("entering main loop") torun = [] while 1: - name, kwargs = self.channel.receive() + try: + name, kwargs = self.channel.receive() + except EOFError: + return True self.log("received command", name, kwargs) if name == "runtests": torun.extend(kwargs['indices']) From 305acdd72a03771137739dd3aa7c5405cf2a8ba9 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 7 Jun 2016 19:35:08 -0300 Subject: [PATCH 38/40] Cleanly shutdown workers if session should be interrupted As discussed in #66 --- xdist/dsession.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xdist/dsession.py b/xdist/dsession.py index ae71db4..46f76a9 100644 --- a/xdist/dsession.py +++ b/xdist/dsession.py @@ -535,6 +535,7 @@ class DSession: while not self.session_finished: self.loop_once() if self.shouldstop: + self.triggershutdown() raise Interrupted(str(self.shouldstop)) return True From 7d124ecf873392a16d0d27550d471f2305707125 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 4 Aug 2016 20:26:08 -0300 Subject: [PATCH 39/40] Replace "pytest_funcarg__" by @pytest.fixture --- testing/acceptance_test.py | 4 +++- testing/conftest.py | 3 ++- testing/test_remote.py | 3 ++- testing/test_slavemanage.py | 9 ++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index cccb8f7..21dfd71 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -411,7 +411,9 @@ def test_session_testscollected(testdir): def test_funcarg_teardown_failure(testdir): p = testdir.makepyfile(""" - def pytest_funcarg__myarg(request): + import pytest + @pytest.fixture + def myarg(request): def teardown(val): raise ValueError(val) return request.cached_setup(setup=lambda: 42, teardown=teardown, diff --git a/testing/conftest.py b/testing/conftest.py index 70537cd..c10007f 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -37,7 +37,8 @@ def pytest_addoption(parser): help=("add a global test environment, XSpec-syntax. ")) -def pytest_funcarg__specssh(request): +@pytest.fixture +def specssh(request): return getspecssh(request.config) diff --git a/testing/test_remote.py b/testing/test_remote.py index 8192b7d..207b926 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -59,7 +59,8 @@ class SlaveSetup: self.slp.sendcommand(name, **kwargs) -def pytest_funcarg__slave(request): +@pytest.fixture +def slave(request): return SlaveSetup(request) diff --git a/testing/test_slavemanage.py b/testing/test_slavemanage.py index 2861329..ac79449 100644 --- a/testing/test_slavemanage.py +++ b/testing/test_slavemanage.py @@ -8,7 +8,8 @@ from xdist.slavemanage import HostRSync, NodeManager pytest_plugins = "pytester" -def pytest_funcarg__hookrecorder(request, config): +@pytest.fixture +def hookrecorder(request, config): hookrecorder = HookRecorder(config.pluginmanager) if hasattr(hookrecorder, "start_recording"): hookrecorder.start_recording(newhooks) @@ -16,11 +17,13 @@ def pytest_funcarg__hookrecorder(request, config): return hookrecorder -def pytest_funcarg__config(testdir): +@pytest.fixture +def config(testdir): return testdir.parseconfig() -def pytest_funcarg__mysetup(tmpdir): +@pytest.fixture +def mysetup(tmpdir): class mysetup: source = tmpdir.mkdir("source") dest = tmpdir.mkdir("dest") From d498cb3e0fad7783866912637365951d77c932e1 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Thu, 4 Aug 2016 20:50:54 -0300 Subject: [PATCH 40/40] Add env variables to identify workers --- README.rst | 11 +++++++++++ testing/test_remote.py | 20 +++++++++++++++++++- xdist/remote.py | 2 ++ xdist/slavemanage.py | 3 ++- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 29fa5dd..7aab2ce 100644 --- a/README.rst +++ b/README.rst @@ -187,6 +187,8 @@ at once. The specifications strings use the `xspec syntax`_. Identifying the worker process during a test +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + If you need to determine the identity of a worker process in a test or fixture, you may use the ``worker_id`` fixture to do so: @@ -200,6 +202,15 @@ a test or fixture, you may use the ``worker_id`` fixture to do so: When ``xdist`` is disabled (running with ``-n0`` for example), then ``worker_id`` will return ``"master"``. +Additionally, worker processes have the following environment variables +defined: + +* ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``. +* ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session, + e.g., ``"4"`` when ``-n 4`` is given in the command-line. + +*New in version 1.15.* + Specifying test exec environments in an ini file +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/testing/test_remote.py b/testing/test_remote.py index 207b926..727aeca 100644 --- a/testing/test_remote.py +++ b/testing/test_remote.py @@ -40,7 +40,12 @@ class SlaveSetup: self.gateway = execnet.makegateway() self.config = config = self.testdir.parseconfigure() putevent = self.use_callback and self.events.put or None - self.slp = SlaveController(None, self.gateway, config, putevent) + + class DummyMananger: + specs = [0, 1] + + self.slp = SlaveController(DummyMananger, self.gateway, config, + putevent) self.request.addfinalizer(self.slp.ensure_teardown) self.slp.setup() @@ -178,6 +183,8 @@ class TestSlaveInteractor: ev = slave.popevent("slavefinished") assert 'slaveoutput' in ev.kwargs + @pytest.mark.skipif(pytest.__version__ >= '3.0', + reason='skip at module level illegal in pytest 3.0') def test_remote_collect_skip(self, slave): slave.testdir.makepyfile(""" import py @@ -255,3 +262,14 @@ class TestSlaveInteractor: ("pytest_pycollect_makeitem", "name == 'test_func'"), ("pytest_collectreport", "report.collector.fspath == bbb"), ]) + + +def test_remote_env_vars(testdir): + testdir.makepyfile(''' + import os + def test(): + assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1') + assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2' + ''') + result = testdir.runpytest('-n2', '--max-slave-restart=0') + assert result.ret == 0 diff --git a/xdist/remote.py b/xdist/remote.py index 226262a..40bbeac 100644 --- a/xdist/remote.py +++ b/xdist/remote.py @@ -148,6 +148,8 @@ if __name__ == '__channelexec__': os.environ['PYTHONPATH'] = ( importpath + os.pathsep + os.environ.get('PYTHONPATH', '')) + os.environ['PYTEST_XDIST_WORKER'] = slaveinput['slaveid'] + os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(slaveinput['slavecount']) # os.environ['PYTHONPATH'] = importpath import py config = remote_initconfig(option_dict, args) diff --git a/xdist/slavemanage.py b/xdist/slavemanage.py index 7632137..6112524 100644 --- a/xdist/slavemanage.py +++ b/xdist/slavemanage.py @@ -205,7 +205,8 @@ class SlaveController(object): self.putevent = putevent self.gateway = gateway self.config = config - self.slaveinput = {'slaveid': gateway.id} + self.slaveinput = {'slaveid': gateway.id, + 'slavecount': len(nodemanager.specs)} self._down = False self._shutdown_sent = False self.log = py.log.Producer("slavectl-%s" % gateway.id)