Merge pull request #122 from wronglink/custom_scheduler_notes

Added notes about new pytest_xdist_make_scheduler hook

.hgtags

@@ -1,21 +0,0 @@
-42c6503ee48fae9c4c96d406afb12bfc86f15803 1.0
-eca7ce17eabf296983c36812c8b8be901e7055a3 1.1
-56d8e5280be224a0ad3220a9deed55334710bd23 1.2
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-e6c4ce20db4bf65086ff55807a3c306cad7ca393 1.3
-eaf8b1cb7c312883598677231be5bbeea3b5c127 1.3
-a423748bf17ee778a37853225210257699cad9c1 1.4
-cd44a941c833c098e4899fe3d42a96703754d0d5 1.5
-4815040bdad8f182a5487f57a9da385483836e75 1.6
-20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
-20875fed94e7f3dff50bdf762df91153b15ceca6 1.7
-29c38e195526f5f0fdd651fb51f59d6efaaafbb0 1.7
-0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
-0d1c00018008433956aa7d93007bab6ea7de96e4 1.8
-1d27987c267577899350a25ba5828d55d87083ad 1.8
-5c5cb6d59e12e566fbb0217aea718dc31578bee1 1.9
-4406fc2a6427fadc021ed7e43e7aa5032b1ea91f 1.10
-220f6e46eb71a6212ccbe6b67b9e6edcf8ee4fa5 1.11
-39ef85dbc893cc63dede11601208098a667b58e9 1.12
-4e25f4c568be2d7cb4d1739638a9e66bbf28f588 v1.13
-67ff3aa4d294f75ade0ebf03267e5739e1bd9473 v1.13.1

.travis.yml

@@ -1,46 +1,47 @@
 sudo: false
 language: python
 python:
-    - '3.5'
+- '3.5'
-# command to install dependencies
+install: pip install -U tox setuptools_scm
-install: "pip install -U tox setuptools_scm"
-# # command to run tests
 env:
   matrix:
-  - TESTENV=flakes
+  # note: please use "tox --listenvs" to populate the build matrix
-  - TESTENV=readme
+  - TOXENV=py26-pytest27
-  # matrix was trimmed to skip
+  - TOXENV=py26-pytest28
-  # some builds that are unnecessary/perceived redundant:
+  - TOXENV=py26-pytest29
-  # py27 and py35 are tested with all pytest versions we declare to support;
+  - TOXENV=py26-pytest30
-  # py26 and py34 only with latest pytest versions.
+  - TOXENV=py27-pytest27
-  # pytest24,25,26 don't work on py35 due to changes in ast module
+  - TOXENV=py27-pytest28
-  - TESTENV=py26-pytest27
+  - TOXENV=py27-pytest29
-  - TESTENV=py26-pytest28
+  - TOXENV=py27-pytest30
-
+  - TOXENV=py34-pytest27
-  - TESTENV=py27-pytest24
+  - TOXENV=py34-pytest28
-  - TESTENV=py27-pytest25
+  - TOXENV=py34-pytest29
-  - TESTENV=py27-pytest26
+  - TOXENV=py34-pytest30
-  - TESTENV=py27-pytest27
+  - TOXENV=py35-pytest27
-  - TESTENV=py27-pytest28
+  - TOXENV=py35-pytest28
-
+  - TOXENV=py35-pytest29
-  - TESTENV=py34-pytest27
+  - TOXENV=py27-pytest28-pexpect
-  - TESTENV=py34-pytest28
+  - TOXENV=py35-pytest28-pexpect
-
+  - TOXENV=flakes
-  - TESTENV=py35-pytest27
+  - TOXENV=readme
-  - TESTENV=py35-pytest28
-
-  - TESTENV=py27-pytest28-pexpect
-  - TESTENV=py34-pytest28-pexpect
-  - TESTENV=pypy-pytest28
-
-script: tox --recreate -e $TESTENV
 
+script: tox
 notifications:
   irc:
     channels:
-      - "chat.freenode.net#pytest"
+    - chat.freenode.net#pytest
     on_success: change
     on_failure: change
     skip_join: true
   email:
-    - pytest-commit@python.org
+  - pytest-commit@python.org
+deploy:
+  provider: pypi
+  user: ronny
+  password:
+    secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
+  on:
+    tags: true
+    distributions: sdist bdist_wheel
+    repo: pytest-dev/pytest-xdist

CHANGELOG

@@ -1,3 +1,28 @@
+1.16.0
+------
+
+- ``pytest-xdist`` now requires pytest 2.7 or later.
+
+- Add ``worker_id`` attribute in the TestReport
+
+- new hook: ``pytest_xdist_make_scheduler(config, log)``, can return custom tests items
+  distribution logic implementation. You can take a look at built-in ``LoadScheduling``
+  and ``EachScheduling`` implementations. Note that required scheduler class public
+  API may change in next ``pytest-xdist`` versions.
+
+1.15.0
+------
+
+- new ``worker_id`` fixture, returns the id of the worker in a test or fixture.
+  Thanks Jared Hellman for the PR.
+
+- display progress during collection only when in a terminal, similar to pytest #1397 issue.
+  Thanks Bruno Oliveira for the PR.
+
+- fix internal error message when ``--maxfail`` is used (#62, #65).
+  Thanks Collin RM Stocks and Bryan A. Jones for reports and Bruno Oliveira for the PR.
+
+
 1.14
 ----
 

MANIFEST.in

@@ -4,4 +4,4 @@ include README.txt
 include setup.py
 include tox.ini
 graft testing
-prune .hg 
+prune .git

OVERVIEW.md

@@ -34,6 +34,8 @@ The execution flow is:
    tests and sends them one by one to each **worker** in a round robin
    fashion. The rest of the tests will be distributed later as **workers**
    finish tests (see below).
+
+1. Note that `pytest_xdist_make_scheduler` hook can be used to implement custom tests distribution logic.
    
 1. **workers** re-implement `pytest_runtestloop`: pytest's default implementation
    basically loops over all collected items in the `session` object and executes

README.rst

@@ -46,7 +46,7 @@ Install the plugin with::
 
     pip install pytest-xdist
 
-or use the package in develope/in-place mode with
+or use the package in develop/in-place mode with
 a checkout of the `pytest-xdist repository`_ ::
 
     python setup.py develop
@@ -185,17 +185,51 @@ at once.   The specifications strings use the `xspec syntax`_.
 
 .. _`execnet`: http://codespeak.net/execnet
 
+Identifying the worker process during a test
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+
+If you need to determine the identity of a worker process in
+a test or fixture, you may use the ``worker_id`` fixture to do so:
+
+.. code-block:: python
+
+    @pytest.fixture()
+    def user_account(worker_id):
+        """ use a different account in each xdist worker """
+        return "account_%s" % worker_id
+
+When ``xdist`` is disabled (running with ``-n0`` for example), then
+``worker_id`` will return ``"master"``.
+
+Additionally, worker processes have the following environment variables
+defined:
+
+* ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
+* ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session,
+  e.g., ``"4"`` when ``-n 4`` is given in the command-line.
+
+The information about the worker_id in a test is stored in the TestReport as
+well, under worker_id attribute.
+
+
+*New in version 1.15.*
+
 Specifying test exec environments in an ini file
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 pytest (since version 2.0) supports ini-style cofiguration.
 You can for example make running with three subprocesses
-your default like this::
+your default like this:
+
+.. code-block:: ini
 
     [pytest]
     addopts = -n3
 
-You can also add default environments like this::
+You can also add default environments like this:
+
+.. code-block:: ini
 
     [pytest]
     addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6
@@ -210,7 +244,9 @@ Specifying "rsync" dirs in an ini-file
 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 
 In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
-you may specify directories to include or to exclude in synchronisation::
+you may specify directories to include or to exclude in synchronisation:
+
+.. code-block:: ini
 
     [pytest]
     rsyncdirs = . mypkg helperpkg

appveyor.yml

@@ -1,17 +1,30 @@
 environment:
   matrix:
-    - PYTHON: "C:\\Python35"
+  # note: please use "tox --listenvs" to populate the build matrix
-      TESTENV: "py35"
+  - TOXENV: "py26-pytest27"
-
+  - TOXENV: "py26-pytest28"
-    - PYTHON: "C:\\Python27"
+  - TOXENV: "py26-pytest29"
-      TESTENV: "py27"
+  - TOXENV: "py26-pytest30"
+  - TOXENV: "py27-pytest27"
+  - TOXENV: "py27-pytest28"
+  - TOXENV: "py27-pytest29"
+  - TOXENV: "py27-pytest30"
+  - TOXENV: "py34-pytest27"
+  - TOXENV: "py34-pytest28"
+  - TOXENV: "py34-pytest29"
+  - TOXENV: "py34-pytest30"
+  - TOXENV: "py35-pytest27"
+  - TOXENV: "py35-pytest28"
+  - TOXENV: "py35-pytest29"
+  - TOXENV: "py27-pytest28-pexpect"
+  - TOXENV: "py35-pytest28-pexpect"
+  - TOXENV: "flakes"
+  - TOXENV: "readme"
 
 install:
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+  - C:\Python35\python -m pip install tox setuptools_scm
-  - pip install tox setuptools_scm"
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
-  # Build the compiled extension and run the project tests
+  - C:\Python35\python -m tox
-  - tox -e %TESTENV%-pytest27,%TESTENV%-pytest28,readme,flakes

setup.cfg

@@ -1,2 +1,5 @@
 [bdist_wheel]
 universal = 1
+
+[metadata]
+license_file = LICENSE

setup.py

@@ -20,10 +20,11 @@ setup(
         ],
     },
     zip_safe=False,
-    install_requires=['execnet>=1.1', 'pytest>=2.4.2', 'py>=1.4.22'],
+    install_requires=['execnet>=1.1', 'pytest>=2.7.0', 'py>=1.4.22'],
     setup_requires=['setuptools_scm'],
     classifiers=[
         'Development Status :: 5 - Production/Stable',
+        'Framework :: Pytest',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: MIT License',
         'Operating System :: POSIX',

testing/acceptance_test.py

@@ -62,7 +62,7 @@ class TestDistribution:
                 pass
         """)
         result = testdir.runpytest(p1, '--tx=popen', '--tx=popen')
-        assert result.ret == 1
+        assert result.ret in (1, 2)
         result.stdout.fnmatch_lines([
             "E   ImportError: *__import_of_missing_module*",
         ])
@@ -411,7 +411,9 @@ def test_session_testscollected(testdir):
 
 def test_funcarg_teardown_failure(testdir):
     p = testdir.makepyfile("""
-        def pytest_funcarg__myarg(request):
+        import pytest
+        @pytest.fixture
+        def myarg(request):
             def teardown(val):
                 raise ValueError(val)
             return request.cached_setup(setup=lambda: 42, teardown=teardown,
@@ -620,3 +622,68 @@ class TestNodeFailure:
             "*Slave*crashed while running*",
             "*1 failed*2 passed*",
         ])
+
+
+@pytest.mark.parametrize('n', [0, 2])
+def test_worker_id_fixture(testdir, n):
+    import glob
+    f = testdir.makepyfile("""
+        import pytest
+        @pytest.mark.parametrize("run_num", range(2))
+        def test_worker_id1(worker_id, run_num):
+            with open("worker_id%s.txt" % run_num, "w") as f:
+                f.write(worker_id)
+    """)
+    result = testdir.runpytest(f, "-n%d" % n)
+    result.stdout.fnmatch_lines('* 2 passed in *')
+    worker_ids = set()
+    for fname in glob.glob(str(testdir.tmpdir.join("*.txt"))):
+        with open(fname) as f:
+            worker_ids.add(f.read().strip())
+    if n == 0:
+        assert worker_ids == set(['master'])
+    else:
+        assert worker_ids == set(['gw0', 'gw1'])
+
+
+def test_color_yes_collection_on_non_atty(testdir, request):
+    """skip collect progress report when working on non-terminals.
+
+    Similar to pytest-dev/pytest#1397
+    """
+    tr = request.config.pluginmanager.getplugin("terminalreporter")
+    if not hasattr(tr, 'isatty'):
+        pytest.skip('only valid for newer pytest versions')
+    testdir.makepyfile("""
+        import pytest
+        @pytest.mark.parametrize('i', range(10))
+        def test_this(i):
+            assert 1
+    """)
+    args = ['--color=yes', '-n2']
+    result = testdir.runpytest(*args)
+    assert 'test session starts' in result.stdout.str()
+    assert '\x1b[1m' in result.stdout.str()
+    assert 'gw0 [10] / gw1 [10]' in result.stdout.str()
+    assert 'gw0 C / gw1 C' not in result.stdout.str()
+
+
+def test_internal_error_with_maxfail(testdir):
+    """
+    Internal error when using --maxfail option (#62, #65).
+    """
+    testdir.makepyfile("""
+        import pytest
+
+        @pytest.fixture(params=['1', '2'])
+        def crasher():
+            raise RuntimeError
+
+        def test_aaa0(crasher):
+            pass
+        def test_aaa1(crasher):
+            pass
+    """)
+    result = testdir.runpytest_subprocess('--maxfail=1', '-n1')
+    result.stdout.fnmatch_lines(['* 1 error in *'])
+    assert 'INTERNALERROR' not in result.stderr.str()

testing/conftest.py

@@ -37,7 +37,8 @@ def pytest_addoption(parser):
                      help=("add a global test environment, XSpec-syntax. "))
 
 
-def pytest_funcarg__specssh(request):
+@pytest.fixture
+def specssh(request):
     return getspecssh(request.config)
 
 

testing/test_dsession.py

@@ -49,80 +49,84 @@ def dumpqueue(queue):
 
 
 class TestEachScheduling:
-    def test_schedule_load_simple(self):
+    def test_schedule_load_simple(self, testdir):
         node1 = MockNode()
         node2 = MockNode()
-        sched = EachScheduling(2)
+        config = testdir.parseconfig("--tx=2*popen")
-        sched.addnode(node1)
+        sched = EachScheduling(config)
-        sched.addnode(node2)
+        sched.add_node(node1)
+        sched.add_node(node2)
         collection = ["a.py::test_1", ]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
+        sched.schedule()
-        assert sched.tests_finished()
+        assert sched.tests_finished
         assert node1.sent == ['ALL']
         assert node2.sent == ['ALL']
-        sched.remove_item(node1, 0)
+        sched.mark_test_complete(node1, 0)
-        assert sched.tests_finished()
+        assert sched.tests_finished
-        sched.remove_item(node2, 0)
+        sched.mark_test_complete(node2, 0)
-        assert sched.tests_finished()
+        assert sched.tests_finished
 
-    def test_schedule_remove_node(self):
+    def test_schedule_remove_node(self, testdir):
         node1 = MockNode()
-        sched = EachScheduling(1)
+        config = testdir.parseconfig("--tx=popen")
-        sched.addnode(node1)
+        sched = EachScheduling(config)
+        sched.add_node(node1)
         collection = ["a.py::test_1", ]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
-        sched.init_distribute()
+        sched.schedule()
-        assert sched.tests_finished()
+        assert sched.tests_finished
         crashitem = sched.remove_node(node1)
         assert crashitem
-        assert sched.tests_finished()
+        assert sched.tests_finished
-        assert not sched.hasnodes()
+        assert not sched.nodes
 
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self):
+    def test_schedule_load_simple(self, testdir):
-        sched = LoadScheduling(2)
+        config = testdir.parseconfig("--tx=2*popen")
-        sched.addnode(MockNode())
+        sched = LoadScheduling(config)
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2 = sched.nodes
         collection = ["a.py::test_1", "a.py::test_2"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
+        sched.schedule()
         assert not sched.pending
-        assert sched.tests_finished()
+        assert sched.tests_finished
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
         assert node1.sent == [0]
         assert node2.sent == [1]
-        sched.remove_item(node1, node1.sent[0])
+        sched.mark_test_complete(node1, node1.sent[0])
-        assert sched.tests_finished()
+        assert sched.tests_finished
 
-    def test_init_distribute_batch_size(self):
+    def test_schedule_batch_size(self, testdir):
-        sched = LoadScheduling(2)
+        config = testdir.parseconfig("--tx=2*popen")
-        sched.addnode(MockNode())
+        sched = LoadScheduling(config)
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2 = sched.nodes
         col = ["xyz"] * (6)
-        sched.addnode_collection(node1, col)
+        sched.add_node_collection(node1, col)
-        sched.addnode_collection(node2, col)
+        sched.add_node_collection(node2, col)
-        sched.init_distribute()
+        sched.schedule()
-        # assert not sched.tests_finished()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         assert sent1 == [0, 2]
@@ -131,25 +135,26 @@ class TestLoadScheduling:
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
         assert len(sched.pending) == 2
-        sched.remove_item(node1, 0)
+        sched.mark_test_complete(node1, 0)
         assert node1.sent == [0, 2, 4]
         assert sched.pending == [5]
         assert node2.sent == [1, 3]
-        sched.remove_item(node1, 2)
+        sched.mark_test_complete(node1, 2)
         assert node1.sent == [0, 2, 4, 5]
         assert not sched.pending
 
-    def test_init_distribute_fewer_tests_than_nodes(self):
+    def test_schedule_fewer_tests_than_nodes(self, testdir):
-        sched = LoadScheduling(2)
+        config = testdir.parseconfig("--tx=2*popen")
-        sched.addnode(MockNode())
+        sched = LoadScheduling(config)
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2, node3 = sched.nodes
         col = ["xyz"] * 2
-        sched.addnode_collection(node1, col)
+        sched.add_node_collection(node1, col)
-        sched.addnode_collection(node2, col)
+        sched.add_node_collection(node2, col)
-        sched.init_distribute()
+        sched.schedule()
-        # assert not sched.tests_finished()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         sent3 = node3.sent
@@ -158,17 +163,18 @@ class TestLoadScheduling:
         assert sent3 == []
         assert not sched.pending
 
-    def test_init_distribute_fewer_than_two_tests_per_node(self):
+    def test_schedule_fewer_than_two_tests_per_node(self, testdir):
-        sched = LoadScheduling(2)
+        config = testdir.parseconfig("--tx=2*popen")
-        sched.addnode(MockNode())
+        sched = LoadScheduling(config)
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
-        sched.addnode(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2, node3 = sched.nodes
         col = ["xyz"] * 5
-        sched.addnode_collection(node1, col)
+        sched.add_node_collection(node1, col)
-        sched.addnode_collection(node2, col)
+        sched.add_node_collection(node2, col)
-        sched.init_distribute()
+        sched.schedule()
-        # assert not sched.tests_finished()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         sent3 = node3.sent
@@ -177,14 +183,15 @@ class TestLoadScheduling:
         assert sent3 == [2]
         assert not sched.pending
 
-    def test_add_remove_node(self):
+    def test_add_remove_node(self, testdir):
         node = MockNode()
-        sched = LoadScheduling(1)
+        config = testdir.parseconfig("--tx=popen")
-        sched.addnode(node)
+        sched = LoadScheduling(config)
+        sched.add_node(node)
         collection = ["test_file.py::test_func"]
-        sched.addnode_collection(node, collection)
+        sched.add_node_collection(node, collection)
         assert sched.collection_is_completed
-        sched.init_distribute()
+        sched.schedule()
         assert not sched.pending
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
@@ -207,16 +214,16 @@ class TestLoadScheduling:
                 self.reports.append(report)
 
         collect_hook = CollectHook()
-        config = testdir.parseconfig()
+        config = testdir.parseconfig("--tx=2*popen")
         config.pluginmanager.register(collect_hook, "collect_hook")
         node1 = MockNode()
         node2 = MockNode()
-        sched = LoadScheduling(2, config=config)
+        sched = LoadScheduling(config)
-        sched.addnode(node1)
+        sched.add_node(node1)
-        sched.addnode(node2)
+        sched.add_node(node2)
-        sched.addnode_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node1, ["a.py::test_1"])
-        sched.addnode_collection(node2, ["a.py::test_2"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
-        sched.init_distribute()
+        sched.schedule()
         assert len(collect_hook.reports) == 1
         rep = collect_hook.reports[0]
         assert 'Different tests were collected between' in rep.longrepr

testing/test_newhooks.py

@@ -14,14 +14,19 @@ class TestHooks:
 
     def test_runtest_logreport(self, testdir):
         """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain a "node" attribute. (#8)
+        with xdist contain "node", "nodeid" and "worker_id" attributes. (#8)
         """
         testdir.makeconftest("""
             def pytest_runtest_logreport(report):
                 if hasattr(report, 'node'):
-                    slaveid = report.node.slaveinput['slaveid']
                     if report.when == "call":
-                        print("HOOK: %s %s" % (report.nodeid, slaveid))
+                        slaveid = report.node.slaveinput['slaveid']
+                        if slaveid != report.worker_id:
+                            print("HOOK: Worker id mismatch: %s %s"
+                                   % (slaveid, report.worker_id))
+                        else:
+                            print("HOOK: %s %s"
+                                   % (report.nodeid, report.worker_id))
         """)
         res = testdir.runpytest('-n1', '-s')
         res.stdout.fnmatch_lines([

testing/test_plugin.py

@@ -26,8 +26,12 @@ def test_dist_options(testdir):
 
 
 def test_auto_detect_cpus(testdir, monkeypatch):
-    import multiprocessing
+    import os
-    monkeypatch.setattr(multiprocessing, 'cpu_count', lambda: 99)
+    if hasattr(os, 'cpu_count'):
+        monkeypatch.setattr(os, 'cpu_count', lambda: 99)
+    else:
+        import multiprocessing
+        monkeypatch.setattr(multiprocessing, 'cpu_count', lambda: 99)
 
     config = testdir.parseconfigure("-n2")
     assert config.getoption('numprocesses') == 2

testing/test_remote.py

@@ -1,10 +1,12 @@
 import py
+import pytest
 from xdist.slavemanage import SlaveController, unserialize_report
 from xdist.remote import serialize_report
 import execnet
-queue = py.builtin._tryimport("queue", "Queue")
 import marshal
 
+queue = py.builtin._tryimport("queue", "Queue")
+
 WAIT_TIMEOUT = 10.0
 
 
@@ -38,7 +40,12 @@ class SlaveSetup:
         self.gateway = execnet.makegateway()
         self.config = config = self.testdir.parseconfigure()
         putevent = self.use_callback and self.events.put or None
-        self.slp = SlaveController(None, self.gateway, config, putevent)
+
+        class DummyMananger:
+            specs = [0, 1]
+
+        self.slp = SlaveController(DummyMananger, self.gateway, config,
+                                   putevent)
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
@@ -57,10 +64,12 @@ class SlaveSetup:
         self.slp.sendcommand(name, **kwargs)
 
 
-def pytest_funcarg__slave(request):
+@pytest.fixture
+def slave(request):
     return SlaveSetup(request)
 
 
+@pytest.mark.xfail(reason='#59')
 def test_remoteinitconfig(testdir):
     from xdist.remote import remote_initconfig
     config1 = testdir.parseconfig()
@@ -174,6 +183,8 @@ class TestSlaveInteractor:
         ev = slave.popevent("slavefinished")
         assert 'slaveoutput' in ev.kwargs
 
+    @pytest.mark.skipif(pytest.__version__ >= '3.0',
+                        reason='skip at module level illegal in pytest 3.0')
     def test_remote_collect_skip(self, slave):
         slave.testdir.makepyfile("""
             import py
@@ -251,3 +262,14 @@ class TestSlaveInteractor:
             ("pytest_pycollect_makeitem", "name == 'test_func'"),
             ("pytest_collectreport", "report.collector.fspath == bbb"),
         ])
+
+
+def test_remote_env_vars(testdir):
+    testdir.makepyfile('''
+        import os
+        def test():
+            assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
+            assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
+    ''')
+    result = testdir.runpytest('-n2', '--max-slave-restart=0')
+    assert result.ret == 0

testing/test_slavemanage.py

@@ -8,7 +8,8 @@ from xdist.slavemanage import HostRSync, NodeManager
 pytest_plugins = "pytester"
 
 
-def pytest_funcarg__hookrecorder(request, config):
+@pytest.fixture
+def hookrecorder(request, config):
     hookrecorder = HookRecorder(config.pluginmanager)
     if hasattr(hookrecorder, "start_recording"):
         hookrecorder.start_recording(newhooks)
@@ -16,11 +17,13 @@ def pytest_funcarg__hookrecorder(request, config):
     return hookrecorder
 
 
-def pytest_funcarg__config(testdir):
+@pytest.fixture
+def config(testdir):
     return testdir.parseconfig()
 
 
-def pytest_funcarg__mysetup(tmpdir):
+@pytest.fixture
+def mysetup(tmpdir):
     class mysetup:
         source = tmpdir.mkdir("source")
         dest = tmpdir.mkdir("dest")

tox.ini

@@ -1,20 +1,26 @@
 [tox]
+# if you change the envlist, please update .travis.yml file as well
 envlist=
-  py{26,27,34,35}-pytest2{4,5,6,7,8},py{27,34}-pytest28-pexpect,flakes,readme
+  py{26,27,34,35}-pytest{27,28,29,30}
+  py{27,35}-pytest28-pexpect
+  flakes
+  readme  
 
 
 [testenv]
 changedir=testing
 passenv = USER USERNAME
 deps =
-    pycmd
+  pycmd
-    setuptools_scm # to avoid .eggs
+  # to avoid .eggs
-    pytest24: pytest~=2.4.0
+  setuptools_scm
-    pytest25: pytest~=2.5.0
+  pytest27: pytest~=2.7.2
-    pytest26: pytest~=2.6.1
+  pytest28: pytest~=2.8.7
-    pytest27: pytest~=2.7.2
+  pytest29: pytest~=2.9.2
-    pytest28: pytest~=2.8.3
+  pytest30: pytest~=3.0.5
-    pexpect: pexpect
+  pexpect: pexpect
+platform=
+  pexpect: linux|darwin    
 commands=
   # always clean to avoid code unmarshal mismatch on old python/pytest
   py.cleanup -aq

xdist/__init__.py

@@ -1,2 +1,3 @@
-__all__ = ['__version__']
 from xdist._version import version as __version__
+
+__all__ = ['__version__']

xdist/dsession.py

@@ -4,7 +4,7 @@ from _pytest.runner import CollectReport
 
 import pytest
 import py
-from xdist.slavemanage import NodeManager
+from xdist.slavemanage import NodeManager, parse_spec_config
 
 
 queue = py.builtin._tryimport('queue', 'Queue')
@@ -24,8 +24,9 @@ class EachScheduling:
     assigned the remaining items from the removed node.
     """
 
-    def __init__(self, numnodes, log=None):
+    def __init__(self, config, log=None):
-        self.numnodes = numnodes
+        self.config = config
+        self.numnodes = len(parse_spec_config(config))
         self.node2collection = {}
         self.node2pending = {}
         self._started = []
@@ -41,10 +42,19 @@ class EachScheduling:
         """A list of all nodes in the scheduler."""
         return list(self.node2pending.keys())
 
-    def hasnodes(self):
+    @property
-        return bool(self.node2pending)
+    def tests_finished(self):
+        if not self.collection_is_completed:
+            return False
+        if self._removed2pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
 
-    def haspending(self):
+    @property
+    def has_pending(self):
         """Return True if there are pending test items
 
         This indicates that collection has finished and nodes are
@@ -56,21 +66,11 @@ class EachScheduling:
                 return True
         return False
 
-    def addnode(self, node):
+    def add_node(self, node):
         assert node not in self.node2pending
         self.node2pending[node] = []
 
-    def tests_finished(self):
+    def add_node_collection(self, node, collection):
-        if not self.collection_is_completed:
-            return False
-        if self._removed2pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
         """Add the collected test items from a node
 
         Collection is complete once all nodes have submitted their
@@ -78,7 +78,7 @@ class EachScheduling:
         list.  When the collection is already completed this
         submission is from a node which was restarted to replace a
         dead node.  In this case we already assign the pending items
-        here.  In either case ``.init_distribute()`` will instruct the
+        here.  In either case ``.schedule()`` will instruct the
         node to start running the required tests.
         """
         assert node in self.node2pending
@@ -102,11 +102,11 @@ class EachScheduling:
                     self.node2pending[node] = pending
                     break
 
-    def remove_item(self, node, item_index, duration=0):
+    def mark_test_complete(self, node, item_index, duration=0):
         self.node2pending[node].remove(item_index)
 
     def remove_node(self, node):
-        # KeyError if we didn't get an addnode() yet
+        # KeyError if we didn't get an add_node() yet
         pending = self.node2pending.pop(node)
         if not pending:
             return
@@ -115,12 +115,12 @@ class EachScheduling:
             self._removed2pending[node] = pending
         return crashitem
 
-    def init_distribute(self):
+    def schedule(self):
         """Schedule the test items on the nodes
 
         If the node's pending list is empty it is a new node which
         needs to run all the tests.  If the pending list is already
-        populated (by ``.addnode_collection()``) then it replaces a
+        populated (by ``.add_node_collection()``) then it replaces a
         dead node and we only need to run those tests.
         """
         assert self.collection_is_completed
@@ -143,7 +143,7 @@ class LoadScheduling:
     when all collections are received it is verified they are
     identical collections.  Then the collection gets divided up in
     chunks and chunks get submitted to nodes.  Whenever a node finishes
-    an item, it calls ``.remove_item()`` which will trigger the
+    an item, it calls ``.mark_test_complete()`` which will trigger the
     scheduler to assign more tests if the number of pending tests for
     the node falls below a low-watermark.
 
@@ -170,7 +170,7 @@ class LoadScheduling:
 
     :collection: The one collection once it is validated to be
        identical between all the nodes.  It is initialised to None
-       until ``.init_distribute()`` is called.
+       until ``.schedule()`` is called.
 
     :pending: List of indices of globally pending tests.  These are
        tests which have not yet been allocated to a chunk for a node
@@ -181,8 +181,8 @@ class LoadScheduling:
     :config: Config object, used for handling hooks.
     """
 
-    def __init__(self, numnodes, log=None, config=None):
+    def __init__(self, config, log=None):
-        self.numnodes = numnodes
+        self.numnodes = len(parse_spec_config(config))
         self.node2collection = {}
         self.node2pending = {}
         self.pending = []
@@ -208,7 +208,20 @@ class LoadScheduling:
         """
         return len(self.node2collection) >= self.numnodes
 
-    def haspending(self):
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
         """Return True if there are pending test items
 
         This indicates that collection has finished and nodes are
@@ -222,11 +235,7 @@ class LoadScheduling:
                 return True
         return False
 
-    def hasnodes(self):
+    def add_node(self, node):
-        """Return True if nodes exist in the scheduler."""
-        return bool(self.node2pending)
-
-    def addnode(self, node):
         """Add a new node to the scheduler.
 
         From now on the node will be allocated chunks of tests to
@@ -238,18 +247,7 @@ class LoadScheduling:
         assert node not in self.node2pending
         self.node2pending[node] = []
 
-    def tests_finished(self):
+    def add_node_collection(self, node, collection):
-        """Return True if all tests have been executed by the nodes."""
-        if not self.collection_is_completed:
-            return False
-        if self.pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
         """Add the collected test items from a node
 
         The collection is stored in the ``.node2collection`` map.
@@ -258,7 +256,7 @@ class LoadScheduling:
         assert node in self.node2pending
         if self.collection_is_completed:
             # A new node has been added later, perhaps an original one died.
-            # .init_distribute() should have
+            # .schedule() should have
             # been called by now
             assert self.collection
             if collection != self.collection:
@@ -271,7 +269,7 @@ class LoadScheduling:
                 return
         self.node2collection[node] = list(collection)
 
-    def remove_item(self, node, item_index, duration=0):
+    def mark_test_complete(self, node, item_index, duration=0):
         """Mark test item as completed by node
 
         The duration it took to execute the item is used as a hint to
@@ -335,7 +333,7 @@ class LoadScheduling:
             self.check_schedule(node)
         return crashitem
 
-    def init_distribute(self):
+    def schedule(self):
         """Initiate distribution of the test collection
 
         Initiate scheduling of the items across the nodes.  If this
@@ -345,8 +343,6 @@ class LoadScheduling:
 
         This is called by the ``DSession.slave_collectionfinish`` hook
         if ``.collection_is_completed`` is True.
-
-        XXX Perhaps this method should have been called ".schedule()".
         """
         assert self.collection_is_completed
 
@@ -466,6 +462,8 @@ class DSession:
         self.log = py.log.Producer("dsession")
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
+        self.nodemanager = None
+        self.sched = None
         self.shuttingdown = False
         self.countfailures = 0
         self.maxfail = config.getvalue("maxfail")
@@ -521,20 +519,26 @@ class DSession:
         # prohibit collection of test items in master process
         return True
 
-    def pytest_runtestloop(self):
+    @pytest.mark.trylast
-        numnodes = len(self.nodemanager.specs)
+    def pytest_xdist_make_scheduler(self, config, log):
-        dist = self.config.getvalue("dist")
+        dist = config.getvalue("dist")
         if dist == "load":
-            self.sched = LoadScheduling(numnodes, log=self.log,
+            return LoadScheduling(config, log)
-                                        config=self.config)
         elif dist == "each":
-            self.sched = EachScheduling(numnodes, log=self.log)
+            return EachScheduling(config, log)
-        else:
+
-            assert 0, dist
+    def pytest_runtestloop(self):
+        self.sched = self.config.hook.pytest_xdist_make_scheduler(
+            config=self.config,
+            log=self.log
+        )
+        assert self.sched is not None
+
         self.shouldstop = False
         while not self.session_finished:
             self.loop_once()
             if self.shouldstop:
+                self.triggershutdown()
                 raise Interrupted(str(self.shouldstop))
         return True
 
@@ -552,7 +556,7 @@ class DSession:
         call = getattr(self, method)
         self.log("calling method", method, kwargs)
         call(**kwargs)
-        if self.sched.tests_finished():
+        if self.sched.tests_finished:
             self.triggershutdown()
 
     #
@@ -572,7 +576,7 @@ class DSession:
         if self.shuttingdown:
             node.shutdown()
         else:
-            self.sched.addnode(node)
+            self.sched.add_node(node)
 
     def slave_slavefinished(self, node):
         """Emitted when node executes its pytest_sessionfinish hook.
@@ -634,16 +638,16 @@ class DSession:
         # tell session which items were effectively collected otherwise
         # the master node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
-        self.sched.addnode_collection(node, ids)
+        self.sched.add_node_collection(node, ids)
         if self.terminal:
             self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids)))
         if self.sched.collection_is_completed:
-            if self.terminal and not self.sched.haspending():
+            if self.terminal and not self.sched.has_pending:
                 self.trdist.ensure_show_status()
                 self.terminal.write_line("")
                 self.terminal.write_line("scheduling tests via %s" % (
                     self.sched.__class__.__name__))
-            self.sched.init_distribute()
+            self.sched.schedule()
 
     def slave_logstart(self, node, nodeid, location):
         """Emitted when a node calls the pytest_runtest_logstart hook."""
@@ -657,7 +661,7 @@ class DSession:
         the item from the pending list in the scheduler.
         """
         if rep.when == "call" or (rep.when == "setup" and not rep.passed):
-            self.sched.remove_item(node, rep.item_index, rep.duration)
+            self.sched.mark_test_complete(node, rep.item_index, rep.duration)
         # self.report_line("testreport %s: %s" %(rep.id, rep.status))
         rep.node = node
         self.config.hook.pytest_runtest_logreport(report=rep)
@@ -722,17 +726,18 @@ class TerminalDistReporter:
         self.tr = config.pluginmanager.getplugin("terminalreporter")
         self._status = {}
         self._lastlen = 0
+        self._isatty = getattr(self.tr, 'isatty', self.tr.hasmarkup)
 
     def write_line(self, msg):
         self.tr.write_line(msg)
 
     def ensure_show_status(self):
-        if not self.tr.hasmarkup:
+        if not self._isatty:
             self.write_line(self.getstatus())
 
     def setstatus(self, spec, status, show=True):
         self._status[spec.id] = status
-        if show and self.tr.hasmarkup:
+        if show and self._isatty:
             self.rewrite(self.getstatus())
 
     def getstatus(self):

xdist/newhooks.py

@@ -11,6 +11,7 @@ must be taken in plugins in case ``xdist`` is not installed. Please see:
 
     http://pytest.org/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
 """
+import pytest
 
 
 def pytest_xdist_setupnodes(config, specs):
@@ -44,3 +45,8 @@ def pytest_testnodedown(node, error):
 def pytest_xdist_node_collection_finished(node, ids):
     """called by the master node when a node finishes collecting.
     """
+
+
+@pytest.mark.firstresult
+def pytest_xdist_make_scheduler(config, log):
+    """ return a node scheduler implementation """

xdist/plugin.py

@@ -4,8 +4,15 @@ import pytest
 
 def parse_numprocesses(s):
     if s == 'auto':
-        import multiprocessing
+        try:
-        return multiprocessing.cpu_count()
+            from os import cpu_count
+        except ImportError:
+            from multiprocessing import cpu_count
+        try:
+            n = cpu_count()
+        except NotImplementedError:
+            return 1
+        return n if n else 1
     else:
         return int(s)
 
@@ -25,7 +32,7 @@ def pytest_addoption(parser):
     group._addoption(
         '--dist', metavar="distmode",
         action="store", choices=['load', 'each', 'no'],
-        type="choice", dest="dist", default="no",
+        dest="dist", default="no",
         help=("set mode for distributing tests to exec environments.\n\n"
               "each: send each test to each available environment.\n\n"
               "load: send each test to available environment.\n\n"
@@ -99,3 +106,15 @@ def pytest_cmdline_main(config):
             if usepdb:
                 raise pytest.UsageError(
                     "--pdb incompatible with distributing tests.")
+
+# -------------------------------------------------------------------------
+# fixtures
+# -------------------------------------------------------------------------
+
+
+@pytest.fixture(scope="session")
+def worker_id(request):
+    if hasattr(request.config, 'slaveinput'):
+        return request.config.slaveinput['slaveid']
+    else:
+        return 'master'

xdist/remote.py

@@ -46,7 +46,10 @@ class SlaveInteractor:
         self.log("entering main loop")
         torun = []
         while 1:
-            name, kwargs = self.channel.receive()
+            try:
+                name, kwargs = self.channel.receive()
+            except EOFError:
+                return True
             self.log("received command", name, kwargs)
             if name == "runtests":
                 torun.extend(kwargs['indices'])
@@ -85,6 +88,7 @@ class SlaveInteractor:
     def pytest_runtest_logreport(self, report):
         data = serialize_report(report)
         data["item_index"] = self.item_index
+        data["worker_id"] = self.slaveid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
@@ -145,6 +149,8 @@ if __name__ == '__channelexec__':
     os.environ['PYTHONPATH'] = (
         importpath + os.pathsep +
         os.environ.get('PYTHONPATH', ''))
+    os.environ['PYTEST_XDIST_WORKER'] = slaveinput['slaveid']
+    os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(slaveinput['slavecount'])
     # os.environ['PYTHONPATH'] = importpath
     import py
     config = remote_initconfig(option_dict, args)

xdist/slavemanage.py

@@ -10,6 +10,22 @@ import xdist.remote
 from _pytest import runner  # XXX load dynamically
 
 
+def parse_spec_config(config):
+    xspeclist = []
+    for xspec in config.getvalue("tx"):
+        i = xspec.find("*")
+        try:
+            num = int(xspec[:i])
+        except ValueError:
+            xspeclist.append(xspec)
+        else:
+            xspeclist.extend([xspec[i + 1:]] * num)
+    if not xspeclist:
+        raise pytest.UsageError(
+            "MISSING test execution (tx) nodes: please specify --tx")
+    return xspeclist
+
+
 class NodeManager(object):
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
@@ -62,19 +78,7 @@ class NodeManager(object):
         self.group.terminate(self.EXIT_TIMEOUT)
 
     def _getxspecs(self):
-        xspeclist = []
+        return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
-        for xspec in self.config.getvalue("tx"):
-            i = xspec.find("*")
-            try:
-                num = int(xspec[:i])
-            except ValueError:
-                xspeclist.append(xspec)
-            else:
-                xspeclist.extend([xspec[i+1:]] * num)
-        if not xspeclist:
-            raise pytest.UsageError(
-                "MISSING test execution (tx) nodes: please specify --tx")
-        return [execnet.XSpec(x) for x in xspeclist]
 
     def _getrsyncdirs(self):
         for spec in self.specs:
@@ -205,7 +209,8 @@ class SlaveController(object):
         self.putevent = putevent
         self.gateway = gateway
         self.config = config
-        self.slaveinput = {'slaveid': gateway.id}
+        self.slaveinput = {'slaveid': gateway.id,
+                           'slavecount': len(nodemanager.specs)}
         self._down = False
         self._shutdown_sent = False
         self.log = py.log.Producer("slavectl-%s" % gateway.id)
@@ -291,7 +296,7 @@ class SlaveController(object):
                     self._down = True
                 return
             eventname, kwargs = eventcall
-            if eventname in ("collectionstart"):
+            if eventname in ("collectionstart",):
                 self.log("ignoring %s(%s)" % (eventname, kwargs))
             elif eventname == "slaveready":
                 self.notify_inproc(eventname, node=self, **kwargs)