Prepare release 1.22.4

This makes using multiple paragraphs per entry possible

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,8 +4,8 @@ Here's a quick checklist that should be present in PRs:
 
 - [ ] Make sure to include reasonable tests for your change if necessary
 
-- [ ] Add a *news* file into the `changelog` folder, following these guidelines:
+- [ ] We use [towncrier](https://pypi.python.org/pypi/towncrier) for changelog management, so please add a *news* file into the `changelog` folder following these guidelines:
-  * Name it `$issue_id.$type` for example `588.bug`
+  * Name it `$issue_id.$type` for example `588.bugfix`;
   * If you don't have an issue_id change it to the PR id after creating it
   * Ensure type is one of `removal`, `feature`, `bugfix`, `vendor`, `doc` or `trivial`
   * Make sure to use full sentences with correct case and punctuation, for example:

.gitignore

@@ -22,6 +22,7 @@ dist/
 include/
 lib/
 bin/
+env/
 xdist/_version.py*
 pytest_xdist.egg-info
 issue/
@@ -29,4 +30,6 @@ issue/
 pytestdebug.log
 .tox/
 .cache/
+.pytest_cache/
 .eggs/
+.idea/

.travis.yml

@@ -1,37 +1,63 @@
 sudo: false
 language: python
-python:
-- '2.6'
-- '2.7'
-- '3.4'
-- '3.5'
-- '3.6'
-install: pip install tox-travis setuptools_scm
-matrix:
-  include:
-  # note: please use "tox --listenvs" to populate the build matrix
-  - python: "3.5"
-    env: TOXENV=flakes
-  - python: "3.5"
-    env: TOXENV=readme
 
-script: tox
 notifications:
   irc:
     channels:
-    - chat.freenode.net#pytest
+    - 'chat.freenode.net#pytest'
     on_success: change
     on_failure: change
     skip_join: true
   email:
   - pytest-commit@python.org
-deploy:
+
-  provider: pypi
+python:
-  user: ronny
+- '2.7'
-  skip_upload_docs: true
+- '3.4'
-  password:
+- '3.5'
-    secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
+- '3.6'
-  on:
+env:
-    tags: true
+- TOXENV=py-pytest34
-    distributions: sdist bdist_wheel
+- TOXENV=py-pytest35
-    repo: pytest-dev/pytest-xdist
+- TOXENV=py-pytest36
+
+install: pip install tox setuptools_scm
+script: tox
+
+stages:
+- test
+- name: deploy
+  if: repo = pytest-dev/pytest-xdist AND tag IS present
+
+jobs:
+  include:
+    - stage: test
+    # python x env above are already included into this stage
+    - python: "2.7"
+      env: TOXENV=py27-pytestmaster
+    - python: "2.7"
+      env: TOXENV=py27-pytestfeatures
+    - python: "3.6"
+      env: TOXENV=py36-pytestmaster
+    - python: "3.6"
+      env: TOXENV=py36-pytestfeatures
+    - python: "3.6"
+      env: TOXENV=flakes
+    - python: "3.6"
+      env: TOXENV=readme
+
+    - stage: deploy
+      python: '3.6'
+      env:
+      install: pip install -U setuptools setuptools_scm
+      script: skip
+      deploy:
+        provider: pypi
+        user: ronny
+        distributions: sdist bdist_wheel
+        skip_upload_docs: true
+        password:
+          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
+        on:
+          tags: true
+          repo: pytest-dev/pytest-xdist

CHANGELOG.rst

@@ -1,3 +1,190 @@
+pytest-xdist 1.22.4 (2018-07-27)
+================================
+
+Bug Fixes
+---------
+
+- `#305 <https://github.com/pytest-dev/pytest-xdist/issues/305>`_: Remove last references to obsolete ``py.code``.
+
+  Remove some unnecessary references to ``py.builtin``.
+
+- `#316 <https://github.com/pytest-dev/pytest-xdist/issues/316>`_: Workaround cpu detection on Travis CI.
+
+
+pytest-xdist 1.22.3 (2018-07-23)
+================================
+
+Bug Fixes
+---------
+
+- Fix issue of virtualized or containerized environments not reporting the number of CPUs correctly. (`#9 <https://github.com/pytest-dev/pytest-xdist/issues/9>`_)
+
+
+Trivial Changes
+---------------
+
+- Make all classes subclass from ``object`` and fix ``super()`` call in ``LoadFileScheduling``; (`#297 <https://github.com/pytest-dev/pytest-xdist/issues/297>`_)
+
+
+pytest-xdist 1.22.2 (2018-02-26)
+================================
+
+Bug Fixes
+---------
+
+- Add backward compatibility for ``slaveoutput`` attribute to
+  ``WorkerController`` instances. (`#285
+  <https://github.com/pytest-dev/pytest-xdist/issues/285>`_)
+
+
+pytest-xdist 1.22.1 (2018-02-19)
+================================
+
+Bug Fixes
+---------
+
+- Fix issue when using ``loadscope`` or ``loadfile`` where tests would fail to
+  start if the first scope had only one test. (`#257
+  <https://github.com/pytest-dev/pytest-xdist/issues/257>`_)
+
+
+Trivial Changes
+---------------
+
+- Change terminology used by ``pytest-xdist`` to *master* and *worker* in
+  arguments and messages (for example ``--max-worker-reset``). (`#234
+  <https://github.com/pytest-dev/pytest-xdist/issues/234>`_)
+
+
+pytest-xdist 1.22.0 (2018-01-11)
+================================
+
+Features
+--------
+
+- Add support for the ``pytest_runtest_logfinish`` hook which will be released
+  in pytest 3.4. (`#266
+  <https://github.com/pytest-dev/pytest-xdist/issues/266>`_)
+
+
+pytest-xdist 1.21.0 (2017-12-22)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- Drop support for EOL Python 2.6. (`#259
+  <https://github.com/pytest-dev/pytest-xdist/issues/259>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadfile`` option which load-distributes test to workers grouped
+  by the file the tests live in. (`#242
+  <https://github.com/pytest-dev/pytest-xdist/issues/242>`_)
+
+
+Bug Fixes
+---------
+
+- Fix accidental mutation of test report during serialization causing longrepr
+  string-ification to break. (`#241
+  <https://github.com/pytest-dev/pytest-xdist/issues/241>`_)
+
+
+pytest-xdist 1.20.1 (2017-10-05)
+================================
+
+Bug Fixes
+---------
+
+- Fix hang when all worker nodes crash and restart limit is reached (`#45
+  <https://github.com/pytest-dev/pytest-xdist/issues/45>`_)
+
+- Fix issue where the -n option would still run distributed tests when pytest
+  was run with the --collect-only option (`#5
+  <https://github.com/pytest-dev/pytest-xdist/issues/5>`_)
+
+
+pytest-xdist 1.20.0 (2017-08-17)
+================================
+
+Features
+--------
+
+- ``xdist`` now supports tests to log results multiple times, improving
+  integration with plugins which require it like `pytest-rerunfailures
+  <https://github.com/gocept/pytest-rerunfailures>`_ and `flaky
+  <https://pypi.python.org/pypi/flaky>`_. (`#206 <https://github.com/pytest-
+  dev/pytest-xdist/issues/206>`_)
+
+
+Bug Fixes
+---------
+
+- Fix issue where tests were being incorrectly identified if a worker crashed
+  during the ``teardown`` stage of the test. (`#124 <https://github.com/pytest-
+  dev/pytest-xdist/issues/124>`_)
+
+
+pytest-xdist 1.19.1 (2017-08-10)
+================================
+
+Bug Fixes
+---------
+
+- Fix crash when transferring internal pytest warnings from workers to the
+  master node. (`#214 <https://github.com/pytest-dev/pytest-
+  xdist/issues/214>`_)
+
+
+pytest-xdist 1.19.0 (2017-08-09)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- ``--boxed`` functionality has been moved to a separate plugin, `pytest-forked
+  <https://github.com/pytest-dev/pytest-forked>`_. This release now depends on
+  `` pytest-forked`` and provides ``--boxed`` as a backward compatibility
+  option. (`#1 <https://github.com/pytest-dev/pytest-xdist/issues/1>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadscope`` option: sends group of related tests to the same
+  worker. Tests are grouped by module for test functions and by class for test
+  methods. See ``README.rst`` for more information. (`#191 <https://github.com
+  /pytest-dev/pytest-xdist/issues/191>`_)
+
+- Warnings are now properly transferred from workers to the master node. (`#92
+  <https://github.com/pytest-dev/pytest-xdist/issues/92>`_)
+
+
+Bug Fixes
+---------
+
+- Fix serialization of native tracebacks (``--tb=native``). (`#196
+  <https://github.com/pytest-dev/pytest-xdist/issues/196>`_)
+
+
+pytest-xdist 1.18.2 (2017-07-28)
+================================
+
+Bug Fixes
+---------
+
+- Removal of unnecessary dependency on incorrect version of py. (`#105
+  <https://github.com/pytest-dev/pytest-xdist/issues/105>`_)
+
+- Fix bug in internal event-loop error handler in the master node. This bug
+  would shadow the original errors making extremely hard/impossible for users
+  to diagnose the problem properly. (`#175 <https://github.com/pytest-
+  dev/pytest-xdist/issues/175>`_)
+
+
 pytest-xdist 1.18.1 (2017-07-05)
 ================================
 
@@ -172,7 +359,7 @@ Improved Documentation
   in master. Thanks Aron Curzon.
 
 - fix pytest issue419 by sending/receiving indices into the test
-  collection instead of node ids (which are not neccessarily unique
+  collection instead of node ids (which are not necessarily unique
   for functions parametrized with duplicate values)
 
 - send multiple "to test" indices in one network message to a slave

HOWTORELEASE.rst

@@ -24,15 +24,13 @@ To publish a new release ``X.Y.Z``, the steps are as follows:
 
 #. Create a new branch named ``release-X.Y.Z`` from the latest ``master``.
 
-#. Install ``pytest-xdist`` and dev requirements in a virtualenv::
+#. Install ``tox`` in a virtualenv::
 
-    $ pip install -e . -r dev-requirements.txt
+    $ pip install tox
 
-#. Update ``CHANGELOG.rst`` file by running::
+#. Update the necessary files with::
 
-    $ towncrier --version X.Y.Z
+    $ tox -e release -- X.Y.Z
-
-   It might ask for confirmation to remove news fragments; answer yes.
 
 #. Commit and push the branch for review.
 

ISSUES.txt

@@ -19,7 +19,7 @@ tag: feature
 
 allow to run xdist own tests using its own mechanism.
 currently this doesn't work because the remote side
-has no py.test plugin.   How to configure/do
+has no pytest plugin.   How to configure/do
 register "xdist.plugin" on the remote side?
 
 see to avoid any "from _pytest" internal imports

README.rst

@@ -1,95 +1,110 @@
 
 
 .. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
+    :alt: PyPI version
     :target: https://pypi.python.org/pypi/pytest-xdist
 
-.. image:: https://anaconda.org/conda-forge/pytest-xdist/badges/version.svg
+.. image:: https://img.shields.io/conda/vn/conda-forge/pytest-xdist.svg
     :target: https://anaconda.org/conda-forge/pytest-xdist
 
+.. image:: https://img.shields.io/pypi/pyversions/pytest-xdist.svg
+    :alt: Python versions
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
 .. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
+    :alt: Travis CI build status
     :target: https://travis-ci.org/pytest-dev/pytest-xdist
 
 .. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
+    :alt: AppVeyor build status
     :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
 
 xdist: pytest distributed testing plugin
-=========================================
+========================================
 
-The `pytest-xdist`_ plugin extends py.test with some unique
+The `pytest-xdist`_ plugin extends pytest with some unique
 test execution modes:
 
 * test run parallelization_: if you have multiple CPUs or hosts you can use
   those for a combined test run.  This allows to speed up
   development or to use special resources of `remote machines`_.
 
-* ``--boxed``: (not available on Windows) run each test in a boxed_
-  subprocess to survive ``SEGFAULTS`` or otherwise dying processes
 
 * ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
-  py.test waits until a file in your project changes and then re-runs
+  pytest waits until a file in your project changes and then re-runs
   the previously failing tests.  This is repeated until all tests pass
   after which again a full run is performed.
 
 * `Multi-Platform`_ coverage: you can specify different Python interpreters
   or different platforms and run tests in parallel on all of them.
 
-Before running tests remotely, ``py.test`` efficiently "rsyncs" your
+Before running tests remotely, ``pytest`` efficiently "rsyncs" your
 program source code to the remote place.  All test results
 are reported back and displayed to your local terminal.
 You may specify different Python versions and interpreters.
 
+If you would like to know how pytest-xdist works under the covers, checkout
+`OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
+
 
 Installation
------------------------
+------------
 
 Install the plugin with::
 
-    easy_install pytest-xdist
-
-    # or
-
     pip install pytest-xdist
 
 or use the package in develop/in-place mode with
 a checkout of the `pytest-xdist repository`_ ::
 
-    python setup.py develop
+    pip install --editable .
-
-Usage examples
----------------------
 
 .. _parallelization:
 
 Speed up test runs by sending tests to multiple CPUs
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+----------------------------------------------------
 
 To send tests to multiple CPUs, type::
 
-    py.test -n NUM
+    pytest -n NUM
 
 Especially for longer running tests or tests requiring
-a lot of IO this can lead to considerable speed ups. This option can
+a lot of I/O this can lead to considerable speed ups. This option can
 also be set to ``auto`` for automatic detection of the number of CPUs.
 
 If a test crashes the interpreter, pytest-xdist will automatically restart
-that slave and report the failure as usual. You can use the
+that worker and report the failure as usual. You can use the
-``--max-slave-restart`` option to limit the number of slaves that can
+``--max-worker-restart`` option to limit the number of workers that can
-be restarted, or disable restarting altogether using ``--max-slave-restart=0``.
+be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+
+By default, the ``-n`` option will send pending tests to any worker that is available, without
+any guaranteed order, but you can control this with these options:
+
+* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
+  by **class** for *test methods*, then each group will be sent to an available worker,
+  guaranteeing that all tests in a group run in the same process. This can be useful if you have
+  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
+  with grouping by class takes priority over grouping by module.
+  This feature was added in version ``1.19``.
+
+* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
+  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
+  in version ``1.21``.
 
 
 Running tests in a Python subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------
 
-To instantiate a python2.5 sub process and send tests to it, you may type::
+To instantiate a python3.5 subprocess and send tests to it, you may type::
 
-    py.test -d --tx popen//python=python2.5
+    pytest -d --tx popen//python=python3.5
 
-This will start a subprocess which is run with the "python2.5"
+This will start a subprocess which is run with the ``python3.5``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python2.5
+    --tx 3*popen//python=python3.5
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -97,38 +112,26 @@ will be load-balanced across these three processes.
 .. _boxed:
 
 Running tests in a boxed subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+-----------------------------------
-
-If you have tests involving C or C++ libraries you might have to deal
-with tests crashing the process.  For this case you may use the boxing
-options::
-
-    py.test --boxed
-
-which will run each test in a subprocess and will report if a test
-crashed the process.  You can also combine this option with
-running multiple processes to speed up the test run and use your CPU cores::
-
-    py.test -n3 --boxed
-
-this would run 3 testing subprocesses in parallel which each
-create new boxed subprocesses for each test.
 
+This functionality has been moved to the
+`pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ plugin, but the ``--boxed`` option
+is still kept for backward compatibility.
 
 .. _`remote machines`:
 
 Sending tests to remote SSH accounts
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------
 
 Suppose you have a package ``mypkg`` which contains some
 tests that you can successfully run locally. And you
 have a ssh-reachable machine ``myhost``.  Then
 you can ad-hoc distribute your tests by typing::
 
-    py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
+    pytest -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg
 
 This will synchronize your :code:`mypkg` package directory
-to an remote ssh account and then locally collect tests
+to a remote ssh account and then locally collect tests
 and send them to remote places for execution.
 
 You can specify multiple :code:`--rsyncdir` directories
@@ -136,11 +139,11 @@ to be sent to the remote side.
 
 .. note::
 
-  For py.test to collect and send tests correctly
+  For pytest to collect and send tests correctly
   you not only need to make sure all code and tests
   directories are rsynced, but that any test (sub) directory
   also has an :code:`__init__.py` file because internally
-  py.test references tests as a fully qualified python
+  pytest references tests as a fully qualified python
   module path.  **You will otherwise get strange errors**
   during setup of the remote side.
 
@@ -153,7 +156,7 @@ ini-file option(s).
 
 
 Sending tests to remote Socket Servers
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------
 
 Download the single-module `socketserver.py`_ Python program
 and run it like this::
@@ -164,7 +167,7 @@ It will tell you that it starts listening on the default
 port.  You can now on your home machine specify this
 new socket host with something like this::
 
-    py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
+    pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg
 
 
 .. _`atonce`:
@@ -172,16 +175,16 @@ new socket host with something like this::
 
 
 Running tests on many platforms at once
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+---------------------------------------
 
 The basic command to run tests on multiple platforms is::
 
-    py.test --dist=each --tx=spec1 --tx=spec2
+    pytest --dist=each --tx=spec1 --tx=spec2
 
 If you specify a windows host, an OSX host and a Linux
 environment this command will send each tests to all
 platforms - and report back failures from all platforms
-at once.   The specifications strings use the `xspec syntax`_.
+at once. The specifications strings use the `xspec syntax`_.
 
 .. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
 
@@ -190,8 +193,9 @@ at once.   The specifications strings use the `xspec syntax`_.
 .. _`execnet`: http://codespeak.net/execnet
 
 Identifying the worker process during a test
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------------
 
+*New in version 1.15.*
 
 If you need to determine the identity of a worker process in
 a test or fixture, you may use the ``worker_id`` fixture to do so:
@@ -213,18 +217,15 @@ defined:
 * ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session,
   e.g., ``"4"`` when ``-n 4`` is given in the command-line.
 
-The information about the worker_id in a test is stored in the TestReport as
+The information about the worker_id in a test is stored in the ``TestReport`` as
-well, under worker_id attribute.
+well, under the ``worker_id`` attribute.
 
 
-*New in version 1.15.*
-
 Specifying test exec environments in an ini file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------------------
 
-pytest (since version 2.0) supports ini-style cofiguration.
+You can use pytest's ini file configuration to avoid typing common options.
-You can for example make running with three subprocesses
+You can for example make running with three subprocesses your default like this:
-your default like this:
 
 .. code-block:: ini
 
@@ -236,16 +237,17 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6
+    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
 
 and then just type::
 
-    py.test --dist=each
+    pytest --dist=each
 
 to run tests in each of the environments.
 
+
 Specifying "rsync" dirs in an ini-file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------
 
 In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
 you may specify directories to include or to exclude in synchronisation:
@@ -262,9 +264,3 @@ where the configuration file was found.
 .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
 .. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
 .. _`pytest`: http://pytest.org
-
-Issue and Bug Tracker
-------------------------
-
-Please use the `pytest issue tracker <https://github.com/pytest-dev/pytest/issues>`_
-for reporting bugs in this plugin.

appveyor.yml

@@ -1,20 +1,22 @@
 environment:
   matrix:
-  # note: please use "tox --listenvs" to populate the build matrix
+  - TOXENV: "py27-pytest36"
-  - TOXENV: "py26-pytest30"
+  - TOXENV: "py34-pytest36"
-  - TOXENV: "py27-pytest30"
+  - TOXENV: "py35-pytest36"
-  - TOXENV: "py34-pytest30"
+  - TOXENV: "py36-pytest36"
-  - TOXENV: "py35-pytest30"
+  - TOXENV: "py27-pytest36-pexpect"
-  - TOXENV: "py36-pytest30"
+  - TOXENV: "py36-pytest36-pexpect"
-  - TOXENV: "py27-pytest30-pexpect"
-  - TOXENV: "py35-pytest30-pexpect"
   - TOXENV: "flakes"
   - TOXENV: "readme"
 
 install:
-  - C:\Python35\python -m pip install tox setuptools_scm
+  - C:\Python36\python -m pip install -U tox setuptools_scm pip
 
 build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
-  - C:\Python35\python -m tox
+  - C:\Python36\python -m tox
+
+# We don't deploy anything on tags with AppVeyor, we use Travis instead, so we
+# might as well save resources
+skip_tags: true

changelog/_template.rst

@@ -13,8 +13,7 @@
 
 {% if definitions[category]['showcontent'] %}
 {% for text, values in sections[section][category]|dictsort(by='value') %}
-- {{ text }}{% if category != 'vendor' %} (`{{ values[0] }} <https://github.com/pytest-dev/pytest-xdist/issues/{{ values[0][1:] }}>`_){% endif %}
+- `{{ values[0] }} <https://github.com/pytest-dev/pytest-xdist/issues/{{ values[0][1:] }}>`_: {{ text }}
-
 
 {% endfor %}
 {% else %}

dev-requirements.txt

@@ -1 +0,0 @@
-towncrier

example/boxed.txt

@@ -1,3 +1,9 @@
+.. note::
+
+  Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a 
+  separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ 
+  which can be installed independently. The ``--boxed`` command-line options remains 
+  for backward compatibility reasons.
 
 
 If your testing involves C or C++ libraries you might have to deal
@@ -19,7 +25,7 @@ to run each test in a controlled subprocess.  Here is a basic example::
 
 If you run this with::
 
-    $ py.test --boxed
+    $ pytest -n1
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
@@ -40,7 +46,7 @@ You'll see that a couple of tests are reported as crashing, indicated
 by lower-case ``f`` and the respective failure summary. You can also use
 the xdist-provided parallelization feature to speed up your testing::
 
-    $ py.test --boxed -n3
+    $ pytest -n3
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov

example/loadscope/epsilon/__init__.py

@@ -0,0 +1,37 @@
+def epsilon1(arg1, arg2=1000):
+    """Do epsilon1
+
+    Usage:
+
+    >>> epsilon1(10, 20)
+    40
+    >>> epsilon1(30)
+    1040
+    """
+    return arg1 + arg2 + 10
+
+
+def epsilon2(arg1, arg2=1000):
+    """Do epsilon2
+
+    Usage:
+
+    >>> epsilon2(10, 20)
+    -20
+    >>> epsilon2(30)
+    -980
+    """
+    return arg1 - arg2 - 10
+
+
+def epsilon3(arg1, arg2=1000):
+    """Do epsilon3
+
+    Usage:
+
+    >>> epsilon3(10, 20)
+    200
+    >>> epsilon3(30)
+    30000
+    """
+    return arg1 * arg2

example/loadscope/requirements.txt

@@ -0,0 +1,3 @@
+ipdb
+pytest
+../../

example/loadscope/test/test_alpha.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_alpha0():
+    sleep(5)
+    assert True
+
+
+def test_alpha1():
+    sleep(5)
+    assert True
+
+
+def test_alpha2():
+    sleep(5)
+    assert True
+
+
+def test_alpha3():
+    sleep(5)
+    assert True
+
+
+def test_alpha4():
+    sleep(5)
+    assert True
+
+
+def test_alpha5():
+    sleep(5)
+    assert True
+
+
+def test_alpha6():
+    sleep(5)
+    assert True
+
+
+def test_alpha7():
+    sleep(5)
+    assert True
+
+
+def test_alpha8():
+    sleep(5)
+    assert True
+
+
+def test_alpha9():
+    sleep(5)
+    assert True

example/loadscope/test/test_beta.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_beta0():
+    sleep(5)
+    assert True
+
+
+def test_beta1():
+    sleep(5)
+    assert True
+
+
+def test_beta2():
+    sleep(5)
+    assert True
+
+
+def test_beta3():
+    sleep(5)
+    assert True
+
+
+def test_beta4():
+    sleep(5)
+    assert True
+
+
+def test_beta5():
+    sleep(5)
+    assert True
+
+
+def test_beta6():
+    sleep(5)
+    assert True
+
+
+def test_beta7():
+    sleep(5)
+    assert True
+
+
+def test_beta8():
+    sleep(5)
+    assert True
+
+
+def test_beta9():
+    sleep(5)
+    assert True

example/loadscope/test/test_delta.py

@@ -0,0 +1,88 @@
+from time import sleep
+from unittest import TestCase
+
+
+class Delta1(TestCase):
+
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True
+
+
+class Delta2(TestCase):
+
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True

example/loadscope/test/test_gamma.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_gamma0():
+    sleep(5)
+    assert True
+
+
+def test_gamma1():
+    sleep(5)
+    assert True
+
+
+def test_gamma2():
+    sleep(5)
+    assert True
+
+
+def test_gamma3():
+    sleep(5)
+    assert True
+
+
+def test_gamma4():
+    sleep(5)
+    assert True
+
+
+def test_gamma5():
+    sleep(5)
+    assert True
+
+
+def test_gamma6():
+    sleep(5)
+    assert True
+
+
+def test_gamma7():
+    sleep(5)
+    assert True
+
+
+def test_gamma8():
+    sleep(5)
+    assert True
+
+
+def test_gamma9():
+    sleep(5)
+    assert True

example/loadscope/tox.ini

@@ -0,0 +1,17 @@
+[tox]
+envlist = test
+setupdir = {toxinidir}/../../
+
+[testenv:test]
+basepython = python3
+passenv = http_proxy https_proxy
+deps = -rrequirements.txt
+changedir = {envtmpdir}
+commands =
+    pytest -s -v \
+        --doctest-modules \
+        --junitxml=tests.xml \
+        --dist=loadscope \
+        --tx=8*popen \
+        {toxinidir}/test \
+        {toxinidir}/epsilon

pyproject.toml

@@ -33,4 +33,4 @@ template = "changelog/_template.rst"
   [[tool.towncrier.type]]
   directory = "trivial"
   name = "Trivial Changes"
-  showcontent = false
+  showcontent = true

setup.cfg

@@ -3,3 +3,6 @@ universal = 1
 
 [metadata]
 license_file = LICENSE
+
+[flake8]
+max-line-length = 100

setup.py

@@ -1,9 +1,12 @@
-from setuptools import setup
+from setuptools import setup, find_packages
+
+install_requires = ['execnet>=1.1', 'pytest>=3.4', 'pytest-forked']
+
 
 setup(
     name="pytest-xdist",
     use_scm_version={'write_to': 'xdist/_version.py'},
-    description='py.test xdist plugin for distributed testing'
+    description='pytest xdist plugin for distributed testing'
                 ' and loop-on-failing modes',
     long_description=open('README.rst').read(),
     license='MIT',
@@ -11,16 +14,16 @@ setup(
     author_email='pytest-dev@python.org,holger@merlinux.eu',
     url='https://github.com/pytest-dev/pytest-xdist',
     platforms=['linux', 'osx', 'win32'],
-    packages=['xdist'],
+    packages=find_packages(exclude=['testing', 'example']),
     entry_points={
         'pytest11': [
             'xdist = xdist.plugin',
             'xdist.looponfail = xdist.looponfail',
-            'xdist.boxed = xdist.boxed',
         ],
     },
     zip_safe=False,
-    install_requires=['execnet>=1.1', 'pytest>=3.0.0', 'py>=1.4.22'],
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    install_requires=install_requires,
     setup_requires=['setuptools_scm'],
     classifiers=[
         'Development Status :: 5 - Production/Stable',
@@ -34,6 +37,11 @@ setup(
         'Topic :: Software Development :: Quality Assurance',
         'Topic :: Utilities',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
     ],
 )

testing/acceptance_test.py

@@ -1,3 +1,7 @@
+import os
+import re
+import textwrap
+
 import py
 import pytest
 
@@ -52,8 +56,8 @@ class TestDistribution:
     def test_n1_skip(self, testdir):
         p1 = testdir.makepyfile("""
             def test_skip():
-                import py
+                import pytest
-                py.test.skip("myreason")
+                pytest.skip("myreason")
         """)
         result = testdir.runpytest(p1, "-n1")
         assert result.ret == 0
@@ -76,7 +80,7 @@ class TestDistribution:
     def test_manytests_to_one_popen(self, testdir):
         p1 = testdir.makepyfile(
             """
-                import py
+                import pytest
                 def test_fail0():
                     assert 0
                 def test_fail1():
@@ -84,7 +88,7 @@ class TestDistribution:
                 def test_ok():
                     pass
                 def test_skip():
-                    py.test.skip("hello")
+                    pytest.skip("hello")
             """, )
         result = testdir.runpytest(p1, "-v", '-d', '--tx=popen', '--tx=popen')
         result.stdout.fnmatch_lines([
@@ -122,7 +126,7 @@ class TestDistribution:
     def test_dist_ini_specified(self, testdir):
         p1 = testdir.makepyfile(
             """
-                import py
+                import pytest
                 def test_fail0():
                     assert 0
                 def test_fail1():
@@ -130,7 +134,7 @@ class TestDistribution:
                 def test_ok():
                     pass
                 def test_skip():
-                    py.test.skip("hello")
+                    pytest.skip("hello")
             """, )
         testdir.makeini("""
             [pytest]
@@ -143,13 +147,13 @@ class TestDistribution:
         ])
         assert result.ret == 1
 
-    @py.test.mark.xfail("sys.platform.startswith('java')", run=False)
+    @pytest.mark.xfail("sys.platform.startswith('java')", run=False)
     def test_dist_tests_with_crash(self, testdir):
-        if not hasattr(py.std.os, 'kill'):
+        if not hasattr(os, 'kill'):
-            py.test.skip("no os.kill")
+            pytest.skip("no os.kill")
 
         p1 = testdir.makepyfile("""
-                import py
+                import pytest
                 def test_fail0():
                     assert 0
                 def test_fail1():
@@ -157,7 +161,7 @@ class TestDistribution:
                 def test_ok():
                     pass
                 def test_skip():
-                    py.test.skip("hello")
+                    pytest.skip("hello")
                 def test_crash():
                     import time
                     import os
@@ -190,27 +194,41 @@ class TestDistribution:
         ])
         assert dest.join(subdir.basename).check(dir=1)
 
+    def test_backward_compatibility_worker_terminology(self, testdir):
+        """Ensure that we still support "config.slaveinput" for backward compatibility (#234).
+
+        Keep in mind that removing this compatibility will break a ton of plugins and user code.
+        """
+        testdir.makepyfile("""
+            def test(pytestconfig):
+                assert hasattr(pytestconfig, 'slaveinput')
+                assert hasattr(pytestconfig, 'workerinput')
+        """)
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines("*1 passed*")
+        assert result.ret == 0
+
     def test_data_exchange(self, testdir):
         testdir.makeconftest("""
             # This hook only called on master.
             def pytest_configure_node(node):
-                node.slaveinput['a'] = 42
+                node.workerinput['a'] = 42
-                node.slaveinput['b'] = 7
+                node.workerinput['b'] = 7
 
             def pytest_configure(config):
-                # this attribute is only set on slaves
+                # this attribute is only set on workers
-                if hasattr(config, 'slaveinput'):
+                if hasattr(config, 'workerinput'):
-                    a = config.slaveinput['a']
+                    a = config.workerinput['a']
-                    b = config.slaveinput['b']
+                    b = config.workerinput['b']
                     r = a + b
-                    config.slaveoutput['r'] = r
+                    config.workeroutput['r'] = r
 
             # This hook only called on master.
             def pytest_testnodedown(node, error):
-                node.config.calc_result = node.slaveoutput['r']
+                node.config.calc_result = node.workeroutput['r']
 
             def pytest_terminal_summary(terminalreporter):
-                if not hasattr(terminalreporter.config, 'slaveinput'):
+                if not hasattr(terminalreporter.config, 'workerinput'):
                     calc_result = terminalreporter.config.calc_result
                     terminalreporter._tw.sep('-',
                         'calculated result is %s' % calc_result)
@@ -230,12 +248,12 @@ class TestDistribution:
         """)
         testdir.makeconftest("""
             def pytest_sessionfinish(session):
-                # on the slave
+                # on the worker
-                if hasattr(session.config, 'slaveoutput'):
+                if hasattr(session.config, 'workeroutput'):
-                    session.config.slaveoutput['s2'] = 42
+                    session.config.workeroutput['s2'] = 42
             # on the master
             def pytest_testnodedown(node, error):
-                assert node.slaveoutput['s2'] == 42
+                assert node.workeroutput['s2'] == 42
                 print ("s2call-finished")
         """)
         args = ["-n1", "--debug"]
@@ -260,6 +278,17 @@ class TestDistribution:
         child.close()
         # assert ret == 2
 
+    def test_dist_with_collectonly(self, testdir):
+        p1 = testdir.makepyfile("""
+            def test_ok():
+                pass
+        """)
+        result = testdir.runpytest(p1, "-n1", "--collect-only")
+        assert result.ret == 0
+        result.stdout.fnmatch_lines([
+            "*collected 1 item*",
+        ])
+
 
 class TestDistEach:
     def test_simple(self, testdir):
@@ -272,7 +301,7 @@ class TestDistEach:
         assert not result.ret
         result.stdout.fnmatch_lines(["*2 pass*"])
 
-    @py.test.mark.xfail(
+    @pytest.mark.xfail(
         run=False,
         reason="other python versions might not have py.test installed")
     def test_simple_diffoutput(self, testdir):
@@ -280,7 +309,7 @@ class TestDistEach:
         for name in ("python2.5", "python2.6"):
             interp = py.path.local.sysfind(name)
             if interp is None:
-                py.test.skip("%s not found" % name)
+                pytest.skip("%s not found" % name)
             interpreters.append(interp)
 
         testdir.makepyfile(
@@ -303,11 +332,11 @@ class TestDistEach:
 class TestTerminalReporting:
     def test_pass_skip_fail(self, testdir):
         testdir.makepyfile("""
-            import py
+            import pytest
             def test_ok():
                 pass
             def test_skip():
-                py.test.skip("xx")
+                pytest.skip("xx")
             def test_func():
                 assert 0
         """)
@@ -337,6 +366,50 @@ class TestTerminalReporting:
             "E       assert 0",
         ])
 
+    @pytest.mark.parametrize('n', ['-n0', '-n1'])
+    @pytest.mark.parametrize('warn_type', ['pytest', 'builtin'])
+    def test_logwarning(self, testdir, n, warn_type):
+        from pkg_resources import parse_version
+        if parse_version(pytest.__version__) < parse_version('3.1'):
+            pytest.skip('pytest warnings requires >= 3.1')
+
+        if warn_type == 'builtin':
+            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
+        elif warn_type == 'pytest':
+            warn_code = """request.config.warn('', 'this is a warning',
+                           fslocation=py.path.local())"""
+        else:
+            assert False
+        testdir.makepyfile("""
+            import warnings, py
+            def test_func(request):
+                {warn_code}
+        """.format(warn_code=warn_code))
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines([
+            "*this is a warning*",
+            "*1 passed, 1 warnings*",
+        ])
+
+    def test_logfinish_hook(self, testdir):
+        """Ensure the pytest_runtest_logfinish hook is being properly handled"""
+        from _pytest import hookspec
+        if not hasattr(hookspec, 'pytest_runtest_logfinish'):
+            pytest.skip('test requires pytest_runtest_logfinish hook in pytest (3.4+)')
+
+        testdir.makeconftest("""
+            def pytest_runtest_logfinish():
+                print('pytest_runtest_logfinish hook called')
+        """)
+        testdir.makepyfile("""
+            def test_func():
+                pass
+        """)
+        result = testdir.runpytest("-n1", "-s")
+        result.stdout.fnmatch_lines([
+            "*pytest_runtest_logfinish hook called*",
+        ])
+
 
 def test_teardownfails_one_function(testdir):
     p = testdir.makepyfile("""
@@ -350,11 +423,11 @@ def test_teardownfails_one_function(testdir):
         ["*def teardown_function(function):*", "*1 passed*1 error*"])
 
 
-@py.test.mark.xfail
+@pytest.mark.xfail
 def test_terminate_on_hangingnode(testdir):
     p = testdir.makeconftest("""
         def pytest_sessionfinish(session):
-            if session.nodeid == "my": # running on slave
+            if session.nodeid == "my": # running on worker
                 import time
                 time.sleep(3)
     """)
@@ -372,15 +445,15 @@ def test_session_hooks(testdir):
         def pytest_sessionstart(session):
             sys.pytestsessionhooks = session
         def pytest_sessionfinish(session):
-            if hasattr(session.config, 'slaveinput'):
+            if hasattr(session.config, 'workerinput'):
-                name = "slave"
+                name = "worker"
             else:
                 name = "master"
             f = open(name, "w")
             f.write("xy")
             f.close()
-            # let's fail on the slave
+            # let's fail on the worker
-            if name == "slave":
+            if name == "worker":
                 raise ValueError(42)
     """)
     p = testdir.makepyfile("""
@@ -396,14 +469,14 @@ def test_session_hooks(testdir):
     assert not result.ret
     d = result.parseoutcomes()
     assert d['passed'] == 1
-    assert testdir.tmpdir.join("slave").check()
+    assert testdir.tmpdir.join("worker").check()
     assert testdir.tmpdir.join("master").check()
 
 
 def test_session_testscollected(testdir):
     """
     Make sure master node is updating the session object with the number
-    of tests collected from the slaves.
+    of tests collected from the workers.
     """
     testdir.makepyfile(test_foo="""
         import pytest
@@ -444,18 +517,60 @@ def test_funcarg_teardown_failure(testdir):
     assert result.ret
 
 
-def test_crashing_item(testdir):
+@pytest.mark.parametrize('when', ['setup', 'call', 'teardown'])
+def test_crashing_item(testdir, when):
+    """Ensure crashing item is correctly reported during all testing stages"""
+    code = dict(setup='', call='', teardown='')
+    code[when] = 'py.process.kill(os.getpid())'
     p = testdir.makepyfile("""
-        import py
         import os
-        def test_crash():
+        import py
-            py.process.kill(os.getpid())
+        import pytest
-        def test_noncrash():
+
+        @pytest.fixture
+        def fix():
+            {setup}
+            yield
+            {teardown}
+
+        def test_crash(fix):
+            {call}
             pass
-    """)
+
+        def test_ok():
+            pass
+    """.format(**code))
+    passes = 2 if when == 'teardown' else 1
     result = testdir.runpytest("-n2", p)
     result.stdout.fnmatch_lines([
-        "*crashed*test_crash*", "*1 failed*1 passed*"
+        "*crashed*test_crash*",
+        "*1 failed*%d passed*" % passes,
+    ])
+
+
+def test_multiple_log_reports(testdir):
+    """
+    Ensure that pytest-xdist supports plugins that emit multiple logreports
+    (#206).
+    Inspired by pytest-rerunfailures.
+    """
+    testdir.makeconftest("""
+        from _pytest.runner import runtestprotocol
+        def pytest_runtest_protocol(item, nextitem):
+            item.ihook.pytest_runtest_logstart(nodeid=item.nodeid,
+                                               location=item.location)
+            reports = runtestprotocol(item, nextitem=nextitem)
+            for report in reports:
+                item.ihook.pytest_runtest_logreport(report=report)
+            return True
+    """)
+    testdir.makepyfile("""
+        def test():
+            pass
+    """)
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines([
+        "*2 passed*",
     ])
 
 
@@ -472,7 +587,7 @@ def test_skipping(testdir):
 
 def test_issue34_pluginloading_in_subprocess(testdir):
     testdir.tmpdir.join("plugin123.py").write(
-        py.code.Source("""
+        textwrap.dedent("""
         def pytest_namespace():
             return {'sample_variable': 'testing'}
     """))
@@ -568,8 +683,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n1')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
+            "*Replacing crashed worker*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*1 failed*1 passed*",
         ])
 
@@ -583,8 +698,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n2')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
+            "*Replacing crashed worker*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*1 failed*3 passed*",
         ])
 
@@ -596,8 +711,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=popen')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
+            "*Replacing crashed worker*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*1 failed*1 passed*",
         ])
 
@@ -610,12 +725,12 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=2*popen')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
+            "*Replacing crashed worker*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*2 failed*2 passed*",
         ])
 
-    def test_max_slave_restart(self, testdir):
+    def test_max_worker_restart(self, testdir):
         f = testdir.makepyfile("""
             import os
             def test_a(): pass
@@ -623,15 +738,26 @@ class TestNodeFailure:
             def test_c(): os._exit(1)
             def test_d(): pass
         """)
-        res = testdir.runpytest(f, '-n4', '--max-slave-restart=1')
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=1')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
+            "*Replacing crashed worker*",
-            "*Maximum crashed slaves reached: 1*",
+            "*Maximum crashed workers reached: 1*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*2 failed*2 passed*",
         ])
 
+    def test_max_worker_restart_die(self, testdir):
+        f = testdir.makepyfile("""
+            import os
+            os._exit(1)
+        """)
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=0')
+        res.stdout.fnmatch_lines([
+            "*Unexpectedly no active workers*",
+            "*INTERNALERROR*"
+        ])
+
     def test_disable_restart(self, testdir):
         f = testdir.makepyfile("""
             import os
@@ -639,10 +765,10 @@ class TestNodeFailure:
             def test_b(): os._exit(1)
             def test_c(): pass
         """)
-        res = testdir.runpytest(f, '-n4', '--max-slave-restart=0')
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=0')
         res.stdout.fnmatch_lines([
-            "*Slave restarting disabled*",
+            "*Worker restarting disabled*",
-            "*Slave*crashed while running*",
+            "*Worker*crashed while running*",
             "*1 failed*2 passed*",
         ])
 
@@ -664,9 +790,23 @@ def test_worker_id_fixture(testdir, n):
         with open(fname) as f:
             worker_ids.add(f.read().strip())
     if n == 0:
-        assert worker_ids == set(['master'])
+        assert worker_ids == {'master'}
     else:
-        assert worker_ids == set(['gw0', 'gw1'])
+        assert worker_ids == {'gw0', 'gw1'}
+
+
+@pytest.mark.parametrize('tb',
+                         ['auto', 'long', 'short', 'no', 'line', 'native'])
+def test_error_report_styles(testdir, tb):
+    testdir.makepyfile("""
+        import pytest
+        def test_error_report_styles():
+            raise RuntimeError('some failure happened')
+    """)
+    result = testdir.runpytest('-n1', '--tb=%s' % tb)
+    if tb != 'no':
+        result.stdout.fnmatch_lines('*some failure happened*')
+    result.assert_outcomes(failed=1)
 
 
 def test_color_yes_collection_on_non_atty(testdir, request):
@@ -710,3 +850,189 @@ def test_internal_error_with_maxfail(testdir):
     result = testdir.runpytest_subprocess('--maxfail=1', '-n1')
     result.stdout.fnmatch_lines(['* 1 error in *'])
     assert 'INTERNALERROR' not in result.stderr.str()
+
+
+class TestLoadScope:
+
+    def test_by_module(self, testdir):
+        test_file = """
+            import pytest
+            @pytest.mark.parametrize('i', range(10))
+            def test(i):
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file,
+            test_b=test_file,
+        )
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::test', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+        assert get_workers_and_test_count_by_prefix(
+            'test_b.py::test', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+
+    def test_by_class(self, testdir):
+        testdir.makepyfile(test_a="""
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """)
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::TestB', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+
+    def test_module_single_start(self, testdir):
+        """Fix test suite never finishing in case all workers start with a single test (#277)."""
+        test_file1 = """
+            import pytest
+            def test():
+                pass
+        """
+        test_file2 = """
+            import pytest
+            def test_1():
+                pass
+            def test_2():
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file1,
+            test_b=test_file1,
+            test_c=test_file2
+        )
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        a = get_workers_and_test_count_by_prefix('test_a.py::test',
+                                                 result.outlines)
+        b = get_workers_and_test_count_by_prefix('test_b.py::test',
+                                                 result.outlines)
+        c1 = get_workers_and_test_count_by_prefix('test_c.py::test_1',
+                                                  result.outlines)
+        c2 = get_workers_and_test_count_by_prefix('test_c.py::test_2',
+                                                  result.outlines)
+        assert a in ({'gw0': 1}, {'gw1': 1})
+        assert b in ({'gw0': 1}, {'gw1': 1})
+        assert a.items() != b.items()
+        assert c1 == c2
+
+
+class TestFileScope:
+
+    def test_by_module(self, testdir):
+        test_file = """
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """
+        testdir.makepyfile(
+            test_a=test_file,
+            test_b=test_file,
+        )
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines)
+        test_b_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_b.py::TestB', result.outlines)
+
+        assert test_a_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_a_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+        assert test_b_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_b_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+
+    def test_by_class(self, testdir):
+        testdir.makepyfile(test_a="""
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """)
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines)
+        test_b_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestB', result.outlines)
+
+        assert test_a_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_a_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+        assert test_b_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_b_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+
+    def test_module_single_start(self, testdir):
+        """Fix test suite never finishing in case all workers start with a single test (#277)."""
+        test_file1 = """
+            import pytest
+            def test():
+                pass
+        """
+        test_file2 = """
+            import pytest
+            def test_1():
+                pass
+            def test_2():
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file1,
+            test_b=test_file1,
+            test_c=test_file2
+        )
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        a = get_workers_and_test_count_by_prefix('test_a.py::test',
+                                                 result.outlines)
+        b = get_workers_and_test_count_by_prefix('test_b.py::test',
+                                                 result.outlines)
+        c1 = get_workers_and_test_count_by_prefix('test_c.py::test_1',
+                                                  result.outlines)
+        c2 = get_workers_and_test_count_by_prefix('test_c.py::test_2',
+                                                  result.outlines)
+        assert a in ({'gw0': 1}, {'gw1': 1})
+        assert b in ({'gw0': 1}, {'gw1': 1})
+        assert a.items() != b.items()
+        assert c1 == c2
+
+
+def parse_tests_and_workers_from_output(lines):
+    result = []
+    for line in lines:
+        # example match: "[gw0] PASSED test_a.py::test[7]"
+        m = re.match(r'''
+            \[(gw\d)\]  # worker
+            \s*
+            (?:\[\s*\d+%\])? # progress indicator (pytest >=3.3)
+            \s(.*?)     # status string ("PASSED")
+            \s(.*::.*)  # nodeid
+        ''', line.strip(), re.VERBOSE)
+        if m:
+            worker, status, nodeid = m.groups()
+            result.append((worker, status, nodeid))
+    return result
+
+
+def get_workers_and_test_count_by_prefix(prefix, lines,
+                                         expected_status='PASSED'):
+    result = {}
+    for worker, status, nodeid in parse_tests_and_workers_from_output(lines):
+        if expected_status == status and nodeid.startswith(prefix):
+            result[worker] = result.get(worker, 0) + 1
+    return result

testing/conftest.py

@@ -20,13 +20,13 @@ pytest_plugins = "pytester"
 @pytest.fixture(autouse=True)
 def _divert_atexit(request, monkeypatch):
     import atexit
-    l = []
+    finalizers = []
 
     def finish():
-        while l:
+        while finalizers:
-            l.pop()()
+            finalizers.pop()()
 
-    monkeypatch.setattr(atexit, "register", l.append)
+    monkeypatch.setattr(atexit, "register", finalizers.append)
     request.addfinalizer(finish)
 
 
@@ -34,7 +34,7 @@ def pytest_addoption(parser):
     parser.addoption('--gx',
                      action="append",
                      dest="gspecs",
-                     help=("add a global test environment, XSpec-syntax. "))
+                     help="add a global test environment, XSpec-syntax. ")
 
 
 @pytest.fixture

testing/test_boxed.py

@@ -1,58 +0,0 @@
-import pytest
-import os
-
-needsfork = pytest.mark.skipif(not hasattr(os, "fork"),
-                               reason="os.fork required")
-
-
-@needsfork
-def test_functional_boxed(testdir):
-    p1 = testdir.makepyfile("""
-        import os
-        def test_function():
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed")
-    result.stdout.fnmatch_lines([
-        "*CRASHED*",
-        "*1 failed*"
-    ])
-
-
-@needsfork
-@pytest.mark.parametrize("capmode", [
-    "no",
-    pytest.mark.xfail("sys", reason="capture cleanup needed"),
-    pytest.mark.xfail("fd", reason="capture cleanup needed")])
-def test_functional_boxed_capturing(testdir, capmode):
-    p1 = testdir.makepyfile("""
-        import os
-        import sys
-        def test_function():
-            sys.stdout.write("hello\\n")
-            sys.stderr.write("world\\n")
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed", "--capture=%s" % capmode)
-    result.stdout.fnmatch_lines("""
-        *CRASHED*
-        *stdout*
-        hello
-        *stderr*
-        world
-        *1 failed*
-    """)
-
-
-class TestOptionEffects:
-    def test_boxed_option_default(self, testdir):
-        tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
-        config = testdir.parseconfig()
-        assert not config.option.boxed
-        pytest.importorskip("execnet")
-        config = testdir.parseconfig('-d', tmpdir)
-        assert not config.option.boxed
-
-    def test_is_not_boxed_by_default(self, testdir):
-        config = testdir.parseconfig(testdir.tmpdir)
-        assert not config.option.boxed

testing/test_dsession.py

@@ -1,6 +1,10 @@
-from xdist.dsession import (
+from xdist.dsession import DSession
-    DSession, LoadScheduling, EachScheduling, report_collection_diff,
+from xdist.report import report_collection_diff
+from xdist.scheduler import (
+    EachScheduling,
+    LoadScheduling,
 )
+
 import py
 import pytest
 import execnet
@@ -122,7 +126,7 @@ class TestLoadScheduling:
         sched.add_node(MockNode())
         sched.add_node(MockNode())
         node1, node2 = sched.nodes
-        col = ["xyz"] * (6)
+        col = ["xyz"] * 6
         sched.add_node_collection(node1, col)
         sched.add_node_collection(node2, col)
         sched.schedule()
@@ -199,7 +203,7 @@ class TestLoadScheduling:
     def test_different_tests_collected(self, testdir):
         """
         Test that LoadScheduling is reporting collection errors when
-        different test ids are collected by slaves.
+        different test ids are collected by workers.
         """
 
         class CollectHook(object):

testing/test_looponfail.py

@@ -83,10 +83,10 @@ class TestStatRecorder:
         tmp = tmpdir
         sd = StatRecorder([tmp])
 
-        l = [True, False]
+        ret_values = [True, False]
-        monkeypatch.setattr(StatRecorder, 'check', lambda self: l.pop())
+        monkeypatch.setattr(StatRecorder, 'check', lambda self: ret_values.pop())
         sd.waitonchange(checkinterval=0.2)
-        assert not l
+        assert not ret_values
 
 
 class TestRemoteControl:
@@ -193,6 +193,8 @@ class TestLooponFailing:
         assert 'test_one' not in remotecontrol.failures[0]
         assert 'test_two' in remotecontrol.failures[0]
 
+    @py.test.mark.xfail(py.test.__version__ >= "3.1",
+                        reason="broken by pytest 3.1+")
     def test_looponfail_removed_test(self, testdir):
         modcol = testdir.getmodulecol("""
             def test_one():

testing/test_newhooks.py

@@ -20,10 +20,10 @@ class TestHooks:
             def pytest_runtest_logreport(report):
                 if hasattr(report, 'node'):
                     if report.when == "call":
-                        slaveid = report.node.slaveinput['slaveid']
+                        workerid = report.node.workerinput['workerid']
-                        if slaveid != report.worker_id:
+                        if workerid != report.worker_id:
                             print("HOOK: Worker id mismatch: %s %s"
-                                   % (slaveid, report.worker_id))
+                                   % (workerid, report.worker_id))
                         else:
                             print("HOOK: %s %s"
                                    % (report.nodeid, report.worker_id))
@@ -41,9 +41,9 @@ class TestHooks:
         """
         testdir.makeconftest("""
             def pytest_xdist_node_collection_finished(node, ids):
-                slaveid = node.slaveinput['slaveid']
+                workerid = node.workerinput['workerid']
                 stripped_ids = [x.split('::')[1] for x in ids]
-                print("HOOK: %s %s" % (slaveid, ', '.join(stripped_ids)))
+                print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
         """)
         res = testdir.runpytest('-n2', '-s')
         res.stdout.fnmatch_lines_random([

testing/test_plugin.py

@@ -1,6 +1,6 @@
 import py
 import execnet
-from xdist.slavemanage import NodeManager
+from xdist.workermanage import NodeManager
 
 
 def test_dist_incompatibility_messages(testdir):
@@ -31,7 +31,9 @@ def test_dist_options(testdir):
 
 def test_auto_detect_cpus(testdir, monkeypatch):
     import os
-    if hasattr(os, 'cpu_count'):
+    if hasattr(os, 'sched_getaffinity'):
+        monkeypatch.setattr(os, 'sched_getaffinity', lambda _pid: set(range(99)))
+    elif hasattr(os, 'cpu_count'):
         monkeypatch.setattr(os, 'cpu_count', lambda: 99)
     else:
         import multiprocessing
@@ -43,6 +45,41 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     config = testdir.parseconfigure("-nauto")
     assert config.getoption('numprocesses') == 99
 
+    monkeypatch.delattr(os, 'sched_getaffinity', raising=False)
+    monkeypatch.setenv('TRAVIS', 'true')
+    config = testdir.parseconfigure("-nauto")
+    assert config.getoption('numprocesses') == 2
+
+
+def test_boxed_with_collect_only(testdir):
+    from xdist.plugin import pytest_cmdline_main as check_options
+    config = testdir.parseconfigure("-n1", "--boxed")
+    check_options(config)
+    assert config.option.forked
+
+    config = testdir.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    assert not config.option.forked
+
+    config = testdir.parseconfigure("-n1", "--boxed", "--collect-only")
+    check_options(config)
+    assert config.option.forked
+
+
+def test_dsession_with_collect_only(testdir):
+    from xdist.plugin import pytest_cmdline_main as check_options
+    from xdist.plugin import pytest_configure as configure
+
+    config = testdir.parseconfigure("-n1")
+    check_options(config)
+    configure(config)
+    assert config.pluginmanager.hasplugin("dsession")
+
+    config = testdir.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    configure(config)
+    assert not config.pluginmanager.hasplugin("dsession")
+
 
 class TestDistOptions:
     def test_getxspecs(self, testdir):

testing/test_remote.py

@@ -1,11 +1,12 @@
 import py
+import pprint
 import pytest
-from xdist.slavemanage import SlaveController, unserialize_report
+from xdist.workermanage import WorkerController, unserialize_report
 from xdist.remote import serialize_report
 import execnet
 import marshal
 
-queue = py.builtin._tryimport("queue", "Queue")
+from six.moves.queue import Queue
 
 WAIT_TIMEOUT = 10.0
 
@@ -14,7 +15,7 @@ def check_marshallable(d):
     try:
         marshal.dumps(d)
     except ValueError:
-        py.std.pprint.pprint(d)
+        pprint.pprint(d)
         raise ValueError("not marshallable")
 
 
@@ -26,13 +27,13 @@ class EventCall:
         return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
 
 
-class SlaveSetup:
+class WorkerSetup:
     use_callback = False
 
-    def __init__(self, request):
+    def __init__(self, request, testdir):
-        self.testdir = request.getfuncargvalue("testdir")
         self.request = request
-        self.events = queue.Queue()
+        self.testdir = testdir
+        self.events = Queue()
 
     def setup(self, ):
         self.testdir.chdir()
@@ -44,8 +45,8 @@ class SlaveSetup:
         class DummyMananger:
             specs = [0, 1]
 
-        self.slp = SlaveController(DummyMananger, self.gateway, config,
+        self.slp = WorkerController(DummyMananger, self.gateway, config,
-                                   putevent)
+                                    putevent)
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
@@ -65,8 +66,8 @@ class SlaveSetup:
 
 
 @pytest.fixture
-def slave(request):
+def worker(request, testdir):
-    return SlaveSetup(request)
+    return WorkerSetup(request, testdir)
 
 
 @pytest.mark.xfail(reason='#59')
@@ -79,6 +80,21 @@ def test_remoteinitconfig(testdir):
 
 
 class TestReportSerialization:
+    def test_xdist_longrepr_to_str_issue_241(self, testdir):
+        testdir.makepyfile("""
+            import os
+            def test_a(): assert False
+            def test_b(): pass
+        """)
+        testdir.makeconftest("""
+            def pytest_runtest_logreport(report):
+                print(report.longrepr)
+        """)
+        res = testdir.runpytest('-n1', '-s')
+        res.stdout.fnmatch_lines([
+            '*1 failed, 1 passed *'
+        ])
+
     def test_xdist_report_longrepr_reprcrash_130(self, testdir):
         reprec = testdir.inline_runsource("""
                     import py
@@ -107,17 +123,16 @@ class TestReportSerialization:
         assert rep.longrepr.reprtraceback.style \
             == a.longrepr.reprtraceback.style
         assert rep.longrepr.sections == a.longrepr.sections
-        assert rep.longrepr.reprtraceback.reprentries \
-            == a.longrepr.reprtraceback.reprentries
         # Missing section attribute PR171
         assert added_section in a.longrepr.sections
 
     def test_reprentries_serialization_170(self, testdir):
+        from _pytest._code.code import ReprEntry
         reprec = testdir.inline_runsource("""
-                            def test_fail():
+                            def test_repr_entry():
                                 x = 0
                                 assert x
-                        """, '--showlocals', '-n1')
+                        """, '--showlocals')
         reports = reprec.getreports("pytest_runtest_logreport")
         assert len(reports) == 3
         rep = reports[1]
@@ -126,15 +141,36 @@ class TestReportSerialization:
 
         rep_entries = rep.longrepr.reprtraceback.reprentries
         a_entries = a.longrepr.reprtraceback.reprentries
-        assert rep_entries == a_entries
         for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntry)
             assert rep_entries[i].lines == a_entries[i].lines
             assert rep_entries[i].localssep == a_entries[i].localssep
-            assert rep_entries[i].reprfileloc == a_entries[i].reprfileloc
+            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
-            assert rep_entries[i].reprfuncargs == a_entries[i].reprfuncargs
+            assert rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
-            assert rep_entries[i].reprlocals == a_entries[i].reprlocals
+            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
+            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
+            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
             assert rep_entries[i].style == a_entries[i].style
 
+    def test_reprentries_serialization_196(self, testdir):
+        from _pytest._code.code import ReprEntryNative
+        reprec = testdir.inline_runsource("""
+                            def test_repr_entry_native():
+                                x = 0
+                                assert x
+                        """, '--tb=native')
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        d = serialize_report(rep)
+        a = unserialize_report("testreport", d)
+
+        rep_entries = rep.longrepr.reprtraceback.reprentries
+        a_entries = a.longrepr.reprtraceback.reprentries
+        for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntryNative)
+            assert rep_entries[i].lines == a_entries[i].lines
+
     def test_itemreport_outcomes(self, testdir):
         reprec = testdir.inline_runsource("""
             import py
@@ -208,108 +244,108 @@ class TestReportSerialization:
                 assert newrep.longrepr == str(rep.longrepr)
 
 
-class TestSlaveInteractor:
+class TestWorkerInteractor:
-    def test_basic_collect_and_runtests(self, slave):
+    def test_basic_collect_and_runtests(self, worker):
-        slave.testdir.makepyfile("""
+        worker.testdir.makepyfile("""
             def test_func():
                 pass
         """)
-        slave.setup()
+        worker.setup()
-        ev = slave.popevent()
+        ev = worker.popevent()
-        assert ev.name == "slaveready"
+        assert ev.name == "workerready"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
-        assert ev.kwargs['topdir'] == slave.testdir.tmpdir
+        assert ev.kwargs['topdir'] == worker.testdir.tmpdir
         ids = ev.kwargs['ids']
         assert len(ids) == 1
-        slave.sendcommand("runtests", indices=list(range(len(ids))))
+        worker.sendcommand("runtests", indices=list(range(len(ids))))
-        slave.sendcommand("shutdown")
+        worker.sendcommand("shutdown")
-        ev = slave.popevent("logstart")
+        ev = worker.popevent("logstart")
         assert ev.kwargs["nodeid"].endswith("test_func")
         assert len(ev.kwargs["location"]) == 3
-        ev = slave.popevent("testreport")  # setup
+        ev = worker.popevent("testreport")  # setup
-        ev = slave.popevent("testreport")
+        ev = worker.popevent("testreport")
         assert ev.name == "testreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        ev = slave.popevent("slavefinished")
+        ev = worker.popevent("workerfinished")
-        assert 'slaveoutput' in ev.kwargs
+        assert 'workeroutput' in ev.kwargs
 
     @pytest.mark.skipif(pytest.__version__ >= '3.0',
                         reason='skip at module level illegal in pytest 3.0')
-    def test_remote_collect_skip(self, slave):
+    def test_remote_collect_skip(self, worker):
-        slave.testdir.makepyfile("""
+        worker.testdir.makepyfile("""
             import py
             py.test.skip("hello")
         """)
-        slave.setup()
+        worker.setup()
-        ev = slave.popevent("collectionstart")
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.skipped
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         assert not ev.kwargs['ids']
 
-    def test_remote_collect_fail(self, slave):
+    def test_remote_collect_fail(self, worker):
-        slave.testdir.makepyfile("""aasd qwe""")
+        worker.testdir.makepyfile("""aasd qwe""")
-        slave.setup()
+        worker.setup()
-        ev = slave.popevent("collectionstart")
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.failed
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         assert not ev.kwargs['ids']
 
-    def test_runtests_all(self, slave):
+    def test_runtests_all(self, worker):
-        slave.testdir.makepyfile("""
+        worker.testdir.makepyfile("""
             def test_func(): pass
             def test_func2(): pass
         """)
-        slave.setup()
+        worker.setup()
-        ev = slave.popevent()
+        ev = worker.popevent()
-        assert ev.name == "slaveready"
+        assert ev.name == "workerready"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         ids = ev.kwargs['ids']
         assert len(ids) == 2
-        slave.sendcommand("runtests_all", )
+        worker.sendcommand("runtests_all", )
-        slave.sendcommand("shutdown", )
+        worker.sendcommand("shutdown", )
         for func in "::test_func", "::test_func2":
             for i in range(3):  # setup/call/teardown
-                ev = slave.popevent("testreport")
+                ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
                 rep = unserialize_report(ev.name, ev.kwargs['data'])
                 assert rep.nodeid.endswith(func)
-        ev = slave.popevent("slavefinished")
+        ev = worker.popevent("workerfinished")
-        assert 'slaveoutput' in ev.kwargs
+        assert 'workeroutput' in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, slave):
+    def test_happy_run_events_converted(self, testdir, worker):
         py.test.xfail("implement a simple test for event production")
-        assert not slave.use_callback
+        assert not worker.use_callback
-        slave.testdir.makepyfile("""
+        worker.testdir.makepyfile("""
             def test_func():
                 pass
         """)
-        slave.setup()
+        worker.setup()
-        hookrec = testdir.getreportrecorder(slave.config)
+        hookrec = testdir.getreportrecorder(worker.config)
-        for data in slave.slp.channel:
+        for data in worker.slp.channel:
-            slave.slp.process_from_remote(data)
+            worker.slp.process_from_remote(data)
-        slave.slp.process_from_remote(slave.slp.ENDMARK)
+        worker.slp.process_from_remote(worker.slp.ENDMARK)
-        py.std.pprint.pprint(hookrec.hookrecorder.calls)
+        pprint.pprint(hookrec.hookrecorder.calls)
         hookrec.hookrecorder.contains([
             ("pytest_collectstart", "collector.fspath == aaa"),
             ("pytest_pycollect_makeitem", "name == 'test_func'"),
@@ -319,6 +355,15 @@ class TestSlaveInteractor:
             ("pytest_collectreport", "report.collector.fspath == bbb"),
         ])
 
+    def test_process_from_remote_error_handling(self, worker, capsys):
+        worker.use_callback = True
+        worker.setup()
+        worker.slp.process_from_remote(('<nonono>', ()))
+        out, err = capsys.readouterr()
+        assert 'INTERNALERROR> ValueError: unknown event: <nonono>' in out
+        ev = worker.popevent()
+        assert ev.name == "errordown"
+
 
 def test_remote_env_vars(testdir):
     testdir.makepyfile('''
@@ -327,5 +372,5 @@ def test_remote_env_vars(testdir):
             assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     ''')
-    result = testdir.runpytest('-n2', '--max-slave-restart=0')
+    result = testdir.runpytest('-n2', '--max-worker-restart=0')
     assert result.ret == 0

testing/test_slavemanage.py

@@ -1,9 +1,11 @@
+from __future__ import print_function
 import py
 import pytest
+import textwrap
 import execnet
 from _pytest.pytester import HookRecorder
-from xdist import slavemanage, newhooks
+from xdist import workermanage, newhooks
-from xdist.slavemanage import HostRSync, NodeManager
+from xdist.workermanage import HostRSync, NodeManager
 
 pytest_plugins = "pytester"
 
@@ -32,7 +34,7 @@ def mysetup(tmpdir):
 
 
 @pytest.fixture
-def slavecontroller(monkeypatch):
+def workercontroller(monkeypatch):
     class MockController(object):
         def __init__(self, *args):
             pass
@@ -40,7 +42,7 @@ def slavecontroller(monkeypatch):
         def setup(self):
             pass
 
-    monkeypatch.setattr(slavemanage, 'SlaveController', MockController)
+    monkeypatch.setattr(workermanage, 'WorkerController', MockController)
     return MockController
 
 
@@ -50,14 +52,14 @@ class TestNodeManagerPopen:
         assert gm.specs[0].chdir is None
 
     def test_default_chdir(self, config):
-        l = ["ssh=noco", "socket=xyz"]
+        specs = ["ssh=noco", "socket=xyz"]
-        for spec in NodeManager(config, l).specs:
+        for spec in NodeManager(config, specs).specs:
             assert spec.chdir == "pyexecnetcache"
-        for spec in NodeManager(config, l, defaultchdir="abc").specs:
+        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
             assert spec.chdir == "abc"
 
     def test_popen_makegateway_events(self, config, hookrecorder,
-                                      slavecontroller):
+                                      workercontroller):
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
         call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -72,7 +74,7 @@ class TestNodeManagerPopen:
         hm.teardown_nodes()
         assert not len(hm.group)
 
-    def test_popens_rsync(self, config, mysetup, slavecontroller):
+    def test_popens_rsync(self, config, mysetup, workercontroller):
         source = mysetup.source
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
@@ -89,24 +91,24 @@ class TestNodeManagerPopen:
                     pass
 
             gw.remote_exec = pseudoexec
-        l = []
+        notifications = []
         for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: l.append(args))
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
-        assert not l
+        assert not notifications
         hm.teardown_nodes()
         assert not len(hm.group)
         assert "sys.path.insert" in gw.remote_exec.args[0]
 
-    def test_rsync_popen_with_path(self, config, mysetup, slavecontroller):
+    def test_rsync_popen_with_path(self, config, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
         hm.setup_nodes(None)
         source.ensure("dir1", "dir2", "hello")
-        l = []
+        notifications = []
         for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: l.append(args))
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
-        assert len(l) == 1
+        assert len(notifications) == 1
-        assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
+        assert notifications[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
         hm.teardown_nodes()
         dest = dest.join(source.basename)
         assert dest.join("dir1").check()
@@ -114,7 +116,7 @@ class TestNodeManagerPopen:
         assert dest.join("dir1", "dir2", 'hello').check()
 
     def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder,
-                                    slavecontroller):
+                                    workercontroller):
         source, dest = mysetup.source, mysetup.dest
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
         hm.roots = []
@@ -137,9 +139,9 @@ class TestHRSync:
         source.ensure(".somedotfile", "moreentries")
         source.ensure("somedir", "editfile~")
         syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        l = list(source.visit(rec=syncer.filter, fil=syncer.filter))
+        files = list(source.visit(rec=syncer.filter, fil=syncer.filter))
-        assert len(l) == 3
+        assert len(files) == 3
-        basenames = [x.basename for x in l]
+        basenames = [x.basename for x in files]
         assert 'dir' in basenames
         assert 'file.txt' in basenames
         assert 'somedir' in basenames
@@ -169,12 +171,12 @@ class TestNodeManager:
         p, = nodemanager.gwmanager.multi_exec(
             "import os ; channel.send(os.getcwd())").receive_each()
         p = py.path.local(p)
-        py.builtin.print_("remote curdir", p)
+        print("remote curdir", p)
         assert p == mysetup.dest.join(config.topdir.basename)
         assert p.join("dir1").check()
         assert p.join("dir1", "file1").check()
 
-    def test_popen_rsync_subdir(self, testdir, mysetup, slavecontroller):
+    def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir1 = mysetup.source.mkdir("dir1")
         dir2 = dir1.mkdir("dir2")
@@ -192,13 +194,13 @@ class TestNodeManager:
             assert dest.join("dir1", "dir2", 'hello').check()
             nodemanager.teardown_nodes()
 
-    def test_init_rsync_roots(self, testdir, mysetup, slavecontroller):
+    def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)
         source.ensure("dir1", "somefile", dir=1)
         dir2.ensure("hello")
         source.ensure("bogusdir", "file")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
+        source.join("tox.ini").write(textwrap.dedent("""
             [pytest]
             rsyncdirs=dir1/dir2
         """))
@@ -209,7 +211,7 @@ class TestNodeManager:
         assert not dest.join("dir1").check()
         assert not dest.join("bogus").check()
 
-    def test_rsyncignore(self, testdir, mysetup, slavecontroller):
+    def test_rsyncignore(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)
         source.ensure("dir5", "dir6", "bogus")
@@ -217,7 +219,7 @@ class TestNodeManager:
         dir2.ensure("hello")
         source.ensure("foo", "bar")
         source.ensure("bar", "foo")
-        source.join("tox.ini").write(py.std.textwrap.dedent("""
+        source.join("tox.ini").write(textwrap.dedent("""
             [pytest]
             rsyncdirs = dir1 dir5
             rsyncignore = dir1/dir2 dir5/dir6 foo*
@@ -233,7 +235,7 @@ class TestNodeManager:
         assert not dest.join('foo').check()
         assert not dest.join('bar').check()
 
-    def test_optimise_popen(self, testdir, mysetup, slavecontroller):
+    def test_optimise_popen(self, testdir, mysetup, workercontroller):
         source = mysetup.source
         specs = ["popen"] * 3
         source.join("conftest.py").write("rsyncdirs = ['a']")

tox.ini

@@ -1,8 +1,9 @@
 [tox]
 # if you change the envlist, please update .travis.yml file as well
 envlist=
-  py{26,27,34,35,36}-pytest{30}
+  py{27,34,35,36}-pytest{34,35,36}
-  py{27,35}-pytest{30}-pexpect
+  py{27,36}-pytest36-pexpect
+  py{27,36}-pytest{master,features}
   flakes
   readme
 
@@ -14,14 +15,18 @@ deps =
   pycmd
   # to avoid .eggs
   setuptools_scm
-  pytest30: pytest~=3.0.5
+  pytest34: pytest~=3.4.0
+  pytest35: pytest~=3.5.0
+  pytest36: pytest~=3.6.0
+  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
+  pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect
 platform=
   pexpect: linux|darwin
 commands=
   # always clean to avoid code unmarshal mismatch on old python/pytest
   py.cleanup -aq
-  py.test {posargs}
+  pytest {posargs}
 
 [testenv:flakes]
 changedir=
@@ -34,7 +39,18 @@ deps = readme
 skip_install = true
 commands = python setup.py check -r -s
 
+[testenv:release]
+changedir=
+decription = do a release, required posarg of the version number
+basepython = python3.6
+skipsdist = True
+usedevelop = True
+passenv = *
+deps =
+    towncrier
+commands =
+    towncrier --version {posargs} --yes
 
 [pytest]
 addopts = -rsfxX
-;; hello
+

xdist/boxed.py

@@ -1,60 +0,0 @@
-
-import py
-
-
-def pytest_addoption(parser):
-    group = parser.getgroup("xdist", "distributed and subprocess testing")
-    group.addoption(
-        '--boxed',
-        action="store_true", dest="boxed", default=False,
-        help="box each test run in a separate process (unix)")
-
-
-def pytest_runtest_protocol(item):
-    if item.config.getvalue("boxed"):
-        reports = forked_run_report(item)
-        for rep in reports:
-            item.ihook.pytest_runtest_logreport(report=rep)
-        return True
-
-
-def forked_run_report(item):
-    # for now, we run setup/teardown in the subprocess
-    # XXX optionally allow sharing of setup/teardown
-    from _pytest.runner import runtestprotocol
-    EXITSTATUS_TESTEXIT = 4
-    import marshal
-    from xdist.remote import serialize_report
-    from xdist.slavemanage import unserialize_report
-
-    def runforked():
-        try:
-            reports = runtestprotocol(item, log=False)
-        except KeyboardInterrupt:
-            py.std.os._exit(EXITSTATUS_TESTEXIT)
-        return marshal.dumps([serialize_report(x) for x in reports])
-
-    ff = py.process.ForkedFunc(runforked)
-    result = ff.waitfinish()
-    if result.retval is not None:
-        report_dumps = marshal.loads(result.retval)
-        return [unserialize_report("testreport", x) for x in report_dumps]
-    else:
-        if result.exitstatus == EXITSTATUS_TESTEXIT:
-            py.test.exit("forked test item %s raised Exit" % (item,))
-        return [report_process_crash(item, result)]
-
-
-def report_process_crash(item, result):
-    path, lineno = item._getfslineno()
-    info = ("%s:%s: running the test CRASHED with signal %d" %
-            (path, lineno, result.signal))
-    from _pytest import runner
-    call = runner.CallInfo(lambda: 0/0, "???")
-    call.excinfo = info
-    rep = runner.pytest_runtest_makereport(item, call)
-    if result.out:
-        rep.sections.append(("captured stdout", result.out))
-    if result.err:
-        rep.sections.append(("captured stderr", result.err))
-    return rep

xdist/dsession.py

@@ -1,462 +1,36 @@
-import difflib
-import itertools
-from _pytest.runner import CollectReport
-
-import pytest
 import py
-from xdist.slavemanage import NodeManager, parse_spec_config
+import pytest
 
+from xdist.workermanage import NodeManager
+from xdist.scheduler import (
+    EachScheduling,
+    LoadScheduling,
+    LoadScopeScheduling,
+    LoadFileScheduling,
+)
 
-queue = py.builtin._tryimport('queue', 'Queue')
 
-
+from six.moves.queue import Empty, Queue
-class EachScheduling:
-    """Implement scheduling of test items on all nodes
-
-    If a node gets added after the test run is started then it is
-    assumed to replace a node which got removed before it finished
-    its collection.  In this case it will only be used if a node
-    with the same spec got removed earlier.
-
-    Any nodes added after the run is started will only get items
-    assigned if a node with a matching spec was removed before it
-    finished all its pending items.  The new node will then be
-    assigned the remaining items from the removed node.
-    """
-
-    def __init__(self, config, log=None):
-        self.config = config
-        self.numnodes = len(parse_spec_config(config))
-        self.node2collection = {}
-        self.node2pending = {}
-        self._started = []
-        self._removed2pending = {}
-        if log is None:
-            self.log = py.log.Producer("eachsched")
-        else:
-            self.log = log.eachsched
-        self.collection_is_completed = False
-
-    @property
-    def nodes(self):
-        """A list of all nodes in the scheduler."""
-        return list(self.node2pending.keys())
-
-    @property
-    def tests_finished(self):
-        if not self.collection_is_completed:
-            return False
-        if self._removed2pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    @property
-    def has_pending(self):
-        """Return True if there are pending test items
-
-        This indicates that collection has finished and nodes are
-        still processing test items, so this can be thought of as
-        "the scheduler is active".
-        """
-        for pending in self.node2pending.values():
-            if pending:
-                return True
-        return False
-
-    def add_node(self, node):
-        assert node not in self.node2pending
-        self.node2pending[node] = []
-
-    def add_node_collection(self, node, collection):
-        """Add the collected test items from a node
-
-        Collection is complete once all nodes have submitted their
-        collection.  In this case its pending list is set to an empty
-        list.  When the collection is already completed this
-        submission is from a node which was restarted to replace a
-        dead node.  In this case we already assign the pending items
-        here.  In either case ``.schedule()`` will instruct the
-        node to start running the required tests.
-        """
-        assert node in self.node2pending
-        if not self.collection_is_completed:
-            self.node2collection[node] = list(collection)
-            self.node2pending[node] = []
-            if len(self.node2collection) >= self.numnodes:
-                self.collection_is_completed = True
-        elif self._removed2pending:
-            for deadnode in self._removed2pending:
-                if deadnode.gateway.spec == node.gateway.spec:
-                    dead_collection = self.node2collection[deadnode]
-                    if collection != dead_collection:
-                        msg = report_collection_diff(dead_collection,
-                                                     collection,
-                                                     deadnode.gateway.id,
-                                                     node.gateway.id)
-                        self.log(msg)
-                        return
-                    pending = self._removed2pending.pop(deadnode)
-                    self.node2pending[node] = pending
-                    break
-
-    def mark_test_complete(self, node, item_index, duration=0):
-        self.node2pending[node].remove(item_index)
-
-    def remove_node(self, node):
-        # KeyError if we didn't get an add_node() yet
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-        crashitem = self.node2collection[node][pending.pop(0)]
-        if pending:
-            self._removed2pending[node] = pending
-        return crashitem
-
-    def schedule(self):
-        """Schedule the test items on the nodes
-
-        If the node's pending list is empty it is a new node which
-        needs to run all the tests.  If the pending list is already
-        populated (by ``.add_node_collection()``) then it replaces a
-        dead node and we only need to run those tests.
-        """
-        assert self.collection_is_completed
-        for node, pending in self.node2pending.items():
-            if node in self._started:
-                continue
-            if not pending:
-                pending[:] = range(len(self.node2collection[node]))
-                node.send_runtest_all()
-            else:
-                node.send_runtest_some(pending)
-            self._started.append(node)
-
-
-class LoadScheduling:
-    """Implement load scheduling across nodes.
-
-    This distributes the tests collected across all nodes so each test
-    is run just once.  All nodes collect and submit the test suite and
-    when all collections are received it is verified they are
-    identical collections.  Then the collection gets divided up in
-    chunks and chunks get submitted to nodes.  Whenever a node finishes
-    an item, it calls ``.mark_test_complete()`` which will trigger the
-    scheduler to assign more tests if the number of pending tests for
-    the node falls below a low-watermark.
-
-    When created, ``numnodes`` defines how many nodes are expected to
-    submit a collection. This is used to know when all nodes have
-    finished collection or how large the chunks need to be created.
-
-    Attributes:
-
-    :numnodes: The expected number of nodes taking part.  The actual
-       number of nodes will vary during the scheduler's lifetime as
-       nodes are added by the DSession as they are brought up and
-       removed either because of a dead node or normal shutdown.  This
-       number is primarily used to know when the initial collection is
-       completed.
-
-    :node2collection: Map of nodes and their test collection.  All
-       collections should always be identical.
-
-    :node2pending: Map of nodes and the indices of their pending
-       tests.  The indices are an index into ``.pending`` (which is
-       identical to their own collection stored in
-       ``.node2collection``).
-
-    :collection: The one collection once it is validated to be
-       identical between all the nodes.  It is initialised to None
-       until ``.schedule()`` is called.
-
-    :pending: List of indices of globally pending tests.  These are
-       tests which have not yet been allocated to a chunk for a node
-       to process.
-
-    :log: A py.log.Producer instance.
-
-    :config: Config object, used for handling hooks.
-    """
-
-    def __init__(self, config, log=None):
-        self.numnodes = len(parse_spec_config(config))
-        self.node2collection = {}
-        self.node2pending = {}
-        self.pending = []
-        self.collection = None
-        if log is None:
-            self.log = py.log.Producer("loadsched")
-        else:
-            self.log = log.loadsched
-        self.config = config
-
-    @property
-    def nodes(self):
-        """A list of all nodes in the scheduler."""
-        return list(self.node2pending.keys())
-
-    @property
-    def collection_is_completed(self):
-        """Boolean indication initial test collection is complete.
-
-        This is a boolean indicating all initial participating nodes
-        have finished collection.  The required number of initial
-        nodes is defined by ``.numnodes``.
-        """
-        return len(self.node2collection) >= self.numnodes
-
-    @property
-    def tests_finished(self):
-        """Return True if all tests have been executed by the nodes."""
-        if not self.collection_is_completed:
-            return False
-        if self.pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    @property
-    def has_pending(self):
-        """Return True if there are pending test items
-
-        This indicates that collection has finished and nodes are
-        still processing test items, so this can be thought of as
-        "the scheduler is active".
-        """
-        if self.pending:
-            return True
-        for pending in self.node2pending.values():
-            if pending:
-                return True
-        return False
-
-    def add_node(self, node):
-        """Add a new node to the scheduler.
-
-        From now on the node will be allocated chunks of tests to
-        execute.
-
-        Called by the ``DSession.slave_slaveready`` hook when it
-        successfully bootstraps a new node.
-        """
-        assert node not in self.node2pending
-        self.node2pending[node] = []
-
-    def add_node_collection(self, node, collection):
-        """Add the collected test items from a node
-
-        The collection is stored in the ``.node2collection`` map.
-        Called by the ``DSession.slave_collectionfinish`` hook.
-        """
-        assert node in self.node2pending
-        if self.collection_is_completed:
-            # A new node has been added later, perhaps an original one died.
-            # .schedule() should have
-            # been called by now
-            assert self.collection
-            if collection != self.collection:
-                other_node = next(iter(self.node2collection.keys()))
-                msg = report_collection_diff(self.collection,
-                                             collection,
-                                             other_node.gateway.id,
-                                             node.gateway.id)
-                self.log(msg)
-                return
-        self.node2collection[node] = list(collection)
-
-    def mark_test_complete(self, node, item_index, duration=0):
-        """Mark test item as completed by node
-
-        The duration it took to execute the item is used as a hint to
-        the scheduler.
-
-        This is called by the ``DSession.slave_testreport`` hook.
-        """
-        self.node2pending[node].remove(item_index)
-        self.check_schedule(node, duration=duration)
-
-    def check_schedule(self, node, duration=0):
-        """Maybe schedule new items on the node
-
-        If there are any globally pending nodes left then this will
-        check if the given node should be given any more tests.  The
-        ``duration`` of the last test is optionally used as a
-        heuristic to influence how many tests the node is assigned.
-        """
-        if node.shutting_down:
-            return
-
-        if self.pending:
-            # how many nodes do we have?
-            num_nodes = len(self.node2pending)
-            # if our node goes below a heuristic minimum, fill it out to
-            # heuristic maximum
-            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
-            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
-            node_pending = self.node2pending[node]
-            if len(node_pending) < items_per_node_min:
-                if duration >= 0.1 and len(node_pending) >= 2:
-                    # seems the node is doing long-running tests
-                    # and has enough items to continue
-                    # so let's rather wait with sending new items
-                    return
-                num_send = items_per_node_max - len(node_pending)
-                self._send_tests(node, num_send)
-        self.log("num items waiting for node:", len(self.pending))
-
-    def remove_node(self, node):
-        """Remove a node from the scheduler
-
-        This should be called either when the node crashed or at
-        shutdown time.  In the former case any pending items assigned
-        to the node will be re-scheduled.  Called by the
-        ``DSession.slave_slavefinished`` and
-        ``DSession.slave_errordown`` hooks.
-
-        Return the item which was being executing while the node
-        crashed or None if the node has no more pending items.
-
-        """
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-
-        # The node crashed, reassing pending items
-        crashitem = self.collection[pending.pop(0)]
-        self.pending.extend(pending)
-        for node in self.node2pending:
-            self.check_schedule(node)
-        return crashitem
-
-    def schedule(self):
-        """Initiate distribution of the test collection
-
-        Initiate scheduling of the items across the nodes.  If this
-        gets called again later it behaves the same as calling
-        ``.check_schedule()`` on all nodes so that newly added nodes
-        will start to be used.
-
-        This is called by the ``DSession.slave_collectionfinish`` hook
-        if ``.collection_is_completed`` is True.
-        """
-        assert self.collection_is_completed
-
-        # Initial distribution already happened, reschedule on all nodes
-        if self.collection is not None:
-            for node in self.nodes:
-                self.check_schedule(node)
-            return
-
-        # XXX allow nodes to have different collections
-        if not self._check_nodes_have_same_collection():
-            self.log('**Different tests collected, aborting run**')
-            return
-
-        # Collections are identical, create the index of pending items.
-        self.collection = list(self.node2collection.values())[0]
-        self.pending[:] = range(len(self.collection))
-        if not self.collection:
-            return
-
-        # Send a batch of tests to run. If we don't have at least two
-        # tests per node, we have to send them all so that we can send
-        # shutdown signals and get all nodes working.
-        initial_batch = max(len(self.pending) // 4,
-                            2 * len(self.nodes))
-
-        # distribute tests round-robin up to the batch size
-        # (or until we run out)
-        nodes = itertools.cycle(self.nodes)
-        for i in range(initial_batch):
-            self._send_tests(next(nodes), 1)
-
-        if not self.pending:
-            # initial distribution sent all tests, start node shutdown
-            for node in self.nodes:
-                node.shutdown()
-
-    def _send_tests(self, node, num):
-        tests_per_node = self.pending[:num]
-        if tests_per_node:
-            del self.pending[:num]
-            self.node2pending[node].extend(tests_per_node)
-            node.send_runtest_some(tests_per_node)
-
-    def _check_nodes_have_same_collection(self):
-        """Return True if all nodes have collected the same items.
-
-        If collections differ, this method returns False while logging
-        the collection differences and posting collection errors to
-        pytest_collectreport hook.
-        """
-        node_collection_items = list(self.node2collection.items())
-        first_node, col = node_collection_items[0]
-        same_collection = True
-        for node, collection in node_collection_items[1:]:
-            msg = report_collection_diff(
-                col,
-                collection,
-                first_node.gateway.id,
-                node.gateway.id,
-            )
-            if msg:
-                same_collection = False
-                self.log(msg)
-                if self.config is not None:
-                    rep = CollectReport(
-                        node.gateway.id, 'failed',
-                        longrepr=msg, result=[])
-                    self.config.hook.pytest_collectreport(report=rep)
-
-        return same_collection
-
-
-def report_collection_diff(from_collection, to_collection, from_id, to_id):
-    """Report the collected test difference between two nodes.
-
-    :returns: detailed message describing the difference between the given
-    collections, or None if they are equal.
-    """
-    if from_collection == to_collection:
-        return None
-
-    diff = difflib.unified_diff(
-        from_collection,
-        to_collection,
-        fromfile=from_id,
-        tofile=to_id,
-    )
-    error_message = py.builtin._totext(
-        'Different tests were collected between {from_id} and {to_id}. '
-        'The difference is:\n'
-        '{diff}'
-    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
-    return msg
 
 
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
 
 
-class DSession:
+class DSession(object):
-    """A py.test plugin which runs a distributed test session
+    """A pytest plugin which runs a distributed test session
 
     At the beginning of the test session this creates a NodeManager
     instance which creates and starts all nodes.  Nodes then emit
-    events processed in the pytest_runtestloop hook using the slave_*
+    events processed in the pytest_runtestloop hook using the worker_*
     methods.
 
     Once a node is started it will automatically start running the
-    py.test mainloop with some custom hooks.  This means a node
+    pytest mainloop with some custom hooks.  This means a node
     automatically starts collecting tests.  Once tests are collected
     it will wait for instructions.
     """
+
     def __init__(self, config):
         self.config = config
         self.log = py.log.Producer("dsession")
@@ -467,14 +41,14 @@ class DSession:
         self.shuttingdown = False
         self.countfailures = 0
         self.maxfail = config.getvalue("maxfail")
-        self.queue = queue.Queue()
+        self.queue = Queue()
         self._session = None
         self._failed_collection_errors = {}
         self._active_nodes = set()
         self._failed_nodes_count = 0
-        self._max_slave_restart = self.config.getoption('max_slave_restart')
+        self._max_worker_restart = self.config.option.maxworkerrestart
-        if self._max_slave_restart is not None:
+        if self._max_worker_restart is not None:
-            self._max_slave_restart = int(self._max_slave_restart)
+            self._max_worker_restart = int(self._max_worker_restart)
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -501,7 +75,7 @@ class DSession:
         """Creates and starts the nodes.
 
         The nodes are setup to put their events onto self.queue.  As
-        soon as nodes start they will emit the slave_slaveready event.
+        soon as nodes start they will emit the worker_workerready event.
         """
         self.nodemanager = NodeManager(self.config)
         nodes = self.nodemanager.setup_nodes(putevent=self.queue.put)
@@ -521,11 +95,14 @@ class DSession:
 
     @pytest.mark.trylast
     def pytest_xdist_make_scheduler(self, config, log):
-        dist = config.getvalue("dist")
+        dist = config.getvalue('dist')
-        if dist == "load":
+        schedulers = {
-            return LoadScheduling(config, log)
+            'each': EachScheduling,
-        elif dist == "each":
+            'load': LoadScheduling,
-            return EachScheduling(config, log)
+            'loadscope': LoadScopeScheduling,
+            'loadfile': LoadFileScheduling,
+        }
+        return schedulers[dist](config, log)
 
     def pytest_runtestloop(self):
         self.sched = self.config.hook.pytest_xdist_make_scheduler(
@@ -543,16 +120,20 @@ class DSession:
         return True
 
     def loop_once(self):
-        """Process one callback from one of the slaves."""
+        """Process one callback from one of the workers."""
         while 1:
+            if not self._active_nodes:
+                # If everything has died stop looping
+                self.triggershutdown()
+                raise RuntimeError("Unexpectedly no active workers available")
             try:
                 eventcall = self.queue.get(timeout=2.0)
                 break
-            except queue.Empty:
+            except Empty:
                 continue
         callname, kwargs = eventcall
         assert callname, kwargs
-        method = "slave_" + callname
+        method = "worker_" + callname
         call = getattr(self, method)
         self.log("calling method", method, kwargs)
         call(**kwargs)
@@ -560,44 +141,48 @@ class DSession:
             self.triggershutdown()
 
     #
-    # callbacks for processing events from slaves
+    # callbacks for processing events from workers
     #
 
-    def slave_slaveready(self, node, slaveinfo):
+    def worker_workerready(self, node, workerinfo):
         """Emitted when a node first starts up.
 
         This adds the node to the scheduler, nodes continue with
         collection without any further input.
         """
-        node.slaveinfo = slaveinfo
+        node.workerinfo = workerinfo
-        node.slaveinfo['id'] = node.gateway.id
+        node.workerinfo['id'] = node.gateway.id
-        node.slaveinfo['spec'] = node.gateway.spec
+        node.workerinfo['spec'] = node.gateway.spec
+
+        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
+        node.slaveinfo = node.workerinfo
+
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
         else:
             self.sched.add_node(node)
 
-    def slave_slavefinished(self, node):
+    def worker_workerfinished(self, node):
         """Emitted when node executes its pytest_sessionfinish hook.
 
         Removes the node from the scheduler.
 
         The node might not be in the scheduler if it had not emitted
-        slaveready before shutdown was triggered.
+        workerready before shutdown was triggered.
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
-        if node.slaveoutput['exitstatus'] == 2:  # keyboard-interrupt
+        if node.workeroutput['exitstatus'] == 2:  # keyboard-interrupt
             self.shouldstop = "%s received keyboard-interrupt" % (node,)
-            self.slave_errordown(node, "keyboard-interrupt")
+            self.worker_errordown(node, "keyboard-interrupt")
             return
         if node in self.sched.nodes:
             crashitem = self.sched.remove_node(node)
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
-    def slave_errordown(self, node, error):
+    def worker_errordown(self, node, error):
-        """Emitted by the SlaveController when a node dies."""
+        """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
         try:
             crashitem = self.sched.remove_node(node)
@@ -608,22 +193,22 @@ class DSession:
                 self.handle_crashitem(crashitem, node)
 
         self._failed_nodes_count += 1
-        maximum_reached = (self._max_slave_restart is not None and
+        maximum_reached = (self._max_worker_restart is not None and
-                           self._failed_nodes_count > self._max_slave_restart)
+                           self._failed_nodes_count > self._max_worker_restart)
         if maximum_reached:
-            if self._max_slave_restart == 0:
+            if self._max_worker_restart == 0:
-                msg = 'Slave restarting disabled'
+                msg = 'Worker restarting disabled'
             else:
-                msg = "Maximum crashed slaves reached: %d" % \
+                msg = "Maximum crashed workers reached: %d" % \
-                      self._max_slave_restart
+                      self._max_worker_restart
             self.report_line(msg)
         else:
-            self.report_line("Replacing crashed slave %s" % node.gateway.id)
+            self.report_line("Replacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
-    def slave_collectionfinish(self, node, ids):
+    def worker_collectionfinish(self, node, ids):
-        """Slave has finished test collection.
+        """worker has finished test collection.
 
         This adds the collection for this node to the scheduler.  If
         the scheduler indicates collection is finished (i.e. all
@@ -649,28 +234,39 @@ class DSession:
                     self.sched.__class__.__name__))
             self.sched.schedule()
 
-    def slave_logstart(self, node, nodeid, location):
+    def worker_logstart(self, node, nodeid, location):
         """Emitted when a node calls the pytest_runtest_logstart hook."""
         self.config.hook.pytest_runtest_logstart(
             nodeid=nodeid, location=location)
 
-    def slave_testreport(self, node, rep):
+    def worker_logfinish(self, node, nodeid, location):
-        """Emitted when a node calls the pytest_runtest_logreport hook.
+        """Emitted when a node calls the pytest_runtest_logfinish hook."""
+        self.config.hook.pytest_runtest_logfinish(
+            nodeid=nodeid, location=location)
 
-        If the node indicates it is finished with a test item, remove
+    def worker_testreport(self, node, rep):
-        the item from the pending list in the scheduler.
+        """Emitted when a node calls the pytest_runtest_logreport hook."""
-        """
-        if rep.when == "call" or (rep.when == "setup" and not rep.passed):
-            self.sched.mark_test_complete(node, rep.item_index, rep.duration)
-        # self.report_line("testreport %s: %s" %(rep.id, rep.status))
         rep.node = node
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
-    def slave_collectreport(self, node, rep):
+    def worker_runtest_protocol_complete(self, node, item_index, duration):
+        """
+        Emitted when a node fires the 'runtest_protocol_complete' event,
+        signalling that a test has completed the runtestprotocol and should be
+        removed from the pending list in the scheduler.
+        """
+        self.sched.mark_test_complete(node, item_index, duration)
+
+    def worker_collectreport(self, node, rep):
         """Emitted when a node calls the pytest_collectreport hook."""
         if rep.failed:
-            self._failed_slave_collectreport(node, rep)
+            self._failed_worker_collectreport(node, rep)
+
+    def worker_logwarning(self, message, code, nodeid, fslocation):
+        """Emitted when a node calls the pytest_logwarning hook."""
+        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
+        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
     def _clone_node(self, node):
         """Return new node based on an existing one.
@@ -678,7 +274,7 @@ class DSession:
         This is normally for when a node dies, this will copy the spec
         of the existing node and create a new one with a new id.  The
         new node will have been setup so it will start calling the
-        "slave_*" hooks and do work soon.
+        "worker_*" hooks and do work soon.
         """
         spec = node.gateway.spec
         spec.id = None
@@ -687,9 +283,9 @@ class DSession:
         self._active_nodes.add(node)
         return node
 
-    def _failed_slave_collectreport(self, node, rep):
+    def _failed_worker_collectreport(self, node, rep):
         # Check we haven't already seen this report (from
-        # another slave).
+        # another worker).
         if rep.longrepr not in self._failed_collection_errors:
             self._failed_collection_errors[rep.longrepr] = True
             self.config.hook.pytest_collectreport(report=rep)
@@ -708,19 +304,19 @@ class DSession:
         for node in self.sched.nodes:
             node.shutdown()
 
-    def handle_crashitem(self, nodeid, slave):
+    def handle_crashitem(self, nodeid, worker):
         # XXX get more reporting info by recording pytest_runtest_logstart?
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Slave %r crashed while running %r" % (slave.gateway.id, nodeid)
+        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
         rep = runner.TestReport(nodeid, (fspath, None, fspath),
                                 (), "failed", msg, "???")
-        rep.node = slave
+        rep.node = worker
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
-class TerminalDistReporter:
+class TerminalDistReporter(object):
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")
@@ -746,7 +342,7 @@ class TerminalDistReporter:
         return " / ".join(parts)
 
     def rewrite(self, line, newline=False):
-        pline = line + " " * max(self._lastlen-len(line), 0)
+        pline = line + " " * max(self._lastlen - len(line), 0)
         if newline:
             self._lastlen = 0
             pline += "\n"
@@ -772,7 +368,7 @@ class TerminalDistReporter:
 
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
-            d = node.slaveinfo
+            d = node.workerinfo
             infoline = "[%s] Python %s" % (
                 d['id'],
                 d['version'].replace('\n', ' -- '),)

xdist/looponfail.py

@@ -1,15 +1,16 @@
 """
-    Implement -f aka looponfailing for py.test.
+    Implement -f aka looponfailing for pytest.
 
     NOTE that we try to avoid loading and depending on application modules
     within the controlling process (the one that starts repeatedly test
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-
+from __future__ import print_function
 import py
 import pytest
 import sys
+import time
 import execnet
 
 
@@ -59,7 +60,7 @@ class RemoteControl(object):
     def trace(self, *args):
         if self.config.option.debug:
             msg = " ".join([str(x) for x in args])
-            py.builtin.print_("RemoteControl:", msg)
+            print("RemoteControl:", msg)
 
     def initgateway(self):
         return execnet.makegateway("popen")
@@ -69,10 +70,10 @@ class RemoteControl(object):
             out = py.io.TerminalWriter()
         if hasattr(self, 'gateway'):
             raise ValueError("already have gateway %r" % self.gateway)
-        self.trace("setting up slave session")
+        self.trace("setting up worker session")
         self.gateway = self.initgateway()
         self.channel = channel = self.gateway.remote_exec(
-            init_slave_session,
+            init_worker_session,
             args=self.config.args,
             option_dict=vars(self.config.option),
         )
@@ -134,7 +135,7 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
         tr.line("### Watching:   %s" % (rootdir,), bold=True)
 
 
-def init_slave_session(channel, args, option_dict):
+def init_worker_session(channel, args, option_dict):
     import os
     import sys
     outchannel = channel.gateway.newchannel()
@@ -153,11 +154,11 @@ def init_slave_session(channel, args, option_dict):
     from _pytest.config import Config
     config = Config.fromdictargs(option_dict, list(args))
     config.args = args
-    from xdist.looponfail import SlaveFailSession
+    from xdist.looponfail import WorkerFailSession
-    SlaveFailSession(config, channel).main()
+    WorkerFailSession(config, channel).main()
 
 
-class SlaveFailSession:
+class WorkerFailSession(object):
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -194,11 +195,11 @@ class SlaveFailSession:
             self.collection_failed = True
 
     def main(self):
-        self.DEBUG("SLAVE: received configuration, waiting for command trails")
+        self.DEBUG("WORKER: received configuration, waiting for command trails")
         try:
             command = self.channel.receive()
         except KeyboardInterrupt:
-            return  # in the slave we can't do much about this
+            return  # in the worker we can't do much about this
         self.DEBUG("received", command)
         self.current_command = command
         self.config.hook.pytest_cmdline_main(config=self.config)
@@ -211,7 +212,7 @@ class SlaveFailSession:
         self.channel.send((trails, failreports, self.collection_failed))
 
 
-class StatRecorder:
+class StatRecorder(object):
     def __init__(self, rootdirlist):
         self.rootdirlist = rootdirlist
         self.statcache = {}
@@ -228,7 +229,7 @@ class StatRecorder:
             changed = self.check()
             if changed:
                 return
-            py.std.time.sleep(checkinterval)
+            time.sleep(checkinterval)
 
     def check(self, removepycfiles=True):  # noqa, too complex
         changed = False
@@ -247,7 +248,7 @@ class StatRecorder:
                         if oldstat.mtime != curstat.mtime or \
                            oldstat.size != curstat.size:
                             changed = True
-                            py.builtin.print_("# MODIFIED", path)
+                            print("# MODIFIED", path)
                             if removepycfiles and path.ext == ".py":
                                 pycfile = path + "c"
                                 if pycfile.check():

xdist/plugin.py

@@ -1,18 +1,34 @@
+import os
+
 import py
 import pytest
 
 
-def parse_numprocesses(s):
+def auto_detect_cpus():
-    if s == 'auto':
+    try:
+        from os import sched_getaffinity
+    except ImportError:
+        if os.environ.get('TRAVIS') == 'true':
+            # workaround https://bitbucket.org/pypy/pypy/issues/2375
+            return 2
         try:
             from os import cpu_count
         except ImportError:
             from multiprocessing import cpu_count
-        try:
+    else:
-            n = cpu_count()
+        def cpu_count():
-        except NotImplementedError:
+            return len(sched_getaffinity(0))
-            return 1
+
-        return n if n else 1
+    try:
+        n = cpu_count()
+    except NotImplementedError:
+        return 1
+    return n if n else 1
+
+
+def parse_numprocesses(s):
+    if s == 'auto':
+        return auto_detect_cpus()
     else:
         return int(s)
 
@@ -26,18 +42,26 @@ def pytest_addoption(parser):
         help="shortcut for '--dist=load --tx=NUM*popen', "
              "you can use 'auto' here for auto detection CPUs number on "
              "host system")
-    group._addoption('--max-slave-restart', action="store", default=None,
+    group.addoption('--max-worker-restart', '--max-slave-restart', action="store", default=None,
-                     help="maximum number of slaves that can be restarted "
+                    dest="maxworkerrestart",
-                          "when crashed (set to zero to disable this feature)")
+                    help="maximum number of workers that can be restarted "
-    group._addoption(
+                         "when crashed (set to zero to disable this feature)\n"
+                         "'--max-slave-restart' option is deprecated and will be removed in "
+                         "a future release")
+    group.addoption(
         '--dist', metavar="distmode",
-        action="store", choices=['load', 'each', 'no'],
+        action="store", choices=['each', 'load', 'loadscope', 'loadfile', 'no'],
         dest="dist", default="no",
         help=("set mode for distributing tests to exec environments.\n\n"
-              "each: send each test to each available environment.\n\n"
+              "each: send each test to all available environments.\n\n"
-              "load: send each test to available environment.\n\n"
+              "load: load balance by sending any pending test to any"
+              " available environment.\n\n"
+              "loadscope: load balance by sending pending groups of tests in"
+              " the same scope to any available environment.\n\n"
+              "loadfile: load balance by sending test grouped by file"
+              " to any available environment.\n\n"
               "(default) no: run tests inprocess, don't distribute."))
-    group._addoption(
+    group.addoption(
         '--tx', dest="tx", action="append", default=[],
         metavar="xspec",
         help=("add a test execution environment. some examples: "
@@ -54,6 +78,9 @@ def pytest_addoption(parser):
         '--rsyncignore', action="append", default=[], metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.")
 
+    group.addoption(
+        "--boxed", action="store_true",
+        help="backward compatibility alias for pytest-forked --forked")
     parser.addini(
         'rsyncdirs', 'list of (relative) paths to be rsynced for'
         ' remote distributed testing.', type="pathlist")
@@ -64,6 +91,7 @@ def pytest_addoption(parser):
         "looponfailroots", type="pathlist",
         help="directories to check for changes", default=[py.path.local()])
 
+
 # -------------------------------------------------------------------------
 # distributed testing hooks
 # -------------------------------------------------------------------------
@@ -84,18 +112,21 @@ def pytest_addhooks(pluginmanager):
 
 @pytest.mark.trylast
 def pytest_configure(config):
-    if config.getoption("dist") != "no":
+    if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
         from xdist.dsession import DSession
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
         tr = config.pluginmanager.getplugin("terminalreporter")
         tr.showfspath = False
+    if config.getoption("boxed"):
+        config.option.forked = True
 
 
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
     if config.option.numprocesses:
-        config.option.dist = "load"
+        if config.option.dist == 'no':
+            config.option.dist = "load"
         config.option.tx = ['popen'] * config.option.numprocesses
     if config.option.distload:
         config.option.dist = "load"
@@ -105,7 +136,7 @@ def pytest_cmdline_main(config):
         if val("dist") != "no":
             if usepdb:
                 raise pytest.UsageError(
-                    "--pdb incompatible with distributing tests.")
+                    "--pdb is incompatible with distributing tests; try using -n0.")  # noqa: E501
 
 # -------------------------------------------------------------------------
 # fixtures
@@ -114,7 +145,10 @@ def pytest_cmdline_main(config):
 
 @pytest.fixture(scope="session")
 def worker_id(request):
-    if hasattr(request.config, 'slaveinput'):
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
-        return request.config.slaveinput['slaveid']
+    if running on the master node.
+    """
+    if hasattr(request.config, 'workerinput'):
+        return request.config.workerinput['workerid']
     else:
         return 'master'

xdist/remote.py

@@ -8,13 +8,17 @@
 
 import sys
 import os
+import time
+
+import _pytest.hookspec
+import pytest
 
 
-class SlaveInteractor:
+class WorkerInteractor(object):
     def __init__(self, config, channel):
         self.config = config
-        self.slaveid = config.slaveinput.get('slaveid', "?")
+        self.workerid = config.workerinput.get('workerid', "?")
-        self.log = py.log.Producer("slave-%s" % self.slaveid)
+        self.log = py.log.Producer("worker-%s" % self.workerid)
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
         self.channel = channel
@@ -30,14 +34,14 @@ class SlaveInteractor:
 
     def pytest_sessionstart(self, session):
         self.session = session
-        slaveinfo = getinfodict()
+        workerinfo = getinfodict()
-        self.sendevent("slaveready", slaveinfo=slaveinfo)
+        self.sendevent("workerready", workerinfo=workerinfo)
 
-    def pytest_sessionfinish(self, __multicall__, exitstatus):
+    @pytest.hookimpl(hookwrapper=True)
-        self.config.slaveoutput['exitstatus'] = exitstatus
+    def pytest_sessionfinish(self, exitstatus):
-        res = __multicall__.execute()
+        self.config.workeroutput['exitstatus'] = exitstatus
-        self.sendevent("slavefinished", slaveoutput=self.config.slaveoutput)
+        yield
-        return res
+        self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
     def pytest_collection(self, session):
         self.sendevent("collectionstart")
@@ -58,23 +62,29 @@ class SlaveInteractor:
             self.log("items to run:", torun)
             # only run if we have an item and a next item
             while len(torun) >= 2:
-                self.run_tests(torun)
+                self.run_one_test(torun)
             if name == "shutdown":
                 if torun:
-                    self.run_tests(torun)
+                    self.run_one_test(torun)
                 break
         return True
 
-    def run_tests(self, torun):
+    def run_one_test(self, torun):
         items = self.session.items
         self.item_index = torun.pop(0)
+        item = items[self.item_index]
         if torun:
             nextitem = items[torun[0]]
         else:
             nextitem = None
+
+        start = time.time()
         self.config.hook.pytest_runtest_protocol(
-            item=items[self.item_index],
+            item=item,
             nextitem=nextitem)
+        duration = time.time() - start
+        self.sendevent("runtest_protocol_complete", item_index=self.item_index,
+                       duration=duration)
 
     def pytest_collection_finish(self, session):
         self.sendevent(
@@ -85,10 +95,15 @@ class SlaveInteractor:
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
+    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
+    if hasattr(_pytest.hookspec, 'pytest_runtest_logfinish'):
+        def pytest_runtest_logfinish(self, nodeid, location):
+            self.sendevent("logfinish", nodeid=nodeid, location=location)
+
     def pytest_runtest_logreport(self, report):
         data = serialize_report(report)
         data["item_index"] = self.item_index
-        data["worker_id"] = self.slaveid
+        data["worker_id"] = self.workerid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
@@ -96,19 +111,26 @@ class SlaveInteractor:
         data = serialize_report(report)
         self.sendevent("collectreport", data=data)
 
+    def pytest_logwarning(self, message, code, nodeid, fslocation):
+        self.sendevent("logwarning", message=message, code=code, nodeid=nodeid,
+                       fslocation=str(fslocation))
+
 
 def serialize_report(rep):
     def disassembled_report(rep):
-        reprtraceback = rep.longrepr.reprtraceback.__dict__
+        reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
-        reprcrash = rep.longrepr.reprcrash.__dict__
+        reprcrash = rep.longrepr.reprcrash.__dict__.copy()
 
         new_entries = []
         for entry in reprtraceback['reprentries']:
-            new_entry = entry.__dict__
+            entry_data = {
-            for key, value in new_entry.items():
+                'type': type(entry).__name__,
+                'data': entry.__dict__.copy(),
+            }
+            for key, value in entry_data['data'].items():
                 if hasattr(value, '__dict__'):
-                    new_entry[key] = value.__dict__
+                    entry_data['data'][key] = value.__dict__.copy()
-            new_entries.append(new_entry)
+            new_entries.append(entry_data)
 
         reprtraceback['reprentries'] = new_entries
 
@@ -163,22 +185,21 @@ def remote_initconfig(option_dict, args):
 
 if __name__ == '__channelexec__':
     channel = channel  # noqa
-    # python3.2 is not concurrent import safe, so let's play it safe
+    workerinput, args, option_dict = channel.receive()
-    # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32
-    if sys.version_info[:2] == (3, 2):
-        os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
-    slaveinput, args, option_dict = channel.receive()
     importpath = os.getcwd()
     sys.path.insert(0, importpath)  # XXX only for remote situations
     os.environ['PYTHONPATH'] = (
         importpath + os.pathsep +
         os.environ.get('PYTHONPATH', ''))
-    os.environ['PYTEST_XDIST_WORKER'] = slaveinput['slaveid']
+    os.environ['PYTEST_XDIST_WORKER'] = workerinput['workerid']
-    os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(slaveinput['slavecount'])
+    os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(workerinput['workercount'])
     # os.environ['PYTHONPATH'] = importpath
     import py
     config = remote_initconfig(option_dict, args)
-    config.slaveinput = slaveinput
+    config.workerinput = workerinput
-    config.slaveoutput = {}
+    config.workeroutput = {}
-    interactor = SlaveInteractor(config, channel)
+    # TODO: deprecated name, backward compatibility only. Remove it in future
+    config.slaveinput = config.workerinput
+    config.slaveoutput = config.workeroutput
+    interactor = WorkerInteractor(config, channel)
     config.hook.pytest_cmdline_main(config=config)

xdist/report.py

@@ -0,0 +1,26 @@
+from __future__ import print_function
+from difflib import unified_diff
+
+
+def report_collection_diff(from_collection, to_collection, from_id, to_id):
+    """Report the collected test difference between two nodes.
+
+    :returns: detailed message describing the difference between the given
+    collections, or None if they are equal.
+    """
+    if from_collection == to_collection:
+        return None
+
+    diff = unified_diff(
+        from_collection,
+        to_collection,
+        fromfile=from_id,
+        tofile=to_id,
+    )
+    error_message = (
+        u'Different tests were collected between {from_id} and {to_id}. '
+        u'The difference is:\n'
+        u'{diff}'
+    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
+    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    return msg

xdist/scheduler/__init__.py

@@ -0,0 +1,4 @@
+from xdist.scheduler.each import EachScheduling  # noqa
+from xdist.scheduler.load import LoadScheduling  # noqa
+from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
+from xdist.scheduler.filescope import LoadFileScheduling  # noqa

xdist/scheduler/each.py

@@ -0,0 +1,129 @@
+from py.log import Producer
+
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class EachScheduling(object):
+    """Implement scheduling of test items on all nodes
+
+    If a node gets added after the test run is started then it is
+    assumed to replace a node which got removed before it finished
+    its collection.  In this case it will only be used if a node
+    with the same spec got removed earlier.
+
+    Any nodes added after the run is started will only get items
+    assigned if a node with a matching spec was removed before it
+    finished all its pending items.  The new node will then be
+    assigned the remaining items from the removed node.
+    """
+
+    def __init__(self, config, log=None):
+        self.config = config
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self._started = []
+        self._removed2pending = {}
+        if log is None:
+            self.log = Producer("eachsched")
+        else:
+            self.log = log.eachsched
+        self.collection_is_completed = False
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def tests_finished(self):
+        if not self.collection_is_completed:
+            return False
+        if self._removed2pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        Collection is complete once all nodes have submitted their
+        collection.  In this case its pending list is set to an empty
+        list.  When the collection is already completed this
+        submission is from a node which was restarted to replace a
+        dead node.  In this case we already assign the pending items
+        here.  In either case ``.schedule()`` will instruct the
+        node to start running the required tests.
+        """
+        assert node in self.node2pending
+        if not self.collection_is_completed:
+            self.node2collection[node] = list(collection)
+            self.node2pending[node] = []
+            if len(self.node2collection) >= self.numnodes:
+                self.collection_is_completed = True
+        elif self._removed2pending:
+            for deadnode in self._removed2pending:
+                if deadnode.gateway.spec == node.gateway.spec:
+                    dead_collection = self.node2collection[deadnode]
+                    if collection != dead_collection:
+                        msg = report_collection_diff(dead_collection,
+                                                     collection,
+                                                     deadnode.gateway.id,
+                                                     node.gateway.id)
+                        self.log(msg)
+                        return
+                    pending = self._removed2pending.pop(deadnode)
+                    self.node2pending[node] = pending
+                    break
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        self.node2pending[node].remove(item_index)
+
+    def remove_node(self, node):
+        # KeyError if we didn't get an add_node() yet
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+        crashitem = self.node2collection[node][pending.pop(0)]
+        if pending:
+            self._removed2pending[node] = pending
+        return crashitem
+
+    def schedule(self):
+        """Schedule the test items on the nodes
+
+        If the node's pending list is empty it is a new node which
+        needs to run all the tests.  If the pending list is already
+        populated (by ``.add_node_collection()``) then it replaces a
+        dead node and we only need to run those tests.
+        """
+        assert self.collection_is_completed
+        for node, pending in self.node2pending.items():
+            if node in self._started:
+                continue
+            if not pending:
+                pending[:] = range(len(self.node2collection[node]))
+                node.send_runtest_all()
+            else:
+                node.send_runtest_some(pending)
+            self._started.append(node)

xdist/scheduler/filescope.py

@@ -0,0 +1,52 @@
+from . import LoadScopeScheduling
+from py.log import Producer
+
+
+class LoadFileScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test test file.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test file,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    This class behaves very much like LoadScopeScheduling, but with a file-level scope.
+    """
+
+    def __init__(self, config, log=None):
+        super(LoadFileScheduling, self).__init__(config, log)
+        if log is None:
+            self.log = Producer('loadfilesched')
+        else:
+            self.log = log.loadfilesched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the left. That is, test will be grouped in a
+        single work unit when they reside in the same file.
+         In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.split('::', 1)[0]

xdist/scheduler/load.py

@@ -0,0 +1,288 @@
+from itertools import cycle
+
+from py.log import Producer
+from _pytest.runner import CollectReport
+
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class LoadScheduling(object):
+    """Implement load scheduling across nodes.
+
+    This distributes the tests collected across all nodes so each test
+    is run just once.  All nodes collect and submit the test suite and
+    when all collections are received it is verified they are
+    identical collections.  Then the collection gets divided up in
+    chunks and chunks get submitted to nodes.  Whenever a node finishes
+    an item, it calls ``.mark_test_complete()`` which will trigger the
+    scheduler to assign more tests if the number of pending tests for
+    the node falls below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to
+    submit a collection. This is used to know when all nodes have
+    finished collection or how large the chunks need to be created.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self.pending = []
+        self.collection = None
+        if log is None:
+            self.log = Producer("loadsched")
+        else:
+            self.log = log.loadsched
+        self.config = config
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(self.collection,
+                                             collection,
+                                             other_node.gateway.id,
+                                             node.gateway.id)
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node
+
+        The duration it took to execute the item is used as a hint to
+        the scheduler.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule(node, duration=duration)
+
+    def check_schedule(self, node, duration=0):
+        """Maybe schedule new items on the node
+
+        If there are any globally pending nodes left then this will
+        check if the given node should be given any more tests.  The
+        ``duration`` of the last test is optionally used as a
+        heuristic to influence how many tests the node is assigned.
+        """
+        if node.shutting_down:
+            return
+
+        if self.pending:
+            # how many nodes do we have?
+            num_nodes = len(self.node2pending)
+            # if our node goes below a heuristic minimum, fill it out to
+            # heuristic maximum
+            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
+            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
+            node_pending = self.node2pending[node]
+            if len(node_pending) < items_per_node_min:
+                if duration >= 0.1 and len(node_pending) >= 2:
+                    # seems the node is doing long-running tests
+                    # and has enough items to continue
+                    # so let's rather wait with sending new items
+                    return
+                num_send = items_per_node_max - len(node_pending)
+                self._send_tests(node, num_send)
+        self.log("num items waiting for node:", len(self.pending))
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+
+        """
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+
+        # The node crashed, reassing pending items
+        crashitem = self.collection[pending.pop(0)]
+        self.pending.extend(pending)
+        for node in self.node2pending:
+            self.check_schedule(node)
+        return crashitem
+
+    def schedule(self):
+        """Initiate distribution of the test collection
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self.check_schedule(node)
+            return
+
+        # XXX allow nodes to have different collections
+        if not self._check_nodes_have_same_collection():
+            self.log('**Different tests collected, aborting run**')
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = list(self.node2collection.values())[0]
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        # Send a batch of tests to run. If we don't have at least two
+        # tests per node, we have to send them all so that we can send
+        # shutdown signals and get all nodes working.
+        initial_batch = max(len(self.pending) // 4,
+                            2 * len(self.nodes))
+
+        # distribute tests round-robin up to the batch size
+        # (or until we run out)
+        nodes = cycle(self.nodes)
+        for i in range(initial_batch):
+            self._send_tests(next(nodes), 1)
+
+        if not self.pending:
+            # initial distribution sent all tests, start node shutdown
+            for node in self.nodes:
+                node.shutdown()
+
+    def _send_tests(self, node, num):
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = CollectReport(
+                        node.gateway.id, 'failed',
+                        longrepr=msg, result=[])
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

xdist/scheduler/loadscope.py

@@ -0,0 +1,427 @@
+from collections import OrderedDict
+
+from _pytest.runner import CollectReport
+from py.log import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_spec_config
+
+
+class LoadScopeScheduling(object):
+    """Implement load scheduling across nodes, but grouping test by scope.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test scope,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual number of
+       nodes will vary during the scheduler's lifetime as nodes are added by
+       the DSession as they are brought up and removed either because of a dead
+       node or normal shutdown.  This number is primarily used to know when the
+       initial collection is completed.
+
+    :collection: The final list of tests collected by all nodes once it is
+       validated to be identical between all the nodes.  It is initialised to
+       None until ``.schedule()`` is called.
+
+    :workqueue: Ordered dictionary that maps all available scopes with their
+       associated tests (nodeid). Nodeids are in turn associated with their
+       completion status. One entry of the workqueue is called a work unit.
+       In turn, a collection of work unit is called a workload.
+
+       ::
+
+            workqueue = {
+                '<full>/<path>/<to>/test_module.py': {
+                    '<full>/<path>/<to>/test_module.py::test_case1': False,
+                    '<full>/<path>/<to>/test_module.py::test_case2': False,
+                    (...)
+                },
+                (...)
+            }
+
+    :assigned_work: Ordered dictionary that maps worker nodes with their
+       assigned work units.
+
+       ::
+
+            assigned_work = {
+                '<worker node A>': {
+                    '<full>/<path>/<to>/test_module.py': {
+                        '<full>/<path>/<to>/test_module.py::test_case1': False,
+                        '<full>/<path>/<to>/test_module.py::test_case2': False,
+                        (...)
+                    },
+                    (...)
+                },
+                (...)
+            }
+
+    :registered_collections: Ordered dictionary that maps worker nodes with
+       their collection of tests gathered during test discovery.
+
+       ::
+
+            registered_collections = {
+                '<worker node A>': [
+                    '<full>/<path>/<to>/test_module.py::test_case1',
+                    '<full>/<path>/<to>/test_module.py::test_case2',
+                ],
+                (...)
+            }
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.collection = None
+
+        self.workqueue = OrderedDict()
+        self.assigned_work = OrderedDict()
+        self.registered_collections = OrderedDict()
+
+        if log is None:
+            self.log = Producer('loadscopesched')
+        else:
+            self.log = log.loadscopesched
+
+        self.config = config
+
+    @property
+    def nodes(self):
+        """A list of all active nodes in the scheduler."""
+        return list(self.assigned_work.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes have
+        finished collection.  The required number of initial nodes is defined
+        by ``.numnodes``.
+        """
+        return len(self.registered_collections) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+
+        if self.workqueue:
+            return False
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) >= 2:
+                return False
+
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are still
+        processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.workqueue:
+            return True
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) > 0:
+                return True
+
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be assigned work units to be executed.
+
+        Called by the ``DSession.worker_workerready`` hook when it successfully
+        bootstraps a new node.
+        """
+        assert node not in self.assigned_work
+        self.assigned_work[node] = OrderedDict()
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at shutdown time.
+        In the former case any pending items assigned to the node will be
+        re-scheduled.
+
+        Called by the hooks:
+
+        - ``DSession.worker_workerfinished``.
+        - ``DSession.worker_errordown``.
+
+        Return the item being executed while the node crashed or None if the
+        node has no more pending items.
+        """
+        workload = self.assigned_work.pop(node)
+        if not self._pending_of(workload):
+            return None
+
+        # The node crashed, identify test that crashed
+        for work_unit in workload.values():
+            for nodeid, completed in work_unit.items():
+                if not completed:
+                    crashitem = nodeid
+                    break
+            else:
+                continue
+            break
+        else:
+            raise RuntimeError(
+                'Unable to identify crashitem on a workload with '
+                'pending items'
+            )
+
+        # Made uncompleted work unit available again
+        self.workqueue.update(workload)
+
+        for node in self.assigned_work:
+            self._reschedule(node)
+
+        return crashitem
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.registered_collections`` dictionary.
+
+        Called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+
+        # Check that add_node() was called on the node before
+        assert node in self.assigned_work
+
+        # A new node has been added later, perhaps an original one died.
+        if self.collection_is_completed:
+
+            # Assert that .schedule() should have been called by now
+            assert self.collection
+
+            # Check that the new collection matches the official collection
+            if collection != self.collection:
+
+                other_node = next(iter(self.registered_collections.keys()))
+
+                msg = report_collection_diff(
+                    self.collection,
+                    collection,
+                    other_node.gateway.id,
+                    node.gateway.id
+                )
+                self.log(msg)
+                return
+
+        self.registered_collections[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node.
+
+        Called by the hook:
+
+        - ``DSession.worker_testreport``.
+        """
+        nodeid = self.registered_collections[node][item_index]
+        scope = self._split_scope(nodeid)
+
+        self.assigned_work[node][scope][nodeid] = True
+        self._reschedule(node)
+
+    def _assign_work_unit(self, node):
+        """Assign a work unit to a node."""
+        assert self.workqueue
+
+        # Grab a unit of work
+        scope, work_unit = self.workqueue.popitem(last=False)
+
+        # Keep track of the assigned work
+        assigned_to_node = self.assigned_work.setdefault(
+            node, default=OrderedDict()
+        )
+        assigned_to_node[scope] = work_unit
+
+        # Ask the node to execute the workload
+        worker_collection = self.registered_collections[node]
+        nodeids_indexes = [
+            worker_collection.index(nodeid)
+            for nodeid, completed in work_unit.items()
+            if not completed
+        ]
+
+        node.send_runtest_some(nodeids_indexes)
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the right. That is, classes will be grouped in a
+        single work unit, and functions from a test module will be grouped by
+        their module. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py::Delta1
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.rsplit('::', 1)[0]
+
+    def _pending_of(self, workload):
+        """Return the number of pending tests in a workload."""
+        pending = sum(
+            list(scope.values()).count(False)
+            for scope in workload.values()
+        )
+        return pending
+
+    def _reschedule(self, node):
+        """Maybe schedule new items on the node.
+
+        If there are any globally pending work units left then this will check
+        if the given node should be given any more tests.
+        """
+
+        # Do not add more work to a node shutting down
+        if node.shutting_down:
+            return
+
+        # Check that more work is available
+        if not self.workqueue:
+            return
+
+        self.log('Number of units waiting for node:', len(self.workqueue))
+
+        # Check that the node is almost depleted of work
+        # 2: Heuristic of minimum tests to enqueue more work
+        if self._pending_of(self.assigned_work[node]) > 2:
+            return
+
+        # Pop one unit of work and assign it
+        self._assign_work_unit(node)
+
+    def schedule(self):
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this gets called
+        again later it behaves the same as calling ``._reschedule()`` on all
+        nodes so that newly added nodes will start to be used.
+
+        If ``.collection_is_completed`` is True, this is called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self._reschedule(node)
+            return
+
+        # Check that all nodes collected the same tests
+        if not self._check_nodes_have_same_collection():
+            self.log('**Different tests collected, aborting run**')
+            return
+
+        # Collections are identical, create the final list of items
+        self.collection = list(
+            next(iter(self.registered_collections.values()))
+        )
+        if not self.collection:
+            return
+
+        # Determine chunks of work (scopes)
+        for nodeid in self.collection:
+            scope = self._split_scope(nodeid)
+            work_unit = self.workqueue.setdefault(scope, default=OrderedDict())
+            work_unit[nodeid] = False
+
+        # Avoid having more workers than work
+        extra_nodes = len(self.nodes) - len(self.workqueue)
+
+        if extra_nodes > 0:
+            self.log('Shuting down {0} nodes'.format(extra_nodes))
+
+            for _ in range(extra_nodes):
+                unused_node, assigned = self.assigned_work.popitem(last=True)
+
+                self.log('Shuting down unused node {0}'.format(unused_node))
+                unused_node.shutdown()
+
+        # Assign initial workload
+        for node in self.nodes:
+            self._assign_work_unit(node)
+
+        # Ensure nodes start with at least two work units if possible (#277)
+        for node in self.nodes:
+            self._reschedule(node)
+
+        # Initial distribution sent all tests, start node shutdown
+        if not self.workqueue:
+            for node in self.nodes:
+                node.shutdown()
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.registered_collections.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+            if not msg:
+                continue
+
+            same_collection = False
+            self.log(msg)
+
+            if self.config is None:
+                continue
+
+            rep = CollectReport(
+                node.gateway.id,
+                'failed',
+                longrepr=msg,
+                result=[]
+            )
+            self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

xdist/workermanage.py

@@ -1,10 +1,13 @@
+from __future__ import print_function
 import fnmatch
 import os
 import re
+import threading
 
 import py
 import pytest
 import execnet
+
 import xdist.remote
 
 from _pytest import runner  # XXX load dynamically
@@ -32,7 +35,7 @@ class NodeManager(object):
 
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
-        self._nodesready = py.std.threading.Event()
+        self._nodesready = threading.Event()
         self.trace = self.config.trace.get("nodemanager")
         self.group = execnet.Group()
         if specs is None:
@@ -47,7 +50,7 @@ class NodeManager(object):
             self.specs.append(spec)
         self.roots = self._getrsyncdirs()
         self.rsyncoptions = self._getrsyncoptions()
-        self._rsynced_specs = py.builtin.set()
+        self._rsynced_specs = set()
 
     def rsync_roots(self, gateway):
         """Rsync the set of roots to the node's gateway cwd."""
@@ -68,7 +71,7 @@ class NodeManager(object):
         gw = self.group.makegateway(spec)
         self.config.hook.pytest_xdist_newgateway(gateway=gw)
         self.rsync_roots(gw)
-        node = SlaveController(self, gw, self.config, putevent)
+        node = WorkerController(self, gw, self.config, putevent)
         gw.node = node          # keep the node alive
         node.setup()
         self.trace("started node %r" % node)
@@ -179,14 +182,13 @@ class HostRSync(execnet.RSync):
         if self._verbose:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
-            py.builtin.print_('%s:%s <= %s' %
+            print('%s:%s <= %s' % (gateway.spec, remotepath, path))
-                              (gateway.spec, remotepath, path))
 
 
 def make_reltoroot(roots, args):
-    # XXX introduce/use public API for splitting py.test args
+    # XXX introduce/use public API for splitting pytest args
     splitcode = "::"
-    l = []
+    result = []
     for arg in args:
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
@@ -197,11 +199,11 @@ def make_reltoroot(roots, args):
                 break
         else:
             raise ValueError("arg %s not relative to an rsync root" % (arg,))
-        l.append(splitcode.join(parts))
+        result.append(splitcode.join(parts))
-    return l
+    return result
 
 
-class SlaveController(object):
+class WorkerController(object):
     ENDMARK = -1
 
     def __init__(self, nodemanager, gateway, config, putevent):
@@ -209,11 +211,16 @@ class SlaveController(object):
         self.putevent = putevent
         self.gateway = gateway
         self.config = config
-        self.slaveinput = {'slaveid': gateway.id,
+        self.workerinput = {'workerid': gateway.id,
-                           'slavecount': len(nodemanager.specs)}
+                            'workercount': len(nodemanager.specs),
+                            'slaveid': gateway.id,
+                            'slavecount': len(nodemanager.specs)
+                            }
+        # TODO: deprecated name, backward compatibility only. Remove it in future
+        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
-        self.log = py.log.Producer("slavectl-%s" % gateway.id)
+        self.log = py.log.Producer("workerctl-%s" % gateway.id)
         if not self.config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
 
@@ -225,7 +232,7 @@ class SlaveController(object):
         return self._down or self._shutdown_sent
 
     def setup(self):
-        self.log("setting up slave session")
+        self.log("setting up worker session")
         spec = self.gateway.spec
         args = self.config.args
         if not spec.popen or spec.chdir:
@@ -238,7 +245,7 @@ class SlaveController(object):
                 option_dict['basetemp'] = str(basetemp.join(name))
         self.config.hook.pytest_configure_node(node=self)
         self.channel = self.gateway.remote_exec(xdist.remote)
-        self.channel.send((self.slaveinput, args, option_dict))
+        self.channel.send((self.workerinput, args, option_dict))
         if self.putevent:
             self.channel.setcallback(
                 self.process_from_remote,
@@ -298,13 +305,13 @@ class SlaveController(object):
             eventname, kwargs = eventcall
             if eventname in ("collectionstart",):
                 self.log("ignoring %s(%s)" % (eventname, kwargs))
-            elif eventname == "slaveready":
+            elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname == "slavefinished":
+            elif eventname == "workerfinished":
                 self._down = True
-                self.slaveoutput = kwargs['slaveoutput']
+                self.workeroutput = self.slaveoutput = kwargs['workeroutput']
-                self.notify_inproc("slavefinished", node=self)
+                self.notify_inproc("workerfinished", node=self)
-            elif eventname == "logstart":
+            elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in (
                     "testreport", "collectreport", "teardownreport"):
@@ -315,21 +322,31 @@ class SlaveController(object):
                 self.notify_inproc(eventname, node=self, rep=rep)
             elif eventname == "collectionfinish":
                 self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
+            elif eventname == "runtest_protocol_complete":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "logwarning":
+                self.notify_inproc(eventname, message=kwargs['message'],
+                                   code=kwargs['code'], nodeid=kwargs['nodeid'],
+                                   fslocation=kwargs['nodeid'])
             else:
                 raise ValueError("unknown event: %s" % (eventname,))
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise
-        except:
+        except:  # noqa
-            excinfo = py.code.ExceptionInfo()
+            from _pytest._code import ExceptionInfo
-            py.builtin.print_("!" * 20, excinfo)
+            excinfo = ExceptionInfo()
-            self.config.pluginmanager.notify_exception(excinfo)
+            print("!" * 20, excinfo)
+            self.config.notify_exception(excinfo)
+            self.shutdown()
+            self.notify_inproc("errordown", node=self, error=excinfo)
 
 
 def unserialize_report(name, reportdict):
     def assembled_report(reportdict):
         from _pytest._code.code import (
             ReprEntry,
+            ReprEntryNative,
             ReprExceptionInfo,
             ReprFileLocation,
             ReprFuncArgs,
@@ -337,28 +354,42 @@ def unserialize_report(name, reportdict):
             ReprTraceback
         )
         if reportdict['longrepr']:
-            if 'reprcrash' and 'reprtraceback' in reportdict['longrepr']:
+            if 'reprcrash' in reportdict['longrepr'] and 'reprtraceback' in reportdict['longrepr']:
 
                 reprtraceback = reportdict['longrepr']['reprtraceback']
                 reprcrash = reportdict['longrepr']['reprcrash']
 
                 unserialized_entries = []
-                for entry in reprtraceback['reprentries']:
+                reprentry = None
-                    reprfuncargs, reprfileloc, reprlocals = None, None, None
+                for entry_data in reprtraceback['reprentries']:
-                    if entry['reprfuncargs']:
+                    data = entry_data['data']
-                        reprfuncargs = ReprFuncArgs(**entry['reprfuncargs'])
+                    entry_type = entry_data['type']
-                    if entry['reprfileloc']:
+                    if entry_type == 'ReprEntry':
-                        reprfileloc = ReprFileLocation(**entry['reprfileloc'])
+                        reprfuncargs = None
-                    if entry['reprlocals']:
+                        reprfileloc = None
-                        reprlocals = ReprLocals(entry['reprlocals']['lines'])
+                        reprlocals = None
+                        if data['reprfuncargs']:
+                            reprfuncargs = ReprFuncArgs(
+                                **data['reprfuncargs'])
+                        if data['reprfileloc']:
+                            reprfileloc = ReprFileLocation(
+                                **data['reprfileloc'])
+                        if data['reprlocals']:
+                            reprlocals = ReprLocals(
+                                data['reprlocals']['lines'])
 
-                    reprentry = ReprEntry(
+                        reprentry = ReprEntry(
-                        lines=entry['lines'],
+                            lines=data['lines'],
-                        reprfuncargs=reprfuncargs,
+                            reprfuncargs=reprfuncargs,
-                        reprlocals=reprlocals,
+                            reprlocals=reprlocals,
-                        filelocrepr=reprfileloc,
+                            filelocrepr=reprfileloc,
-                        style=entry['style']
+                            style=data['style']
-                    )
+                        )
+                    elif entry_type == 'ReprEntryNative':
+                        reprentry = ReprEntryNative(data['lines'])
+                    else:
+                        report_unserialization_failure(
+                            entry_type, name, reportdict)
                     unserialized_entries.append(reprentry)
                 reprtraceback['reprentries'] = unserialized_entries
 
@@ -376,3 +407,17 @@ def unserialize_report(name, reportdict):
         return runner.TestReport(**assembled_report(reportdict))
     elif name == "collectreport":
         return runner.CollectReport(**assembled_report(reportdict))
+
+
+def report_unserialization_failure(type_name, report_name, reportdict):
+    from pprint import pprint
+    url = 'https://github.com/pytest-dev/pytest-xdist/issues'
+    stream = py.io.TextIO()
+    pprint('-' * 100, stream=stream)
+    pprint('INTERNALERROR: Unknown entry type returned: %s' % type_name,
+           stream=stream)
+    pprint('report_name: %s' % report_name, stream=stream)
+    pprint(reportdict, stream=stream)
+    pprint('Please report this bug at %s' % url, stream=stream)
+    pprint('-' * 100, stream=stream)
+    assert 0, stream.getvalue()