Prepare changelog for 1.22.1

Ensure loadscope and loadfile work when the first scope has one work unit

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+Thanks for submitting a PR, your contribution is really appreciated!
+
+Here's a quick checklist that should be present in PRs:
+
+- [ ] Make sure to include reasonable tests for your change if necessary
+
+- [ ] We use [towncrier](https://pypi.python.org/pypi/towncrier) for changelog management, so please add a *news* file into the `changelog` folder following these guidelines:
+  * Name it `$issue_id.$type` for example `588.bugfix`;
+  * If you don't have an issue_id change it to the PR id after creating it
+  * Ensure type is one of `removal`, `feature`, `bugfix`, `vendor`, `doc` or `trivial`
+  * Make sure to use full sentences with correct case and punctuation, for example:
+
+    ```
+    Fix issue with non-ascii contents in doctest text files.
+    ```
+
+

.gitignore

@@ -22,6 +22,7 @@ dist/
 include/
 lib/
 bin/
+env/
 xdist/_version.py*
 pytest_xdist.egg-info
 issue/
@@ -30,3 +31,4 @@ pytestdebug.log
 .tox/
 .cache/
 .eggs/
+.idea/

.travis.yml

@@ -1,45 +1,59 @@
 sudo: false
 language: python
-python:
-- '3.5'
-install: pip install -U tox setuptools_scm
-env:
-  matrix:
-  - TESTENV=py26-pytest26
-  - TESTENV=py26-pytest27
-  - TESTENV=py26-pytest28
-  - TESTENV=py26-pytest29
-  - TESTENV=py27-pytest26
-  - TESTENV=py27-pytest27
-  - TESTENV=py27-pytest28
-  - TESTENV=py27-pytest29
-  - TESTENV=py34-pytest26
-  - TESTENV=py34-pytest27
-  - TESTENV=py34-pytest28
-  - TESTENV=py34-pytest29
-  - TESTENV=py35-pytest27
-  - TESTENV=py35-pytest28
-  - TESTENV=py35-pytest29
-  - TESTENV=py27-pytest28-pexpect
-  - TESTENV=py35-pytest28-pexpect
-  - TESTENV=flakes
-  - TESTENV=readme
-script: tox --recreate -e $TESTENV
+
 notifications:
   irc:
     channels:
-    - chat.freenode.net#pytest
+    - 'chat.freenode.net#pytest'
     on_success: change
     on_failure: change
     skip_join: true
   email:
   - pytest-commit@python.org
-deploy:
-  provider: pypi
-  user: ronny
-  password:
-    secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
-  on:
-    tags: true
-    distributions: sdist bdist_wheel
-    repo: pytest-dev/pytest-xdist
+
+python:
+- '2.7'
+- '3.4'
+- '3.5'
+- '3.6'
+env:
+- TOXENV=py-pytest30
+- TOXENV=py-pytest31
+- TOXENV=py-pytest32
+- TOXENV=py-pytest33
+
+install: pip install tox setuptools_scm
+script: tox
+
+jobs:
+  include:
+    - stage: test
+    # python x env above are already included into this stage
+    - python: "2.7"
+      env: TOXENV=py27-pytestmaster
+    - python: "2.7"
+      env: TOXENV=py27-pytestfeatures
+    - python: "3.6"
+      env: TOXENV=py36-pytestmaster
+    - python: "3.6"
+      env: TOXENV=py36-pytestfeatures
+    - python: "3.6"
+      env: TOXENV=flakes
+    - python: "3.6"
+      env: TOXENV=readme
+
+    - stage: deploy
+      python: '3.6'
+      env:
+      install: pip install -U setuptools setuptools_scm
+      script: skip
+      deploy:
+        provider: pypi
+        user: ronny
+        distributions: sdist bdist_wheel  
+        skip_upload_docs: true
+        password:
+          secure: cxmSDho5d+PYKEM4ZCg8ms1P4lzhYkrw6fEOm2HtTcsuCyY6aZMSgImWAnEYbJHSkdzgcxlXK9UKJ9B0YenXmBCkAr7UjdnpNXNmkySr0sYzlH/sfqt/dDATCHFaRKxnkOSOVywaDYhT9n8YudbXI77pXwD12i/CeSSJDbHhsu0JYUfAcb+D6YjRYoA2SEGCnzSzg+gDDfwXZx4ZiODCGLVwieNp1klCg88YROUE1BaYYNuUOONvfXX8+TWowbCF6ChH1WL/bZ49OStEYQNuYxZQZr4yClIqu9VJbchrU8j860K9ott2kkGTgfB/dDrQB/XncBubyIX9ikzCQAmmBXWAI3eyvWLPDk2Jz7kW2l2RT7syct80tCq3JhvQ1qdwr5ap7siocTLgnBW0tF4tkHSTFN3510fkc43npnp6FThebESQpnI24vqpwJ9hI/kW5mYi014Og2E/cpCXnz2XO8iZPDbqAMQpDsqEQoyhfGNgPTGp4K30TxRtwZBI5hHhDKnnR16fXtRgt1gYPvz/peUQvvpOm4JzIzGXPzluuutpnCBy75v5+oiwT3YRrLL/Meims9FtDDXL3qQubAE/ezIOOpm0N5XXV8DxIom8EN71yq5ab1tqhM+tBX7owRjy4FR4If2Q8feBdmTuh26DIQt/y+qSG8VkB9Sw/JCjc7c=
+        on:
+          tags: true
+          repo: pytest-dev/pytest-xdist

CHANGELOG

@@ -1,207 +0,0 @@
-1.15.0
-------
-
-- new ``worker_id`` fixture, returns the id of the worker in a test or fixture.
-  Thanks Jared Hellman for the PR.
-
-- display progress during collection only when in a terminal, similar to pytest #1397 issue.
-  Thanks Bruno Oliveira for the PR.
-
-- fix internal error message when ``--maxfail`` is used (#62, #65).
-  Thanks Collin RM Stocks and Bryan A. Jones for reports and Bruno Oliveira for the PR.
-
-
-1.14
-----
-
-- new hook: ``pytest_xdist_node_collection_finished(node, ids)``, called when
-  a worker has finished collection. Thanks Omer Katz for the request and
-  Bruno Oliveira for the PR.
-
-- fix README display on pypi
-
-- fix #22: xdist now works if the internal tmpdir plugin is disabled.
-  Thanks Bruno Oliveira for the PR.
-
-- fix #32: xdist now works if looponfail or boxed are disabled.
-  Thanks Bruno Oliveira for the PR.
-
-
-1.13.1
--------
-
-- fix a regression -n 0 now disables xdist again
-
-
-1.13
--------------------------
-
-- extended the tox matrix with the supported py.test versions
-
-- split up the plugin into 3 plugin's
-  to prepare the departure of boxed and looponfail.
-
-  looponfail will be a part of core
-  and forked boxed will be replaced
-  with a more reliable primitive based on xdist
-
-- conforming with new pytest-2.8 behavior of returning non-zero when all
-  tests were skipped or deselected.
-
-- new "--max-slave-restart" option that can be used to control maximum
-  number of times pytest-xdist can restart slaves due to crashes. Thanks to
-  Anatoly Bubenkov for the report and Bruno Oliveira for the PR.
-
-- release as wheel
-
-- "-n" option now can be set to "auto" for automatic detection of number
-  of cpus in the host system. Thanks Suloev Dmitry for the PR.
-
-1.12
--------------------------
-
-- fix issue594: properly report errors when the test collection
-  is random.  Thanks Bruno Oliveira.
-
-- some internal test suite adaptation (to become forward
-  compatible with the upcoming pytest-2.8)
-
-
-1.11
--------------------------
-
-- fix pytest/xdist issue485 (also depends on py-1.4.22):
-  attach stdout/stderr on --boxed processes that die.
-
-- fix pytest/xdist issue503: make sure that a node has usually
-  two items to execute to avoid scoped fixtures to be torn down
-  pre-maturely (fixture teardown/setup is "nextitem" sensitive).
-  Thanks to Andreas Pelme for bug analysis and failing test.
-
-- restart crashed nodes by internally refactoring setup handling
-  of nodes.  Also includes better code documentation.
-  Many thanks to Floris Bruynooghe for the complete PR.
-
-
-1.10
--------------------------
-
-- add glob support for rsyncignores, add command line option to pass
-  additional rsyncignores. Thanks Anatoly Bubenkov.
-
-- fix pytest issue382 - produce "pytest_runtest_logstart" event again
-  in master. Thanks Aron Curzon.
-
-- fix pytest issue419 by sending/receiving indices into the test
-  collection instead of node ids (which are not neccessarily unique
-  for functions parametrized with duplicate values)
-
-- send multiple "to test" indices in one network message to a slave
-  and improve heuristics for sending chunks where the chunksize
-  depends on the number of remaining tests rather than fixed numbers.
-  This reduces the number of master -> node messages (but not the
-  reverse direction)
-
-
-1.9
--------------------------
-
-- changed LICENSE to MIT
-
-- fix duplicate reported test ids with --looponfailing
-  (thanks Jeremy Thurgood)
-
-- fix pytest issue41: re-run tests on all file changes, not just
-  randomly select ones like .py/.c.
-
-- fix pytest issue347: slaves running on top of Python3.2
-  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
-  bugs.
-
-1.8
--------------------------
-
-- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
-  interface to perform eager teardowns for test items.
-
-1.7
--------------------------
-
-- fix incompatibilities with pytest-2.2.0 (allow multiple
-  pytest_runtest_logreport reports for a test item)
-
-1.6
--------------------------
-
-- terser collection reporting
-
-- fix issue34 - distributed testing with -p plugin now works correctly
-
-- fix race condition in looponfail mode where a concurrent file removal
-  could cause a crash
-
-1.5
--------------------------
-
-- adapt to and require pytest-2.0 changes, rsyncdirs and rsyncignore can now
-  only be specified in [pytest] sections of ini files, see "py.test -h"
-  for details.
-- major internal refactoring to match the pytest-2.0 event refactoring
-  - perform test collection always at slave side instead of at the master
-  - make python2/python3 bridging work, remove usage of pickling
-- improve initial reporting by using line-rewriting
-- remove all trailing whitespace from source
-
-1.4
--------------------------
-
-- perform distributed testing related reporting in the plugin
-  rather than having dist-related code in the generic py.test
-  distribution
-
-- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to
-  gateway specification strings.
-
-- show detailed gateway setup and platform information only when
-  "-v" or "--verbose" is specified.
-
-1.3
--------------------------
-
-- fix --looponfailing - it would not actually run against the fully changed
-  source tree when initial conftest files load application state.
-
-- adapt for py-1.3.1's new --maxfailure option
-
-1.2
--------------------------
-
-- fix issue79: sessionfinish/teardown hooks are now called systematically
-  on the slave side
-- introduce a new data input/output mechanism to allow the master side
-  to send and receive data from a slave.
-- fix race condition in underlying pickling/unpickling handling
-- use and require new register hooks facility of py.test>=1.3.0
-- require improved execnet>=1.0.6 because of various race conditions
-  that can arise in xdist testing modes.
-- fix some python3 related pickling related race conditions
-- fix PyPI description
-
-1.1
--------------------------
-
-- fix an indefinite hang which would wait for events although no events
-  are pending - this happened if items arrive very quickly while
-  the "reschedule-event" tried unconditionally avoiding a busy-loop
-  and not schedule new work.
-
-1.0
--------------------------
-
-- moved code out of py-1.1.1 into its own plugin
-- use a new, faster and more sensible model to do load-balancing
-  of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing
-  works effectively even with very few tests.
-- cleaned up termination handling
-- make -x cause hard killing of test nodes to decrease wait time
-  until the traceback shows up on first failure

CHANGELOG.rst

@@ -0,0 +1,434 @@
+pytest-xdist 1.22.1 (2018-02-19)
+================================
+
+Bug Fixes
+---------
+
+- Fix issue when using ``loadscope`` or ``loadfile`` where tests would fail to
+  start if the first scope had only one test. (`#257
+  <https://github.com/pytest-dev/pytest-xdist/issues/257>`_)
+
+
+Trivial Changes
+---------------
+
+- Change terminology used by ``pytest-xdist`` to *master* and *worker* in
+  arguments and messages (for example ``--max-worker-reset``). (`#234
+  <https://github.com/pytest-dev/pytest-xdist/issues/234>`_)
+
+
+pytest-xdist 1.22.0 (2018-01-11)
+================================
+
+Features
+--------
+
+- Add support for the ``pytest_runtest_logfinish`` hook which will be released
+  in pytest 3.4. (`#266
+  <https://github.com/pytest-dev/pytest-xdist/issues/266>`_)
+
+
+pytest-xdist 1.21.0 (2017-12-22)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- Drop support for EOL Python 2.6. (`#259
+  <https://github.com/pytest-dev/pytest-xdist/issues/259>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadfile`` option which load-distributes test to workers grouped
+  by the file the tests live in. (`#242
+  <https://github.com/pytest-dev/pytest-xdist/issues/242>`_)
+
+
+Bug Fixes
+---------
+
+- Fix accidental mutation of test report during serialization causing longrepr
+  string-ification to break. (`#241
+  <https://github.com/pytest-dev/pytest-xdist/issues/241>`_)
+
+
+pytest-xdist 1.20.1 (2017-10-05)
+================================
+
+Bug Fixes
+---------
+
+- Fix hang when all worker nodes crash and restart limit is reached (`#45
+  <https://github.com/pytest-dev/pytest-xdist/issues/45>`_)
+
+- Fix issue where the -n option would still run distributed tests when pytest
+  was run with the --collect-only option (`#5
+  <https://github.com/pytest-dev/pytest-xdist/issues/5>`_)
+
+
+pytest-xdist 1.20.0 (2017-08-17)
+================================
+
+Features
+--------
+
+- ``xdist`` now supports tests to log results multiple times, improving
+  integration with plugins which require it like `pytest-rerunfailures
+  <https://github.com/gocept/pytest-rerunfailures>`_ and `flaky
+  <https://pypi.python.org/pypi/flaky>`_. (`#206 <https://github.com/pytest-
+  dev/pytest-xdist/issues/206>`_)
+
+
+Bug Fixes
+---------
+
+- Fix issue where tests were being incorrectly identified if a worker crashed
+  during the ``teardown`` stage of the test. (`#124 <https://github.com/pytest-
+  dev/pytest-xdist/issues/124>`_)
+
+
+pytest-xdist 1.19.1 (2017-08-10)
+================================
+
+Bug Fixes
+---------
+
+- Fix crash when transferring internal pytest warnings from workers to the
+  master node. (`#214 <https://github.com/pytest-dev/pytest-
+  xdist/issues/214>`_)
+
+
+pytest-xdist 1.19.0 (2017-08-09)
+================================
+
+Deprecations and Removals
+-------------------------
+
+- ``--boxed`` functionality has been moved to a separate plugin, `pytest-forked
+  <https://github.com/pytest-dev/pytest-forked>`_. This release now depends on
+  `` pytest-forked`` and provides ``--boxed`` as a backward compatibility
+  option. (`#1 <https://github.com/pytest-dev/pytest-xdist/issues/1>`_)
+
+
+Features
+--------
+
+- New ``--dist=loadscope`` option: sends group of related tests to the same
+  worker. Tests are grouped by module for test functions and by class for test
+  methods. See ``README.rst`` for more information. (`#191 <https://github.com
+  /pytest-dev/pytest-xdist/issues/191>`_)
+
+- Warnings are now properly transferred from workers to the master node. (`#92
+  <https://github.com/pytest-dev/pytest-xdist/issues/92>`_)
+
+
+Bug Fixes
+---------
+
+- Fix serialization of native tracebacks (``--tb=native``). (`#196
+  <https://github.com/pytest-dev/pytest-xdist/issues/196>`_)
+
+
+pytest-xdist 1.18.2 (2017-07-28)
+================================
+
+Bug Fixes
+---------
+
+- Removal of unnecessary dependency on incorrect version of py. (`#105
+  <https://github.com/pytest-dev/pytest-xdist/issues/105>`_)
+
+- Fix bug in internal event-loop error handler in the master node. This bug
+  would shadow the original errors making extremely hard/impossible for users
+  to diagnose the problem properly. (`#175 <https://github.com/pytest-
+  dev/pytest-xdist/issues/175>`_)
+
+
+pytest-xdist 1.18.1 (2017-07-05)
+================================
+
+Bug Fixes
+---------
+
+- Fixed serialization of ``longrepr.sections`` during error reporting from
+  workers. (`#171 <https://github.com/pytest-dev/pytest-xdist/issues/171>`_)
+
+- Fix ``ReprLocal`` not being unserialized breaking --showlocals usages. (`#176
+  <https://github.com/pytest-dev/pytest-xdist/issues/176>`_)
+
+
+pytest-xdist 1.18.0 (2017-06-26)
+================================
+
+- ``pytest-xdist`` now requires ``pytest>=3.0.0``.
+
+Features
+--------
+
+- Add long option `--numprocesses` as alternative for `-n`. (#168)
+
+
+Bug Fixes
+---------
+
+- Fix serialization and deserialization dropping longrepr details. (#133)
+
+
+pytest-xdist 1.17.1 (2017-06-10)
+================================
+
+Bug Fixes
+---------
+
+- Hot fix release reverting the change introduced by #124, unfortunately it
+  broke a number of test suites so we are reversing this change while we
+  investigate the problem. (#157)
+
+
+Improved Documentation
+----------------------
+
+- Introduced ``towncrier`` for ``CHANGELOG`` management. (#154)
+
+- Added ``HOWTORELEASE`` documentation. (#155)
+
+
+..
+    You should *NOT* be adding new change log entries to this file, this
+    file is managed by towncrier. You *may* edit previous change logs to
+    fix problems like typo corrections or such.
+    To add a new change log entry, please see
+    https://pip.pypa.io/en/latest/development/#adding-a-news-entry
+    We named the news folder ``changelog``
+
+.. towncrier release notes start
+
+
+1.17.0
+------
+
+- fix #124: xdist would mark test as complete after 'call' step. As a result,
+  xdist could identify the wrong test as failing when test crashes at teardown.
+  To address this issue, xdist now marks test as complete at teardown.
+
+1.16.0
+------
+
+- ``pytest-xdist`` now requires pytest 2.7 or later.
+
+- Add ``worker_id`` attribute in the TestReport
+
+- new hook: ``pytest_xdist_make_scheduler(config, log)``, can return custom tests items
+  distribution logic implementation. You can take a look at built-in ``LoadScheduling``
+  and ``EachScheduling`` implementations. Note that required scheduler class public
+  API may change in next ``pytest-xdist`` versions.
+
+1.15.0
+------
+
+- new ``worker_id`` fixture, returns the id of the worker in a test or fixture.
+  Thanks Jared Hellman for the PR.
+
+- display progress during collection only when in a terminal, similar to pytest #1397 issue.
+  Thanks Bruno Oliveira for the PR.
+
+- fix internal error message when ``--maxfail`` is used (#62, #65).
+  Thanks Collin RM Stocks and Bryan A. Jones for reports and Bruno Oliveira for the PR.
+
+
+1.14
+----
+
+- new hook: ``pytest_xdist_node_collection_finished(node, ids)``, called when
+  a worker has finished collection. Thanks Omer Katz for the request and
+  Bruno Oliveira for the PR.
+
+- fix README display on pypi
+
+- fix #22: xdist now works if the internal tmpdir plugin is disabled.
+  Thanks Bruno Oliveira for the PR.
+
+- fix #32: xdist now works if looponfail or boxed are disabled.
+  Thanks Bruno Oliveira for the PR.
+
+
+1.13.1
+-------
+
+- fix a regression -n 0 now disables xdist again
+
+
+1.13
+-------------------------
+
+- extended the tox matrix with the supported py.test versions
+
+- split up the plugin into 3 plugin's
+  to prepare the departure of boxed and looponfail.
+
+  looponfail will be a part of core
+  and forked boxed will be replaced
+  with a more reliable primitive based on xdist
+
+- conforming with new pytest-2.8 behavior of returning non-zero when all
+  tests were skipped or deselected.
+
+- new "--max-slave-restart" option that can be used to control maximum
+  number of times pytest-xdist can restart slaves due to crashes. Thanks to
+  Anatoly Bubenkov for the report and Bruno Oliveira for the PR.
+
+- release as wheel
+
+- "-n" option now can be set to "auto" for automatic detection of number
+  of cpus in the host system. Thanks Suloev Dmitry for the PR.
+
+1.12
+-------------------------
+
+- fix issue594: properly report errors when the test collection
+  is random.  Thanks Bruno Oliveira.
+
+- some internal test suite adaptation (to become forward
+  compatible with the upcoming pytest-2.8)
+
+
+1.11
+-------------------------
+
+- fix pytest/xdist issue485 (also depends on py-1.4.22):
+  attach stdout/stderr on --boxed processes that die.
+
+- fix pytest/xdist issue503: make sure that a node has usually
+  two items to execute to avoid scoped fixtures to be torn down
+  pre-maturely (fixture teardown/setup is "nextitem" sensitive).
+  Thanks to Andreas Pelme for bug analysis and failing test.
+
+- restart crashed nodes by internally refactoring setup handling
+  of nodes.  Also includes better code documentation.
+  Many thanks to Floris Bruynooghe for the complete PR.
+
+
+1.10
+-------------------------
+
+- add glob support for rsyncignores, add command line option to pass
+  additional rsyncignores. Thanks Anatoly Bubenkov.
+
+- fix pytest issue382 - produce "pytest_runtest_logstart" event again
+  in master. Thanks Aron Curzon.
+
+- fix pytest issue419 by sending/receiving indices into the test
+  collection instead of node ids (which are not necessarily unique
+  for functions parametrized with duplicate values)
+
+- send multiple "to test" indices in one network message to a slave
+  and improve heuristics for sending chunks where the chunksize
+  depends on the number of remaining tests rather than fixed numbers.
+  This reduces the number of master -> node messages (but not the
+  reverse direction)
+
+
+1.9
+-------------------------
+
+- changed LICENSE to MIT
+
+- fix duplicate reported test ids with --looponfailing
+  (thanks Jeremy Thurgood)
+
+- fix pytest issue41: re-run tests on all file changes, not just
+  randomly select ones like .py/.c.
+
+- fix pytest issue347: slaves running on top of Python3.2
+  will set PYTHONDONTWRITEYBTECODE to 1 to avoid import concurrency
+  bugs.
+
+1.8
+-------------------------
+
+- fix pytest-issue93 - use the refined pytest-2.2.1 runtestprotocol
+  interface to perform eager teardowns for test items.
+
+1.7
+-------------------------
+
+- fix incompatibilities with pytest-2.2.0 (allow multiple
+  pytest_runtest_logreport reports for a test item)
+
+1.6
+-------------------------
+
+- terser collection reporting
+
+- fix issue34 - distributed testing with -p plugin now works correctly
+
+- fix race condition in looponfail mode where a concurrent file removal
+  could cause a crash
+
+1.5
+-------------------------
+
+- adapt to and require pytest-2.0 changes, rsyncdirs and rsyncignore can now
+  only be specified in [pytest] sections of ini files, see "py.test -h"
+  for details.
+- major internal refactoring to match the pytest-2.0 event refactoring
+  - perform test collection always at slave side instead of at the master
+  - make python2/python3 bridging work, remove usage of pickling
+- improve initial reporting by using line-rewriting
+- remove all trailing whitespace from source
+
+1.4
+-------------------------
+
+- perform distributed testing related reporting in the plugin
+  rather than having dist-related code in the generic py.test
+  distribution
+
+- depend on execnet-1.0.7 which adds "env1:NAME=value" keys to
+  gateway specification strings.
+
+- show detailed gateway setup and platform information only when
+  "-v" or "--verbose" is specified.
+
+1.3
+-------------------------
+
+- fix --looponfailing - it would not actually run against the fully changed
+  source tree when initial conftest files load application state.
+
+- adapt for py-1.3.1's new --maxfailure option
+
+1.2
+-------------------------
+
+- fix issue79: sessionfinish/teardown hooks are now called systematically
+  on the slave side
+- introduce a new data input/output mechanism to allow the master side
+  to send and receive data from a slave.
+- fix race condition in underlying pickling/unpickling handling
+- use and require new register hooks facility of py.test>=1.3.0
+- require improved execnet>=1.0.6 because of various race conditions
+  that can arise in xdist testing modes.
+- fix some python3 related pickling related race conditions
+- fix PyPI description
+
+1.1
+-------------------------
+
+- fix an indefinite hang which would wait for events although no events
+  are pending - this happened if items arrive very quickly while
+  the "reschedule-event" tried unconditionally avoiding a busy-loop
+  and not schedule new work.
+
+1.0
+-------------------------
+
+- moved code out of py-1.1.1 into its own plugin
+- use a new, faster and more sensible model to do load-balancing
+  of tests - now no magic "MAXITEMSPERHOST" is needed and load-testing
+  works effectively even with very few tests.
+- cleaned up termination handling
+- make -x cause hard killing of test nodes to decrease wait time
+  until the traceback shows up on first failure

HOWTORELEASE.rst

@@ -0,0 +1,43 @@
+======================
+Releasing pytest-xdist
+======================
+
+This document describes the steps to make a new ``pytest-xdist`` release.
+
+Version
+-------
+
+``master`` should always be green and a potential release candidate. ``pytest-xdist`` follows
+semantic versioning, so given that the current version is ``X.Y.Z``, to find the next version number
+one needs to look at the ``changelog`` folder:
+
+- If there is any file named ``*.feature``, then we must make a new **minor** release: next
+  release will be ``X.Y+1.0``.
+
+- Otherwise it is just a **bug fix** release: ``X.Y.Z+1``.
+
+
+Steps
+-----
+
+To publish a new release ``X.Y.Z``, the steps are as follows:
+
+#. Create a new branch named ``release-X.Y.Z`` from the latest ``master``.
+
+#. Install ``pytest-xdist`` and dev requirements in a virtualenv::
+
+    $ pip install -e . -U -r dev-requirements.txt
+
+#. Update ``CHANGELOG.rst`` file by running::
+
+    $ towncrier --version X.Y.Z --yes
+
+#. Commit and push the branch for review.
+
+#. Once PR is **green** and **approved**, create and push a tag::
+
+    $ export VERSION=X.Y.Z
+    $ git tag v$VERSION release-$VERSION
+    $ git push git@github.com:pytest-dev/pytest-xdist.git v$VERSION
+
+That will build the package and publish it on ``PyPI`` automatically.

MANIFEST.in

@@ -1,7 +0,0 @@
-include CHANGELOG
-include LICENSE 
-include README.txt
-include setup.py
-include tox.ini
-graft testing
-prune .git

OVERVIEW.md

@@ -35,6 +35,8 @@ The execution flow is:
    fashion. The rest of the tests will be distributed later as **workers**
    finish tests (see below).
 
+1. Note that `pytest_xdist_make_scheduler` hook can be used to implement custom tests distribution logic.
+   
 1. **workers** re-implement `pytest_runtestloop`: pytest's default implementation
    basically loops over all collected items in the `session` object and executes
    the `pytest_runtest_protocol` for each test item, but in xdist **workers** sit idly 

README.rst

@@ -1,15 +1,27 @@
 
-.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
-    :target: https://travis-ci.org/pytest-dev/pytest-xdist
 
 .. image:: http://img.shields.io/pypi/v/pytest-xdist.svg
-   :target: https://pypi.python.org/pypi/pytest-xdist
+    :alt: PyPI version
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://img.shields.io/pypi/pyversions/pytest-xdist.svg
+    :alt: Python versions
+    :target: https://pypi.python.org/pypi/pytest-xdist
+
+.. image:: https://anaconda.org/conda-forge/pytest-xdist/badges/version.svg
+    :alt: Anaconda version
+    :target: https://anaconda.org/conda-forge/pytest-xdist
+
+.. image:: https://travis-ci.org/pytest-dev/pytest-xdist.svg?branch=master
+    :alt: Travis CI build status
+    :target: https://travis-ci.org/pytest-dev/pytest-xdist
 
 .. image:: https://ci.appveyor.com/api/projects/status/56eq1a1avd4sdd7e/branch/master?svg=true
+    :alt: AppVeyor build status
     :target: https://ci.appveyor.com/project/pytestbot/pytest-xdist
 
 xdist: pytest distributed testing plugin
-=========================================
+========================================
 
 The `pytest-xdist`_ plugin extends py.test with some unique
 test execution modes:
@@ -18,8 +30,6 @@ test execution modes:
   those for a combined test run.  This allows to speed up
   development or to use special resources of `remote machines`_.
 
-* ``--boxed``: (not available on Windows) run each test in a boxed_
-  subprocess to survive ``SEGFAULTS`` or otherwise dying processes
 
 * ``--looponfail``: run your tests repeatedly in a subprocess.  After each run
   py.test waits until a file in your project changes and then re-runs
@@ -34,58 +44,68 @@ program source code to the remote place.  All test results
 are reported back and displayed to your local terminal.
 You may specify different Python versions and interpreters.
 
+If you would like to know how pytest-xdist works under the covers, checkout 
+`OVERVIEW <https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md>`_.
+
 
 Installation
------------------------
+------------
 
 Install the plugin with::
 
-    easy_install pytest-xdist
-
-    # or
-
     pip install pytest-xdist
 
-or use the package in develope/in-place mode with
+or use the package in develop/in-place mode with
 a checkout of the `pytest-xdist repository`_ ::
 
-    python setup.py develop
-
-Usage examples
----------------------
+    pip install --editable .
 
 .. _parallelization:
 
 Speed up test runs by sending tests to multiple CPUs
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+----------------------------------------------------
 
 To send tests to multiple CPUs, type::
 
     py.test -n NUM
 
 Especially for longer running tests or tests requiring
-a lot of IO this can lead to considerable speed ups. This option can
+a lot of I/O this can lead to considerable speed ups. This option can
 also be set to ``auto`` for automatic detection of the number of CPUs.
 
 If a test crashes the interpreter, pytest-xdist will automatically restart
-that slave and report the failure as usual. You can use the
-``--max-slave-restart`` option to limit the number of slaves that can
-be restarted, or disable restarting altogether using ``--max-slave-restart=0``.
+that worker and report the failure as usual. You can use the
+``--max-worker-restart`` option to limit the number of workers that can
+be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+
+By default, the ``-n`` option will send pending tests to any worker that is available, without
+any guaranteed order, but you can control this with these options:
+
+* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
+  by **class** for *test methods*, then each group will be sent to an available worker,
+  guaranteeing that all tests in a group run in the same process. This can be useful if you have
+  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
+  with grouping by class takes priority over grouping by module.
+  This feature was added in version ``1.19``.
+
+* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
+  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
+  in version ``1.21``.
 
 
 Running tests in a Python subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------
 
-To instantiate a python2.5 sub process and send tests to it, you may type::
+To instantiate a python3.5 subprocess and send tests to it, you may type::
 
-    py.test -d --tx popen//python=python2.5
+    py.test -d --tx popen//python=python3.5
 
-This will start a subprocess which is run with the "python2.5"
+This will start a subprocess which is run with the ``python3.5``
 Python interpreter, found in your system binary lookup path.
 
 If you prefix the --tx option value like this::
 
-    --tx 3*popen//python=python2.5
+    --tx 3*popen//python=python3.5
 
 then three subprocesses would be created and tests
 will be load-balanced across these three processes.
@@ -93,28 +113,16 @@ will be load-balanced across these three processes.
 .. _boxed:
 
 Running tests in a boxed subprocess
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
-
-If you have tests involving C or C++ libraries you might have to deal
-with tests crashing the process.  For this case you may use the boxing
-options::
-
-    py.test --boxed
-
-which will run each test in a subprocess and will report if a test
-crashed the process.  You can also combine this option with
-running multiple processes to speed up the test run and use your CPU cores::
-
-    py.test -n3 --boxed
-
-this would run 3 testing subprocesses in parallel which each
-create new boxed subprocesses for each test.
+-----------------------------------
 
+This functionality has been moved to the
+`pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ plugin, but the ``--boxed`` option
+is still kept for backward compatibility.
 
 .. _`remote machines`:
 
 Sending tests to remote SSH accounts
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------
 
 Suppose you have a package ``mypkg`` which contains some
 tests that you can successfully run locally. And you
@@ -149,7 +157,7 @@ ini-file option(s).
 
 
 Sending tests to remote Socket Servers
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------
 
 Download the single-module `socketserver.py`_ Python program
 and run it like this::
@@ -168,7 +176,7 @@ new socket host with something like this::
 
 
 Running tests on many platforms at once
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+---------------------------------------
 
 The basic command to run tests on multiple platforms is::
 
@@ -177,7 +185,7 @@ The basic command to run tests on multiple platforms is::
 If you specify a windows host, an OSX host and a Linux
 environment this command will send each tests to all
 platforms - and report back failures from all platforms
-at once.   The specifications strings use the `xspec syntax`_.
+at once. The specifications strings use the `xspec syntax`_.
 
 .. _`xspec syntax`: http://codespeak.net/execnet/basics.html#xspec
 
@@ -186,8 +194,9 @@ at once.   The specifications strings use the `xspec syntax`_.
 .. _`execnet`: http://codespeak.net/execnet
 
 Identifying the worker process during a test
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------------
 
+*New in version 1.15.*
 
 If you need to determine the identity of a worker process in
 a test or fixture, you may use the ``worker_id`` fixture to do so:
@@ -209,14 +218,15 @@ defined:
 * ``PYTEST_XDIST_WORKER_COUNT``: the total number of workers in this session,
   e.g., ``"4"`` when ``-n 4`` is given in the command-line.
 
-*New in version 1.15.*
+The information about the worker_id in a test is stored in the ``TestReport`` as
+well, under the ``worker_id`` attribute.
+
 
 Specifying test exec environments in an ini file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+------------------------------------------------
 
-pytest (since version 2.0) supports ini-style cofiguration.
-You can for example make running with three subprocesses
-your default like this:
+You can use pytest's ini file configuration to avoid typing common options.
+You can for example make running with three subprocesses your default like this:
 
 .. code-block:: ini
 
@@ -228,7 +238,7 @@ You can also add default environments like this:
 .. code-block:: ini
 
     [pytest]
-    addopts = --tx ssh=myhost//python=python2.5 --tx ssh=myhost//python=python2.6
+    addopts = --tx ssh=myhost//python=python3.5 --tx ssh=myhost//python=python3.6
 
 and then just type::
 
@@ -236,8 +246,9 @@ and then just type::
 
 to run tests in each of the environments.
 
+
 Specifying "rsync" dirs in an ini-file
-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+--------------------------------------
 
 In a ``tox.ini`` or ``setup.cfg`` file in your root project directory
 you may specify directories to include or to exclude in synchronisation:
@@ -254,9 +265,3 @@ where the configuration file was found.
 .. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist
 .. _`pytest-xdist repository`: https://github.com/pytest-dev/pytest-xdist
 .. _`pytest`: http://pytest.org
-
-Issue and Bug Tracker
-------------------------
-
-Please use the `pytest issue tracker <https://github.com/pytest-dev/pytest/issues>`_
-for reporting bugs in this plugin.

appveyor.yml

@@ -1,5 +1,17 @@
+environment:
+  matrix:
+  # note: please use "tox --listenvs" to populate the build matrix
+  - TOXENV: "py27-pytest33"
+  - TOXENV: "py34-pytest33"
+  - TOXENV: "py35-pytest33"
+  - TOXENV: "py36-pytest33"
+  - TOXENV: "py27-pytest33-pexpect"
+  - TOXENV: "py36-pytest33-pexpect"
+  - TOXENV: "flakes"
+  - TOXENV: "readme"
+
 install:
-  - C:\Python35\python -m pip install tox setuptools_scm
+  - C:\Python35\python -m pip install -U tox setuptools_scm pip
 
 build: false  # Not a C# project, build stuff at the test step instead.
 

changelog/_template.rst

@@ -0,0 +1,39 @@
+{% for section in sections %}
+{% set underline = "-" %}
+{% if section %}
+{{section}}
+{{ underline * section|length }}{% set underline = "~" %}
+
+{% endif %}
+{% if sections[section] %}
+{% for category, val in definitions.items() if category in sections[section] %}
+
+{{ definitions[category]['name'] }}
+{{ underline * definitions[category]['name']|length }}
+
+{% if definitions[category]['showcontent'] %}
+{% for text, values in sections[section][category]|dictsort(by='value') %}
+- {{ text }}{% if category != 'vendor' %} (`{{ values[0] }} <https://github.com/pytest-dev/pytest-xdist/issues/{{ values[0][1:] }}>`_){% endif %}
+
+
+{% endfor %}
+{% else %}
+- {{ sections[section][category]['']|sort|join(', ') }}
+
+
+{% endif %}
+{% if sections[section][category]|length == 0 %}
+
+No significant changes.
+
+
+{% else %}
+{% endif %}
+{% endfor %}
+{% else %}
+
+No significant changes.
+
+
+{% endif %}
+{% endfor %}

dev-requirements.txt

@@ -0,0 +1 @@
+towncrier

example/boxed.txt

@@ -1,3 +1,9 @@
+.. note::
+
+  Since 1.19.0, the actual implementation of the ``--boxed`` option has been moved to a 
+  separate plugin, `pytest-forked <https://github.com/pytest-dev/pytest-forked>`_ 
+  which can be installed independently. The ``--boxed`` command-line options remains 
+  for backward compatibility reasons.
 
 
 If your testing involves C or C++ libraries you might have to deal
@@ -19,7 +25,7 @@ to run each test in a controlled subprocess.  Here is a basic example::
 
 If you run this with::
 
-    $ py.test --boxed
+    $ py.test -n1
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov
@@ -40,7 +46,7 @@ You'll see that a couple of tests are reported as crashing, indicated
 by lower-case ``f`` and the respective failure summary. You can also use
 the xdist-provided parallelization feature to speed up your testing::
 
-    $ py.test --boxed -n3
+    $ py.test -n3
     =========================== test session starts ============================
     platform linux2 -- Python 2.7.3 -- pytest-2.3.0.dev8
     plugins: xdist, bugzilla, cache, oejskit, cli, pep8, cov

example/loadscope/epsilon/__init__.py

@@ -0,0 +1,37 @@
+def epsilon1(arg1, arg2=1000):
+    """Do epsilon1
+
+    Usage:
+
+    >>> epsilon1(10, 20)
+    40
+    >>> epsilon1(30)
+    1040
+    """
+    return arg1 + arg2 + 10
+
+
+def epsilon2(arg1, arg2=1000):
+    """Do epsilon2
+
+    Usage:
+
+    >>> epsilon2(10, 20)
+    -20
+    >>> epsilon2(30)
+    -980
+    """
+    return arg1 - arg2 - 10
+
+
+def epsilon3(arg1, arg2=1000):
+    """Do epsilon3
+
+    Usage:
+
+    >>> epsilon3(10, 20)
+    200
+    >>> epsilon3(30)
+    30000
+    """
+    return arg1 * arg2

example/loadscope/requirements.txt

@@ -0,0 +1,3 @@
+ipdb
+pytest
+../../

example/loadscope/test/test_alpha.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_alpha0():
+    sleep(5)
+    assert True
+
+
+def test_alpha1():
+    sleep(5)
+    assert True
+
+
+def test_alpha2():
+    sleep(5)
+    assert True
+
+
+def test_alpha3():
+    sleep(5)
+    assert True
+
+
+def test_alpha4():
+    sleep(5)
+    assert True
+
+
+def test_alpha5():
+    sleep(5)
+    assert True
+
+
+def test_alpha6():
+    sleep(5)
+    assert True
+
+
+def test_alpha7():
+    sleep(5)
+    assert True
+
+
+def test_alpha8():
+    sleep(5)
+    assert True
+
+
+def test_alpha9():
+    sleep(5)
+    assert True

example/loadscope/test/test_beta.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_beta0():
+    sleep(5)
+    assert True
+
+
+def test_beta1():
+    sleep(5)
+    assert True
+
+
+def test_beta2():
+    sleep(5)
+    assert True
+
+
+def test_beta3():
+    sleep(5)
+    assert True
+
+
+def test_beta4():
+    sleep(5)
+    assert True
+
+
+def test_beta5():
+    sleep(5)
+    assert True
+
+
+def test_beta6():
+    sleep(5)
+    assert True
+
+
+def test_beta7():
+    sleep(5)
+    assert True
+
+
+def test_beta8():
+    sleep(5)
+    assert True
+
+
+def test_beta9():
+    sleep(5)
+    assert True

example/loadscope/test/test_delta.py

@@ -0,0 +1,88 @@
+from time import sleep
+from unittest import TestCase
+
+
+class Delta1(TestCase):
+
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True
+
+
+class Delta2(TestCase):
+
+    def test_delta0(self):
+        sleep(5)
+        assert True
+
+    def test_delta1(self):
+        sleep(5)
+        assert True
+
+    def test_delta2(self):
+        sleep(5)
+        assert True
+
+    def test_delta3(self):
+        sleep(5)
+        assert True
+
+    def test_delta4(self):
+        sleep(5)
+        assert True
+
+    def test_delta5(self):
+        sleep(5)
+        assert True
+
+    def test_delta6(self):
+        sleep(5)
+        assert True
+
+    def test_delta7(self):
+        sleep(5)
+        assert True
+
+    def test_delta8(self):
+        sleep(5)
+        assert True
+
+    def test_delta9(self):
+        sleep(5)
+        assert True

example/loadscope/test/test_gamma.py

@@ -0,0 +1,51 @@
+from time import sleep
+
+
+def test_gamma0():
+    sleep(5)
+    assert True
+
+
+def test_gamma1():
+    sleep(5)
+    assert True
+
+
+def test_gamma2():
+    sleep(5)
+    assert True
+
+
+def test_gamma3():
+    sleep(5)
+    assert True
+
+
+def test_gamma4():
+    sleep(5)
+    assert True
+
+
+def test_gamma5():
+    sleep(5)
+    assert True
+
+
+def test_gamma6():
+    sleep(5)
+    assert True
+
+
+def test_gamma7():
+    sleep(5)
+    assert True
+
+
+def test_gamma8():
+    sleep(5)
+    assert True
+
+
+def test_gamma9():
+    sleep(5)
+    assert True

example/loadscope/tox.ini

@@ -0,0 +1,17 @@
+[tox]
+envlist = test
+setupdir = {toxinidir}/../../
+
+[testenv:test]
+basepython = python3
+passenv = http_proxy https_proxy
+deps = -rrequirements.txt
+changedir = {envtmpdir}
+commands =
+    py.test -s -v \
+        --doctest-modules \
+        --junitxml=tests.xml \
+        --dist=loadscope \
+        --tx=8*popen \
+        {toxinidir}/test \
+        {toxinidir}/epsilon

pyproject.toml

@@ -0,0 +1,36 @@
+[tool.towncrier]
+package = "xdist"
+filename = "CHANGELOG.rst"
+directory = "changelog/"
+title_format = "pytest-xdist {version} ({project_date})"
+template = "changelog/_template.rst"
+
+  [[tool.towncrier.type]]
+  directory = "removal"
+  name = "Deprecations and Removals"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "feature"
+  name = "Features"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "bugfix"
+  name = "Bug Fixes"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "vendor"
+  name = "Vendored Libraries"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "doc"
+  name = "Improved Documentation"
+  showcontent = true
+
+  [[tool.towncrier.type]]
+  directory = "trivial"
+  name = "Trivial Changes"
+  showcontent = true

setup.cfg

@@ -1,2 +1,8 @@
 [bdist_wheel]
 universal = 1
+
+[metadata]
+license_file = LICENSE
+
+[flake8]
+max-line-length = 100

setup.py

@@ -1,4 +1,7 @@
-from setuptools import setup
+from setuptools import setup, find_packages
+
+install_requires = ['execnet>=1.1', 'pytest>=3.0.0', 'pytest-forked']
+
 
 setup(
     name="pytest-xdist",
@@ -11,16 +14,16 @@ setup(
     author_email='pytest-dev@python.org,holger@merlinux.eu',
     url='https://github.com/pytest-dev/pytest-xdist',
     platforms=['linux', 'osx', 'win32'],
-    packages=['xdist'],
+    packages=find_packages(exclude=['testing', 'example']),
     entry_points={
         'pytest11': [
             'xdist = xdist.plugin',
             'xdist.looponfail = xdist.looponfail',
-            'xdist.boxed = xdist.boxed',
         ],
     },
     zip_safe=False,
-    install_requires=['execnet>=1.1', 'pytest>=2.4.2', 'py>=1.4.22'],
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+    install_requires=install_requires,
     setup_requires=['setuptools_scm'],
     classifiers=[
         'Development Status :: 5 - Production/Stable',
@@ -34,6 +37,11 @@ setup(
         'Topic :: Software Development :: Quality Assurance',
         'Topic :: Utilities',
         'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
     ],
 )

testing/acceptance_test.py

@@ -1,3 +1,5 @@
+import re
+
 import py
 import pytest
 
@@ -10,7 +12,9 @@ class TestDistribution:
         """)
         result = testdir.runpytest(p1, "-n1")
         assert result.ret == 0
-        result.stdout.fnmatch_lines(["*1 passed*", ])
+        result.stdout.fnmatch_lines([
+            "*1 passed*",
+        ])
 
     def test_n1_fail(self, testdir):
         p1 = testdir.makepyfile("""
@@ -19,7 +23,9 @@ class TestDistribution:
         """)
         result = testdir.runpytest(p1, "-n1")
         assert result.ret == 1
-        result.stdout.fnmatch_lines(["*1 failed*", ])
+        result.stdout.fnmatch_lines([
+            "*1 failed*",
+        ])
 
     def test_n1_import_error(self, testdir):
         p1 = testdir.makepyfile("""
@@ -30,7 +36,7 @@ class TestDistribution:
         result = testdir.runpytest(p1, "-n1")
         assert result.ret == 1
         result.stdout.fnmatch_lines([
-            "E   ImportError: *__import_of_missing_module*",
+            "E   *Error: No module named *__import_of_missing_module*",
         ])
 
     def test_n2_import_error(self, testdir):
@@ -53,7 +59,9 @@ class TestDistribution:
         """)
         result = testdir.runpytest(p1, "-n1")
         assert result.ret == 0
-        result.stdout.fnmatch_lines(["*1 skipped*", ])
+        result.stdout.fnmatch_lines([
+            "*1 skipped*",
+        ])
 
     def test_manytests_to_one_import_error(self, testdir):
         p1 = testdir.makepyfile("""
@@ -62,13 +70,14 @@ class TestDistribution:
                 pass
         """)
         result = testdir.runpytest(p1, '--tx=popen', '--tx=popen')
-        assert result.ret == 1
+        assert result.ret in (1, 2)
         result.stdout.fnmatch_lines([
-            "E   ImportError: *__import_of_missing_module*",
+            "E   *Error: No module named *__import_of_missing_module*",
         ])
 
     def test_manytests_to_one_popen(self, testdir):
-        p1 = testdir.makepyfile("""
+        p1 = testdir.makepyfile(
+            """
                 import py
                 def test_fail0():
                     assert 0
@@ -108,10 +117,13 @@ class TestDistribution:
         """ % str(testdir.tmpdir))
         result = testdir.runpytest_subprocess(p1, "-n1")
         assert result.ret == 0
-        result.stdout.fnmatch_lines(["*1 passed*", ])
+        result.stdout.fnmatch_lines([
+            "*1 passed*",
+        ])
 
     def test_dist_ini_specified(self, testdir):
-        p1 = testdir.makepyfile("""
+        p1 = testdir.makepyfile(
+            """
                 import py
                 def test_fail0():
                     assert 0
@@ -180,52 +192,66 @@ class TestDistribution:
         ])
         assert dest.join(subdir.basename).check(dir=1)
 
+    def test_backward_compatibility_worker_terminology(self, testdir):
+        """Ensure that we still support "config.slaveinput" for backward compatibility (#234).
+
+        Keep in mind that removing this compatibility will break a ton of plugins and user code.
+        """
+        testdir.makepyfile("""
+            def test(pytestconfig):
+                assert hasattr(pytestconfig, 'slaveinput')
+                assert hasattr(pytestconfig, 'workerinput')
+        """)
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines("*1 passed*")
+        assert result.ret == 0
+
     def test_data_exchange(self, testdir):
         testdir.makeconftest("""
             # This hook only called on master.
             def pytest_configure_node(node):
-                node.slaveinput['a'] = 42
-                node.slaveinput['b'] = 7
+                node.workerinput['a'] = 42
+                node.workerinput['b'] = 7
 
             def pytest_configure(config):
-                # this attribute is only set on slaves
-                if hasattr(config, 'slaveinput'):
-                    a = config.slaveinput['a']
-                    b = config.slaveinput['b']
+                # this attribute is only set on workers
+                if hasattr(config, 'workerinput'):
+                    a = config.workerinput['a']
+                    b = config.workerinput['b']
                     r = a + b
-                    config.slaveoutput['r'] = r
+                    config.workeroutput['r'] = r
 
             # This hook only called on master.
             def pytest_testnodedown(node, error):
-                node.config.calc_result = node.slaveoutput['r']
+                node.config.calc_result = node.workeroutput['r']
 
             def pytest_terminal_summary(terminalreporter):
-                if not hasattr(terminalreporter.config, 'slaveinput'):
+                if not hasattr(terminalreporter.config, 'workerinput'):
                     calc_result = terminalreporter.config.calc_result
                     terminalreporter._tw.sep('-',
                         'calculated result is %s' % calc_result)
         """)
         p1 = testdir.makepyfile("def test_func(): pass")
         result = testdir.runpytest("-v", p1, '-d', '--tx=popen')
-        result.stdout.fnmatch_lines([
-            "*0*Python*", "*calculated result is 49*", "*1 passed*"
-        ])
+        result.stdout.fnmatch_lines(
+            ["*0*Python*", "*calculated result is 49*", "*1 passed*"])
         assert result.ret == 0
 
     def test_keyboardinterrupt_hooks_issue79(self, testdir):
-        testdir.makepyfile(__init__="",
-                           test_one="""
+        testdir.makepyfile(
+            __init__="",
+            test_one="""
             def test_hello():
                 raise KeyboardInterrupt()
         """)
         testdir.makeconftest("""
             def pytest_sessionfinish(session):
-                # on the slave
-                if hasattr(session.config, 'slaveoutput'):
-                    session.config.slaveoutput['s2'] = 42
+                # on the worker
+                if hasattr(session.config, 'workeroutput'):
+                    session.config.workeroutput['s2'] = 42
             # on the master
             def pytest_testnodedown(node, error):
-                assert node.slaveoutput['s2'] == 42
+                assert node.workeroutput['s2'] == 42
                 print ("s2call-finished")
         """)
         args = ["-n1", "--debug"]
@@ -250,6 +276,17 @@ class TestDistribution:
         child.close()
         # assert ret == 2
 
+    def test_dist_with_collectonly(self, testdir):
+        p1 = testdir.makepyfile("""
+            def test_ok():
+                pass
+        """)
+        result = testdir.runpytest(p1, "-n1", "--collect-only")
+        assert result.ret == 0
+        result.stdout.fnmatch_lines([
+            "*collected 1 item*",
+        ])
+
 
 class TestDistEach:
     def test_simple(self, testdir):
@@ -273,8 +310,9 @@ class TestDistEach:
                 py.test.skip("%s not found" % name)
             interpreters.append(interp)
 
-        testdir.makepyfile(__init__="",
-                           test_one="""
+        testdir.makepyfile(
+            __init__="",
+            test_one="""
             import sys
             def test_hello():
                 print("%s...%s" % sys.version_info[:2])
@@ -326,6 +364,50 @@ class TestTerminalReporting:
             "E       assert 0",
         ])
 
+    @pytest.mark.parametrize('n', ['-n0', '-n1'])
+    @pytest.mark.parametrize('warn_type', ['pytest', 'builtin'])
+    def test_logwarning(self, testdir, n, warn_type):
+        from pkg_resources import parse_version
+        if parse_version(pytest.__version__) < parse_version('3.1'):
+            pytest.skip('pytest warnings requires >= 3.1')
+
+        if warn_type == 'builtin':
+            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
+        elif warn_type == 'pytest':
+            warn_code = """request.config.warn('', 'this is a warning',
+                           fslocation=py.path.local())"""
+        else:
+            assert False
+        testdir.makepyfile("""
+            import warnings, py
+            def test_func(request):
+                {warn_code}
+        """.format(warn_code=warn_code))
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines([
+            "*this is a warning*",
+            "*1 passed, 1 warnings*",
+        ])
+
+    def test_logfinish_hook(self, testdir):
+        """Ensure the pytest_runtest_logfinish hook is being properly handled"""
+        from _pytest import hookspec
+        if not hasattr(hookspec, 'pytest_runtest_logfinish'):
+            pytest.skip('test requires pytest_runtest_logfinish hook in pytest (3.4+)')
+
+        testdir.makeconftest("""
+            def pytest_runtest_logfinish():
+                print('pytest_runtest_logfinish hook called')
+        """)
+        testdir.makepyfile("""
+            def test_func():
+                pass
+        """)
+        result = testdir.runpytest("-n1", "-s")
+        result.stdout.fnmatch_lines([
+            "*pytest_runtest_logfinish hook called*",
+        ])
+
 
 def test_teardownfails_one_function(testdir):
     p = testdir.makepyfile("""
@@ -335,22 +417,23 @@ def test_teardownfails_one_function(testdir):
             assert 0
     """)
     result = testdir.runpytest(p, '-n1', '--tx=popen')
-    result.stdout.fnmatch_lines([
-        "*def teardown_function(function):*", "*1 passed*1 error*"
-    ])
+    result.stdout.fnmatch_lines(
+        ["*def teardown_function(function):*", "*1 passed*1 error*"])
 
 
 @py.test.mark.xfail
 def test_terminate_on_hangingnode(testdir):
     p = testdir.makeconftest("""
         def pytest_sessionfinish(session):
-            if session.nodeid == "my": # running on slave
+            if session.nodeid == "my": # running on worker
                 import time
                 time.sleep(3)
     """)
     result = testdir.runpytest(p, '--dist=each', '--tx=popen//id=my')
     assert result.duration < 2.0
-    result.stdout.fnmatch_lines(["*killed*my*", ])
+    result.stdout.fnmatch_lines([
+        "*killed*my*",
+    ])
 
 
 @pytest.mark.xfail(reason="works if run outside test suite", run=False)
@@ -360,15 +443,15 @@ def test_session_hooks(testdir):
         def pytest_sessionstart(session):
             sys.pytestsessionhooks = session
         def pytest_sessionfinish(session):
-            if hasattr(session.config, 'slaveinput'):
-                name = "slave"
+            if hasattr(session.config, 'workerinput'):
+                name = "worker"
             else:
                 name = "master"
             f = open(name, "w")
             f.write("xy")
             f.close()
-            # let's fail on the slave
-            if name == "slave":
+            # let's fail on the worker
+            if name == "worker":
                 raise ValueError(42)
     """)
     p = testdir.makepyfile("""
@@ -377,18 +460,21 @@ def test_session_hooks(testdir):
             assert hasattr(sys, 'pytestsessionhooks')
     """)
     result = testdir.runpytest(p, "--dist=each", "--tx=popen")
-    result.stdout.fnmatch_lines(["*ValueError*", "*1 passed*", ])
+    result.stdout.fnmatch_lines([
+        "*ValueError*",
+        "*1 passed*",
+    ])
     assert not result.ret
     d = result.parseoutcomes()
     assert d['passed'] == 1
-    assert testdir.tmpdir.join("slave").check()
+    assert testdir.tmpdir.join("worker").check()
     assert testdir.tmpdir.join("master").check()
 
 
 def test_session_testscollected(testdir):
     """
     Make sure master node is updating the session object with the number
-    of tests collected from the slaves.
+    of tests collected from the workers.
     """
     testdir.makepyfile(test_foo="""
         import pytest
@@ -422,22 +508,67 @@ def test_funcarg_teardown_failure(testdir):
             pass
     """)
     result = testdir.runpytest_subprocess("--debug", p)  # , "-n1")
-    result.stdout.fnmatch_lines(["*ValueError*42*", "*1 passed*1 error*", ])
+    result.stdout.fnmatch_lines([
+        "*ValueError*42*",
+        "*1 passed*1 error*",
+    ])
     assert result.ret
 
 
-def test_crashing_item(testdir):
+@pytest.mark.parametrize('when', ['setup', 'call', 'teardown'])
+def test_crashing_item(testdir, when):
+    """Ensure crashing item is correctly reported during all testing stages"""
+    code = dict(setup='', call='', teardown='')
+    code[when] = 'py.process.kill(os.getpid())'
     p = testdir.makepyfile("""
-        import py
         import os
-        def test_crash():
-            py.process.kill(os.getpid())
-        def test_noncrash():
+        import py
+        import pytest
+
+        @pytest.fixture
+        def fix():
+            {setup}
+            yield
+            {teardown}
+
+        def test_crash(fix):
+            {call}
             pass
-    """)
+
+        def test_ok():
+            pass
+    """.format(**code))
+    passes = 2 if when == 'teardown' else 1
     result = testdir.runpytest("-n2", p)
     result.stdout.fnmatch_lines([
-        "*crashed*test_crash*", "*1 failed*1 passed*"
+        "*crashed*test_crash*",
+        "*1 failed*%d passed*" % passes,
+    ])
+
+
+def test_multiple_log_reports(testdir):
+    """
+    Ensure that pytest-xdist supports plugins that emit multiple logreports
+    (#206).
+    Inspired by pytest-rerunfailures.
+    """
+    testdir.makeconftest("""
+        from _pytest.runner import runtestprotocol
+        def pytest_runtest_protocol(item, nextitem):
+            item.ihook.pytest_runtest_logstart(nodeid=item.nodeid,
+                                               location=item.location)
+            reports = runtestprotocol(item, nextitem=nextitem)
+            for report in reports:
+                item.ihook.pytest_runtest_logreport(report=report)
+            return True
+    """)
+    testdir.makepyfile("""
+        def test():
+            pass
+    """)
+    result = testdir.runpytest("-n1")
+    result.stdout.fnmatch_lines([
+        "*2 passed*",
     ])
 
 
@@ -453,7 +584,8 @@ def test_skipping(testdir):
 
 
 def test_issue34_pluginloading_in_subprocess(testdir):
-    testdir.tmpdir.join("plugin123.py").write(py.code.Source("""
+    testdir.tmpdir.join("plugin123.py").write(
+        py.code.Source("""
         def pytest_namespace():
             return {'sample_variable': 'testing'}
     """))
@@ -464,7 +596,9 @@ def test_issue34_pluginloading_in_subprocess(testdir):
     """)
     result = testdir.runpytest_subprocess("-n1", "-p", "plugin123")
     assert result.ret == 0
-    result.stdout.fnmatch_lines(["*1 passed*", ])
+    result.stdout.fnmatch_lines([
+        "*1 passed*",
+    ])
 
 
 def test_fixture_scope_caching_issue503(testdir):
@@ -486,7 +620,9 @@ def test_fixture_scope_caching_issue503(testdir):
     """)
     result = testdir.runpytest(p1, '-v', '-n1')
     assert result.ret == 0
-    result.stdout.fnmatch_lines(["*2 passed*", ])
+    result.stdout.fnmatch_lines([
+        "*2 passed*",
+    ])
 
 
 def test_issue_594_random_parametrize(testdir):
@@ -545,8 +681,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n1')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
-            "*Slave*crashed while running*",
+            "*Replacing crashed worker*",
+            "*Worker*crashed while running*",
             "*1 failed*1 passed*",
         ])
 
@@ -560,8 +696,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '-n2')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
-            "*Slave*crashed while running*",
+            "*Replacing crashed worker*",
+            "*Worker*crashed while running*",
             "*1 failed*3 passed*",
         ])
 
@@ -573,8 +709,8 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=popen')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
-            "*Slave*crashed while running*",
+            "*Replacing crashed worker*",
+            "*Worker*crashed while running*",
             "*1 failed*1 passed*",
         ])
 
@@ -587,12 +723,12 @@ class TestNodeFailure:
         """)
         res = testdir.runpytest(f, '--dist=each', '--tx=2*popen')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
-            "*Slave*crashed while running*",
+            "*Replacing crashed worker*",
+            "*Worker*crashed while running*",
             "*2 failed*2 passed*",
         ])
 
-    def test_max_slave_restart(self, testdir):
+    def test_max_worker_restart(self, testdir):
         f = testdir.makepyfile("""
             import os
             def test_a(): pass
@@ -600,15 +736,26 @@ class TestNodeFailure:
             def test_c(): os._exit(1)
             def test_d(): pass
         """)
-        res = testdir.runpytest(f, '-n4', '--max-slave-restart=1')
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=1')
         res.stdout.fnmatch_lines([
-            "*Replacing crashed slave*",
-            "*Maximum crashed slaves reached: 1*",
-            "*Slave*crashed while running*",
-            "*Slave*crashed while running*",
+            "*Replacing crashed worker*",
+            "*Maximum crashed workers reached: 1*",
+            "*Worker*crashed while running*",
+            "*Worker*crashed while running*",
             "*2 failed*2 passed*",
         ])
 
+    def test_max_worker_restart_die(self, testdir):
+        f = testdir.makepyfile("""
+            import os
+            os._exit(1)
+        """)
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=0')
+        res.stdout.fnmatch_lines([
+            "*Unexpectedly no active workers*",
+            "*INTERNALERROR*"
+        ])
+
     def test_disable_restart(self, testdir):
         f = testdir.makepyfile("""
             import os
@@ -616,10 +763,10 @@ class TestNodeFailure:
             def test_b(): os._exit(1)
             def test_c(): pass
         """)
-        res = testdir.runpytest(f, '-n4', '--max-slave-restart=0')
+        res = testdir.runpytest(f, '-n4', '--max-worker-restart=0')
         res.stdout.fnmatch_lines([
-            "*Slave restarting disabled*",
-            "*Slave*crashed while running*",
+            "*Worker restarting disabled*",
+            "*Worker*crashed while running*",
             "*1 failed*2 passed*",
         ])
 
@@ -641,9 +788,23 @@ def test_worker_id_fixture(testdir, n):
         with open(fname) as f:
             worker_ids.add(f.read().strip())
     if n == 0:
-        assert worker_ids == set(['master'])
+        assert worker_ids == {'master'}
     else:
-        assert worker_ids == set(['gw0', 'gw1'])
+        assert worker_ids == {'gw0', 'gw1'}
+
+
+@pytest.mark.parametrize('tb',
+                         ['auto', 'long', 'short', 'no', 'line', 'native'])
+def test_error_report_styles(testdir, tb):
+    testdir.makepyfile("""
+        import pytest
+        def test_error_report_styles():
+            raise RuntimeError('some failure happened')
+    """)
+    result = testdir.runpytest('-n1', '--tb=%s' % tb)
+    if tb != 'no':
+        result.stdout.fnmatch_lines('*some failure happened*')
+    result.assert_outcomes(failed=1)
 
 
 def test_color_yes_collection_on_non_atty(testdir, request):
@@ -687,3 +848,189 @@ def test_internal_error_with_maxfail(testdir):
     result = testdir.runpytest_subprocess('--maxfail=1', '-n1')
     result.stdout.fnmatch_lines(['* 1 error in *'])
     assert 'INTERNALERROR' not in result.stderr.str()
+
+
+class TestLoadScope:
+
+    def test_by_module(self, testdir):
+        test_file = """
+            import pytest
+            @pytest.mark.parametrize('i', range(10))
+            def test(i):
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file,
+            test_b=test_file,
+        )
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::test', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+        assert get_workers_and_test_count_by_prefix(
+            'test_b.py::test', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+
+    def test_by_class(self, testdir):
+        testdir.makepyfile(test_a="""
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """)
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+        assert get_workers_and_test_count_by_prefix(
+            'test_a.py::TestB', result.outlines) in ({'gw0': 10}, {'gw1': 10})
+
+    def test_module_single_start(self, testdir):
+        """Fix test suite never finishing in case all workers start with a single test (#277)."""
+        test_file1 = """
+            import pytest
+            def test():
+                pass
+        """
+        test_file2 = """
+            import pytest
+            def test_1():
+                pass
+            def test_2():
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file1,
+            test_b=test_file1,
+            test_c=test_file2
+        )
+        result = testdir.runpytest('-n2', '--dist=loadscope', '-v')
+        a = get_workers_and_test_count_by_prefix('test_a.py::test',
+                                                 result.outlines)
+        b = get_workers_and_test_count_by_prefix('test_b.py::test',
+                                                 result.outlines)
+        c1 = get_workers_and_test_count_by_prefix('test_c.py::test_1',
+                                                  result.outlines)
+        c2 = get_workers_and_test_count_by_prefix('test_c.py::test_2',
+                                                  result.outlines)
+        assert a in ({'gw0': 1}, {'gw1': 1})
+        assert b in ({'gw0': 1}, {'gw1': 1})
+        assert a.items() != b.items()
+        assert c1 == c2
+
+
+class TestFileScope:
+
+    def test_by_module(self, testdir):
+        test_file = """
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """
+        testdir.makepyfile(
+            test_a=test_file,
+            test_b=test_file,
+        )
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines)
+        test_b_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_b.py::TestB', result.outlines)
+
+        assert test_a_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_a_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+        assert test_b_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_b_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+
+    def test_by_class(self, testdir):
+        testdir.makepyfile(test_a="""
+            import pytest
+            class TestA:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+
+            class TestB:
+                @pytest.mark.parametrize('i', range(10))
+                def test(self, i):
+                    pass
+        """)
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        test_a_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestA', result.outlines)
+        test_b_workers_and_test_count = get_workers_and_test_count_by_prefix(
+            'test_a.py::TestB', result.outlines)
+
+        assert test_a_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_a_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+        assert test_b_workers_and_test_count in ({'gw0': 10}, {'gw1': 0}) or \
+            test_b_workers_and_test_count in ({'gw0': 0}, {'gw1': 10})
+
+    def test_module_single_start(self, testdir):
+        """Fix test suite never finishing in case all workers start with a single test (#277)."""
+        test_file1 = """
+            import pytest
+            def test():
+                pass
+        """
+        test_file2 = """
+            import pytest
+            def test_1():
+                pass
+            def test_2():
+                pass
+        """
+        testdir.makepyfile(
+            test_a=test_file1,
+            test_b=test_file1,
+            test_c=test_file2
+        )
+        result = testdir.runpytest('-n2', '--dist=loadfile', '-v')
+        a = get_workers_and_test_count_by_prefix('test_a.py::test',
+                                                 result.outlines)
+        b = get_workers_and_test_count_by_prefix('test_b.py::test',
+                                                 result.outlines)
+        c1 = get_workers_and_test_count_by_prefix('test_c.py::test_1',
+                                                  result.outlines)
+        c2 = get_workers_and_test_count_by_prefix('test_c.py::test_2',
+                                                  result.outlines)
+        assert a in ({'gw0': 1}, {'gw1': 1})
+        assert b in ({'gw0': 1}, {'gw1': 1})
+        assert a.items() != b.items()
+        assert c1 == c2
+
+
+def parse_tests_and_workers_from_output(lines):
+    result = []
+    for line in lines:
+        # example match: "[gw0] PASSED test_a.py::test[7]"
+        m = re.match(r'''
+            \[(gw\d)\]  # worker
+            \s*
+            (?:\[\s*\d+%\])? # progress indicator (pytest >=3.3)
+            \s(.*?)     # status string ("PASSED")
+            \s(.*::.*)  # nodeid
+        ''', line.strip(), re.VERBOSE)
+        if m:
+            worker, status, nodeid = m.groups()
+            result.append((worker, status, nodeid))
+    return result
+
+
+def get_workers_and_test_count_by_prefix(prefix, lines,
+                                         expected_status='PASSED'):
+    result = {}
+    for worker, status, nodeid in parse_tests_and_workers_from_output(lines):
+        if expected_status == status and nodeid.startswith(prefix):
+            result[worker] = result.get(worker, 0) + 1
+    return result

testing/conftest.py

@@ -20,13 +20,13 @@ pytest_plugins = "pytester"
 @pytest.fixture(autouse=True)
 def _divert_atexit(request, monkeypatch):
     import atexit
-    l = []
+    finalizers = []
 
     def finish():
-        while l:
-            l.pop()()
+        while finalizers:
+            finalizers.pop()()
 
-    monkeypatch.setattr(atexit, "register", l.append)
+    monkeypatch.setattr(atexit, "register", finalizers.append)
     request.addfinalizer(finish)
 
 
@@ -34,7 +34,7 @@ def pytest_addoption(parser):
     parser.addoption('--gx',
                      action="append",
                      dest="gspecs",
-                     help=("add a global test environment, XSpec-syntax. "))
+                     help="add a global test environment, XSpec-syntax. ")
 
 
 @pytest.fixture

testing/test_boxed.py

@@ -1,58 +0,0 @@
-import pytest
-import os
-
-needsfork = pytest.mark.skipif(not hasattr(os, "fork"),
-                               reason="os.fork required")
-
-
-@needsfork
-def test_functional_boxed(testdir):
-    p1 = testdir.makepyfile("""
-        import os
-        def test_function():
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed")
-    result.stdout.fnmatch_lines([
-        "*CRASHED*",
-        "*1 failed*"
-    ])
-
-
-@needsfork
-@pytest.mark.parametrize("capmode", [
-    "no",
-    pytest.mark.xfail("sys", reason="capture cleanup needed"),
-    pytest.mark.xfail("fd", reason="capture cleanup needed")])
-def test_functional_boxed_capturing(testdir, capmode):
-    p1 = testdir.makepyfile("""
-        import os
-        import sys
-        def test_function():
-            sys.stdout.write("hello\\n")
-            sys.stderr.write("world\\n")
-            os.kill(os.getpid(), 15)
-    """)
-    result = testdir.runpytest(p1, "--boxed", "--capture=%s" % capmode)
-    result.stdout.fnmatch_lines("""
-        *CRASHED*
-        *stdout*
-        hello
-        *stderr*
-        world
-        *1 failed*
-    """)
-
-
-class TestOptionEffects:
-    def test_boxed_option_default(self, testdir):
-        tmpdir = testdir.tmpdir.ensure("subdir", dir=1)
-        config = testdir.parseconfig()
-        assert not config.option.boxed
-        pytest.importorskip("execnet")
-        config = testdir.parseconfig('-d', tmpdir)
-        assert not config.option.boxed
-
-    def test_is_not_boxed_by_default(self, testdir):
-        config = testdir.parseconfig(testdir.tmpdir)
-        assert not config.option.boxed

testing/test_dsession.py

@@ -1,6 +1,10 @@
-from xdist.dsession import (
-    DSession, LoadScheduling, EachScheduling, report_collection_diff,
+from xdist.dsession import DSession
+from xdist.report import report_collection_diff
+from xdist.scheduler import (
+    EachScheduling,
+    LoadScheduling,
 )
+
 import py
 import pytest
 import execnet
@@ -49,80 +53,84 @@ def dumpqueue(queue):
 
 
 class TestEachScheduling:
-    def test_schedule_load_simple(self):
+    def test_schedule_load_simple(self, testdir):
         node1 = MockNode()
         node2 = MockNode()
-        sched = EachScheduling(2)
-        sched.addnode(node1)
-        sched.addnode(node2)
+        config = testdir.parseconfig("--tx=2*popen")
+        sched = EachScheduling(config)
+        sched.add_node(node1)
+        sched.add_node(node2)
         collection = ["a.py::test_1", ]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
-        assert sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
         assert node1.sent == ['ALL']
         assert node2.sent == ['ALL']
-        sched.remove_item(node1, 0)
-        assert sched.tests_finished()
-        sched.remove_item(node2, 0)
-        assert sched.tests_finished()
+        sched.mark_test_complete(node1, 0)
+        assert sched.tests_finished
+        sched.mark_test_complete(node2, 0)
+        assert sched.tests_finished
 
-    def test_schedule_remove_node(self):
+    def test_schedule_remove_node(self, testdir):
         node1 = MockNode()
-        sched = EachScheduling(1)
-        sched.addnode(node1)
+        config = testdir.parseconfig("--tx=popen")
+        sched = EachScheduling(config)
+        sched.add_node(node1)
         collection = ["a.py::test_1", ]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
-        sched.init_distribute()
-        assert sched.tests_finished()
+        sched.schedule()
+        assert sched.tests_finished
         crashitem = sched.remove_node(node1)
         assert crashitem
-        assert sched.tests_finished()
-        assert not sched.hasnodes()
+        assert sched.tests_finished
+        assert not sched.nodes
 
 
 class TestLoadScheduling:
-    def test_schedule_load_simple(self):
-        sched = LoadScheduling(2)
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
+    def test_schedule_load_simple(self, testdir):
+        config = testdir.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2 = sched.nodes
         collection = ["a.py::test_1", "a.py::test_2"]
         assert not sched.collection_is_completed
-        sched.addnode_collection(node1, collection)
+        sched.add_node_collection(node1, collection)
         assert not sched.collection_is_completed
-        sched.addnode_collection(node2, collection)
+        sched.add_node_collection(node2, collection)
         assert sched.collection_is_completed
         assert sched.node2collection[node1] == collection
         assert sched.node2collection[node2] == collection
-        sched.init_distribute()
+        sched.schedule()
         assert not sched.pending
-        assert sched.tests_finished()
+        assert sched.tests_finished
         assert len(node1.sent) == 1
         assert len(node2.sent) == 1
         assert node1.sent == [0]
         assert node2.sent == [1]
-        sched.remove_item(node1, node1.sent[0])
-        assert sched.tests_finished()
+        sched.mark_test_complete(node1, node1.sent[0])
+        assert sched.tests_finished
 
-    def test_init_distribute_batch_size(self):
-        sched = LoadScheduling(2)
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
+    def test_schedule_batch_size(self, testdir):
+        config = testdir.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2 = sched.nodes
-        col = ["xyz"] * (6)
-        sched.addnode_collection(node1, col)
-        sched.addnode_collection(node2, col)
-        sched.init_distribute()
-        # assert not sched.tests_finished()
+        col = ["xyz"] * 6
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         assert sent1 == [0, 2]
@@ -131,25 +139,26 @@ class TestLoadScheduling:
         assert sched.node2pending[node1] == sent1
         assert sched.node2pending[node2] == sent2
         assert len(sched.pending) == 2
-        sched.remove_item(node1, 0)
+        sched.mark_test_complete(node1, 0)
         assert node1.sent == [0, 2, 4]
         assert sched.pending == [5]
         assert node2.sent == [1, 3]
-        sched.remove_item(node1, 2)
+        sched.mark_test_complete(node1, 2)
         assert node1.sent == [0, 2, 4, 5]
         assert not sched.pending
 
-    def test_init_distribute_fewer_tests_than_nodes(self):
-        sched = LoadScheduling(2)
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
+    def test_schedule_fewer_tests_than_nodes(self, testdir):
+        config = testdir.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2, node3 = sched.nodes
         col = ["xyz"] * 2
-        sched.addnode_collection(node1, col)
-        sched.addnode_collection(node2, col)
-        sched.init_distribute()
-        # assert not sched.tests_finished()
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         sent3 = node3.sent
@@ -158,17 +167,18 @@ class TestLoadScheduling:
         assert sent3 == []
         assert not sched.pending
 
-    def test_init_distribute_fewer_than_two_tests_per_node(self):
-        sched = LoadScheduling(2)
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
-        sched.addnode(MockNode())
+    def test_schedule_fewer_than_two_tests_per_node(self, testdir):
+        config = testdir.parseconfig("--tx=2*popen")
+        sched = LoadScheduling(config)
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
+        sched.add_node(MockNode())
         node1, node2, node3 = sched.nodes
         col = ["xyz"] * 5
-        sched.addnode_collection(node1, col)
-        sched.addnode_collection(node2, col)
-        sched.init_distribute()
-        # assert not sched.tests_finished()
+        sched.add_node_collection(node1, col)
+        sched.add_node_collection(node2, col)
+        sched.schedule()
+        # assert not sched.tests_finished
         sent1 = node1.sent
         sent2 = node2.sent
         sent3 = node3.sent
@@ -177,14 +187,15 @@ class TestLoadScheduling:
         assert sent3 == [2]
         assert not sched.pending
 
-    def test_add_remove_node(self):
+    def test_add_remove_node(self, testdir):
         node = MockNode()
-        sched = LoadScheduling(1)
-        sched.addnode(node)
+        config = testdir.parseconfig("--tx=popen")
+        sched = LoadScheduling(config)
+        sched.add_node(node)
         collection = ["test_file.py::test_func"]
-        sched.addnode_collection(node, collection)
+        sched.add_node_collection(node, collection)
         assert sched.collection_is_completed
-        sched.init_distribute()
+        sched.schedule()
         assert not sched.pending
         crashitem = sched.remove_node(node)
         assert crashitem == collection[0]
@@ -192,7 +203,7 @@ class TestLoadScheduling:
     def test_different_tests_collected(self, testdir):
         """
         Test that LoadScheduling is reporting collection errors when
-        different test ids are collected by slaves.
+        different test ids are collected by workers.
         """
 
         class CollectHook(object):
@@ -207,16 +218,16 @@ class TestLoadScheduling:
                 self.reports.append(report)
 
         collect_hook = CollectHook()
-        config = testdir.parseconfig()
+        config = testdir.parseconfig("--tx=2*popen")
         config.pluginmanager.register(collect_hook, "collect_hook")
         node1 = MockNode()
         node2 = MockNode()
-        sched = LoadScheduling(2, config=config)
-        sched.addnode(node1)
-        sched.addnode(node2)
-        sched.addnode_collection(node1, ["a.py::test_1"])
-        sched.addnode_collection(node2, ["a.py::test_2"])
-        sched.init_distribute()
+        sched = LoadScheduling(config)
+        sched.add_node(node1)
+        sched.add_node(node2)
+        sched.add_node_collection(node1, ["a.py::test_1"])
+        sched.add_node_collection(node2, ["a.py::test_2"])
+        sched.schedule()
         assert len(collect_hook.reports) == 1
         rep = collect_hook.reports[0]
         assert 'Different tests were collected between' in rep.longrepr

testing/test_looponfail.py

@@ -83,10 +83,10 @@ class TestStatRecorder:
         tmp = tmpdir
         sd = StatRecorder([tmp])
 
-        l = [True, False]
-        monkeypatch.setattr(StatRecorder, 'check', lambda self: l.pop())
+        ret_values = [True, False]
+        monkeypatch.setattr(StatRecorder, 'check', lambda self: ret_values.pop())
         sd.waitonchange(checkinterval=0.2)
-        assert not l
+        assert not ret_values
 
 
 class TestRemoteControl:
@@ -193,6 +193,8 @@ class TestLooponFailing:
         assert 'test_one' not in remotecontrol.failures[0]
         assert 'test_two' in remotecontrol.failures[0]
 
+    @py.test.mark.xfail(py.test.__version__ >= "3.1",
+                        reason="broken by pytest 3.1+")
     def test_looponfail_removed_test(self, testdir):
         modcol = testdir.getmodulecol("""
             def test_one():
@@ -268,7 +270,7 @@ class TestFunctional:
         """)
         child = testdir.spawn_pytest("-f %s" % p)
         child.expect("1 xpass")
-        child.expect("### LOOPONFAILING ####")
+        # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")
         child.kill(15)
 

testing/test_newhooks.py

@@ -14,14 +14,19 @@ class TestHooks:
 
     def test_runtest_logreport(self, testdir):
         """Test that log reports from pytest_runtest_logreport when running
-        with xdist contain a "node" attribute. (#8)
+        with xdist contain "node", "nodeid" and "worker_id" attributes. (#8)
         """
         testdir.makeconftest("""
             def pytest_runtest_logreport(report):
                 if hasattr(report, 'node'):
-                    slaveid = report.node.slaveinput['slaveid']
                     if report.when == "call":
-                        print("HOOK: %s %s" % (report.nodeid, slaveid))
+                        workerid = report.node.workerinput['workerid']
+                        if workerid != report.worker_id:
+                            print("HOOK: Worker id mismatch: %s %s"
+                                   % (workerid, report.worker_id))
+                        else:
+                            print("HOOK: %s %s"
+                                   % (report.nodeid, report.worker_id))
         """)
         res = testdir.runpytest('-n1', '-s')
         res.stdout.fnmatch_lines([
@@ -36,9 +41,9 @@ class TestHooks:
         """
         testdir.makeconftest("""
             def pytest_xdist_node_collection_finished(node, ids):
-                slaveid = node.slaveinput['slaveid']
+                workerid = node.workerinput['workerid']
                 stripped_ids = [x.split('::')[1] for x in ids]
-                print("HOOK: %s %s" % (slaveid, ', '.join(stripped_ids)))
+                print("HOOK: %s %s" % (workerid, ', '.join(stripped_ids)))
         """)
         res = testdir.runpytest('-n2', '-s')
         res.stdout.fnmatch_lines_random([

testing/test_plugin.py

@@ -1,6 +1,6 @@
 import py
 import execnet
-from xdist.slavemanage import NodeManager
+from xdist.workermanage import NodeManager
 
 
 def test_dist_incompatibility_messages(testdir):
@@ -20,6 +20,10 @@ def test_dist_options(testdir):
     check_options(config)
     assert config.option.dist == "load"
     assert config.option.tx == ['popen'] * 2
+    config = testdir.parseconfigure("--numprocesses", "2")
+    check_options(config)
+    assert config.option.dist == "load"
+    assert config.option.tx == ['popen'] * 2
     config = testdir.parseconfigure("-d")
     check_options(config)
     assert config.option.dist == "load"
@@ -40,6 +44,36 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     assert config.getoption('numprocesses') == 99
 
 
+def test_boxed_with_collect_only(testdir):
+    from xdist.plugin import pytest_cmdline_main as check_options
+    config = testdir.parseconfigure("-n1", "--boxed")
+    check_options(config)
+    assert config.option.forked
+
+    config = testdir.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    assert not config.option.forked
+
+    config = testdir.parseconfigure("-n1", "--boxed", "--collect-only")
+    check_options(config)
+    assert config.option.forked
+
+
+def test_dsession_with_collect_only(testdir):
+    from xdist.plugin import pytest_cmdline_main as check_options
+    from xdist.plugin import pytest_configure as configure
+
+    config = testdir.parseconfigure("-n1")
+    check_options(config)
+    configure(config)
+    assert config.pluginmanager.hasplugin("dsession")
+
+    config = testdir.parseconfigure("-n1", "--collect-only")
+    check_options(config)
+    configure(config)
+    assert not config.pluginmanager.hasplugin("dsession")
+
+
 class TestDistOptions:
     def test_getxspecs(self, testdir):
         config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")

testing/test_remote.py

@@ -1,6 +1,6 @@
 import py
 import pytest
-from xdist.slavemanage import SlaveController, unserialize_report
+from xdist.workermanage import WorkerController, unserialize_report
 from xdist.remote import serialize_report
 import execnet
 import marshal
@@ -26,12 +26,12 @@ class EventCall:
         return "<EventCall %s(**%s)>" % (self.name, self.kwargs)
 
 
-class SlaveSetup:
+class WorkerSetup:
     use_callback = False
 
-    def __init__(self, request):
-        self.testdir = request.getfuncargvalue("testdir")
+    def __init__(self, request, testdir):
         self.request = request
+        self.testdir = testdir
         self.events = queue.Queue()
 
     def setup(self, ):
@@ -44,8 +44,8 @@ class SlaveSetup:
         class DummyMananger:
             specs = [0, 1]
 
-        self.slp = SlaveController(DummyMananger, self.gateway, config,
-                                   putevent)
+        self.slp = WorkerController(DummyMananger, self.gateway, config,
+                                    putevent)
         self.request.addfinalizer(self.slp.ensure_teardown)
         self.slp.setup()
 
@@ -65,8 +65,8 @@ class SlaveSetup:
 
 
 @pytest.fixture
-def slave(request):
-    return SlaveSetup(request)
+def worker(request, testdir):
+    return WorkerSetup(request, testdir)
 
 
 @pytest.mark.xfail(reason='#59')
@@ -79,6 +79,97 @@ def test_remoteinitconfig(testdir):
 
 
 class TestReportSerialization:
+    def test_xdist_longrepr_to_str_issue_241(self, testdir):
+        testdir.makepyfile("""
+            import os
+            def test_a(): assert False
+            def test_b(): pass
+        """)
+        testdir.makeconftest("""
+            def pytest_runtest_logreport(report):
+                print(report.longrepr)
+        """)
+        res = testdir.runpytest('-n1', '-s')
+        res.stdout.fnmatch_lines([
+            '*1 failed, 1 passed *'
+        ])
+
+    def test_xdist_report_longrepr_reprcrash_130(self, testdir):
+        reprec = testdir.inline_runsource("""
+                    import py
+                    def test_fail(): assert False, 'Expected Message'
+                """)
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        added_section = ('Failure Metadata', str("metadata metadata"), "*")
+        rep.longrepr.sections.append(added_section)
+        d = serialize_report(rep)
+        check_marshallable(d)
+        a = unserialize_report("testreport", d)
+        # Check assembled == rep
+        assert a.__dict__.keys() == rep.__dict__.keys()
+        for key in rep.__dict__.keys():
+            if key != 'longrepr':
+                assert getattr(a, key) == getattr(rep, key)
+        assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno
+        assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message
+        assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path
+        assert rep.longrepr.reprtraceback.entrysep \
+            == a.longrepr.reprtraceback.entrysep
+        assert rep.longrepr.reprtraceback.extraline \
+            == a.longrepr.reprtraceback.extraline
+        assert rep.longrepr.reprtraceback.style \
+            == a.longrepr.reprtraceback.style
+        assert rep.longrepr.sections == a.longrepr.sections
+        # Missing section attribute PR171
+        assert added_section in a.longrepr.sections
+
+    def test_reprentries_serialization_170(self, testdir):
+        from _pytest._code.code import ReprEntry
+        reprec = testdir.inline_runsource("""
+                            def test_repr_entry():
+                                x = 0
+                                assert x
+                        """, '--showlocals')
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        d = serialize_report(rep)
+        a = unserialize_report("testreport", d)
+
+        rep_entries = rep.longrepr.reprtraceback.reprentries
+        a_entries = a.longrepr.reprtraceback.reprentries
+        for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntry)
+            assert rep_entries[i].lines == a_entries[i].lines
+            assert rep_entries[i].localssep == a_entries[i].localssep
+            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
+            assert rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
+            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
+            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
+            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
+            assert rep_entries[i].style == a_entries[i].style
+
+    def test_reprentries_serialization_196(self, testdir):
+        from _pytest._code.code import ReprEntryNative
+        reprec = testdir.inline_runsource("""
+                            def test_repr_entry_native():
+                                x = 0
+                                assert x
+                        """, '--tb=native')
+        reports = reprec.getreports("pytest_runtest_logreport")
+        assert len(reports) == 3
+        rep = reports[1]
+        d = serialize_report(rep)
+        a = unserialize_report("testreport", d)
+
+        rep_entries = rep.longrepr.reprtraceback.reprentries
+        a_entries = a.longrepr.reprtraceback.reprentries
+        for i in range(len(a_entries)):
+            assert isinstance(rep_entries[i], ReprEntryNative)
+            assert rep_entries[i].lines == a_entries[i].lines
+
     def test_itemreport_outcomes(self, testdir):
         reprec = testdir.inline_runsource("""
             import py
@@ -108,7 +199,7 @@ class TestReportSerialization:
             assert newrep.when == rep.when
             assert newrep.keywords == rep.keywords
             if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
+                assert newrep.longreprtext == rep.longreprtext
 
     def test_collectreport_passed(self, testdir):
         reprec = testdir.inline_runsource("def test_func(): pass")
@@ -152,107 +243,107 @@ class TestReportSerialization:
                 assert newrep.longrepr == str(rep.longrepr)
 
 
-class TestSlaveInteractor:
-    def test_basic_collect_and_runtests(self, slave):
-        slave.testdir.makepyfile("""
+class TestWorkerInteractor:
+    def test_basic_collect_and_runtests(self, worker):
+        worker.testdir.makepyfile("""
             def test_func():
                 pass
         """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
-        assert ev.kwargs['topdir'] == slave.testdir.tmpdir
+        ev = worker.popevent("collectionfinish")
+        assert ev.kwargs['topdir'] == worker.testdir.tmpdir
         ids = ev.kwargs['ids']
         assert len(ids) == 1
-        slave.sendcommand("runtests", indices=list(range(len(ids))))
-        slave.sendcommand("shutdown")
-        ev = slave.popevent("logstart")
+        worker.sendcommand("runtests", indices=list(range(len(ids))))
+        worker.sendcommand("shutdown")
+        ev = worker.popevent("logstart")
         assert ev.kwargs["nodeid"].endswith("test_func")
         assert len(ev.kwargs["location"]) == 3
-        ev = slave.popevent("testreport")  # setup
-        ev = slave.popevent("testreport")
+        ev = worker.popevent("testreport")  # setup
+        ev = worker.popevent("testreport")
         assert ev.name == "testreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert 'workeroutput' in ev.kwargs
 
     @pytest.mark.skipif(pytest.__version__ >= '3.0',
                         reason='skip at module level illegal in pytest 3.0')
-    def test_remote_collect_skip(self, slave):
-        slave.testdir.makepyfile("""
+    def test_remote_collect_skip(self, worker):
+        worker.testdir.makepyfile("""
             import py
             py.test.skip("hello")
         """)
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.skipped
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         assert not ev.kwargs['ids']
 
-    def test_remote_collect_fail(self, slave):
-        slave.testdir.makepyfile("""aasd qwe""")
-        slave.setup()
-        ev = slave.popevent("collectionstart")
+    def test_remote_collect_fail(self, worker):
+        worker.testdir.makepyfile("""aasd qwe""")
+        worker.setup()
+        ev = worker.popevent("collectionstart")
         assert not ev.kwargs
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = slave.popevent()
+        ev = worker.popevent()
         assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs['data'])
         assert rep.failed
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         assert not ev.kwargs['ids']
 
-    def test_runtests_all(self, slave):
-        slave.testdir.makepyfile("""
+    def test_runtests_all(self, worker):
+        worker.testdir.makepyfile("""
             def test_func(): pass
             def test_func2(): pass
         """)
-        slave.setup()
-        ev = slave.popevent()
-        assert ev.name == "slaveready"
-        ev = slave.popevent()
+        worker.setup()
+        ev = worker.popevent()
+        assert ev.name == "workerready"
+        ev = worker.popevent()
         assert ev.name == "collectionstart"
         assert not ev.kwargs
-        ev = slave.popevent("collectionfinish")
+        ev = worker.popevent("collectionfinish")
         ids = ev.kwargs['ids']
         assert len(ids) == 2
-        slave.sendcommand("runtests_all", )
-        slave.sendcommand("shutdown", )
+        worker.sendcommand("runtests_all", )
+        worker.sendcommand("shutdown", )
         for func in "::test_func", "::test_func2":
             for i in range(3):  # setup/call/teardown
-                ev = slave.popevent("testreport")
+                ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
                 rep = unserialize_report(ev.name, ev.kwargs['data'])
                 assert rep.nodeid.endswith(func)
-        ev = slave.popevent("slavefinished")
-        assert 'slaveoutput' in ev.kwargs
+        ev = worker.popevent("workerfinished")
+        assert 'workeroutput' in ev.kwargs
 
-    def test_happy_run_events_converted(self, testdir, slave):
+    def test_happy_run_events_converted(self, testdir, worker):
         py.test.xfail("implement a simple test for event production")
-        assert not slave.use_callback
-        slave.testdir.makepyfile("""
+        assert not worker.use_callback
+        worker.testdir.makepyfile("""
             def test_func():
                 pass
         """)
-        slave.setup()
-        hookrec = testdir.getreportrecorder(slave.config)
-        for data in slave.slp.channel:
-            slave.slp.process_from_remote(data)
-        slave.slp.process_from_remote(slave.slp.ENDMARK)
+        worker.setup()
+        hookrec = testdir.getreportrecorder(worker.config)
+        for data in worker.slp.channel:
+            worker.slp.process_from_remote(data)
+        worker.slp.process_from_remote(worker.slp.ENDMARK)
         py.std.pprint.pprint(hookrec.hookrecorder.calls)
         hookrec.hookrecorder.contains([
             ("pytest_collectstart", "collector.fspath == aaa"),
@@ -263,6 +354,15 @@ class TestSlaveInteractor:
             ("pytest_collectreport", "report.collector.fspath == bbb"),
         ])
 
+    def test_process_from_remote_error_handling(self, worker, capsys):
+        worker.use_callback = True
+        worker.setup()
+        worker.slp.process_from_remote(('<nonono>', ()))
+        out, err = capsys.readouterr()
+        assert 'INTERNALERROR> ValueError: unknown event: <nonono>' in out
+        ev = worker.popevent()
+        assert ev.name == "errordown"
+
 
 def test_remote_env_vars(testdir):
     testdir.makepyfile('''
@@ -271,5 +371,5 @@ def test_remote_env_vars(testdir):
             assert os.environ['PYTEST_XDIST_WORKER'] in ('gw0', 'gw1')
             assert os.environ['PYTEST_XDIST_WORKER_COUNT'] == '2'
     ''')
-    result = testdir.runpytest('-n2', '--max-slave-restart=0')
+    result = testdir.runpytest('-n2', '--max-worker-restart=0')
     assert result.ret == 0

testing/test_slavemanage.py

@@ -2,8 +2,8 @@ import py
 import pytest
 import execnet
 from _pytest.pytester import HookRecorder
-from xdist import slavemanage, newhooks
-from xdist.slavemanage import HostRSync, NodeManager
+from xdist import workermanage, newhooks
+from xdist.workermanage import HostRSync, NodeManager
 
 pytest_plugins = "pytester"
 
@@ -32,7 +32,7 @@ def mysetup(tmpdir):
 
 
 @pytest.fixture
-def slavecontroller(monkeypatch):
+def workercontroller(monkeypatch):
     class MockController(object):
         def __init__(self, *args):
             pass
@@ -40,7 +40,7 @@ def slavecontroller(monkeypatch):
         def setup(self):
             pass
 
-    monkeypatch.setattr(slavemanage, 'SlaveController', MockController)
+    monkeypatch.setattr(workermanage, 'WorkerController', MockController)
     return MockController
 
 
@@ -50,14 +50,14 @@ class TestNodeManagerPopen:
         assert gm.specs[0].chdir is None
 
     def test_default_chdir(self, config):
-        l = ["ssh=noco", "socket=xyz"]
-        for spec in NodeManager(config, l).specs:
+        specs = ["ssh=noco", "socket=xyz"]
+        for spec in NodeManager(config, specs).specs:
             assert spec.chdir == "pyexecnetcache"
-        for spec in NodeManager(config, l, defaultchdir="abc").specs:
+        for spec in NodeManager(config, specs, defaultchdir="abc").specs:
             assert spec.chdir == "abc"
 
     def test_popen_makegateway_events(self, config, hookrecorder,
-                                      slavecontroller):
+                                      workercontroller):
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
         call = hookrecorder.popcall("pytest_xdist_setupnodes")
@@ -72,7 +72,7 @@ class TestNodeManagerPopen:
         hm.teardown_nodes()
         assert not len(hm.group)
 
-    def test_popens_rsync(self, config, mysetup, slavecontroller):
+    def test_popens_rsync(self, config, mysetup, workercontroller):
         source = mysetup.source
         hm = NodeManager(config, ["popen"] * 2)
         hm.setup_nodes(None)
@@ -89,24 +89,24 @@ class TestNodeManagerPopen:
                     pass
 
             gw.remote_exec = pseudoexec
-        l = []
+        notifications = []
         for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: l.append(args))
-        assert not l
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert not notifications
         hm.teardown_nodes()
         assert not len(hm.group)
         assert "sys.path.insert" in gw.remote_exec.args[0]
 
-    def test_rsync_popen_with_path(self, config, mysetup, slavecontroller):
+    def test_rsync_popen_with_path(self, config, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
         hm.setup_nodes(None)
         source.ensure("dir1", "dir2", "hello")
-        l = []
+        notifications = []
         for gw in hm.group:
-            hm.rsync(gw, source, notify=lambda *args: l.append(args))
-        assert len(l) == 1
-        assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
+            hm.rsync(gw, source, notify=lambda *args: notifications.append(args))
+        assert len(notifications) == 1
+        assert notifications[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
         hm.teardown_nodes()
         dest = dest.join(source.basename)
         assert dest.join("dir1").check()
@@ -114,7 +114,7 @@ class TestNodeManagerPopen:
         assert dest.join("dir1", "dir2", 'hello').check()
 
     def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder,
-                                    slavecontroller):
+                                    workercontroller):
         source, dest = mysetup.source, mysetup.dest
         hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
         hm.roots = []
@@ -137,9 +137,9 @@ class TestHRSync:
         source.ensure(".somedotfile", "moreentries")
         source.ensure("somedir", "editfile~")
         syncer = HostRSync(source, ignores=NodeManager.DEFAULT_IGNORES)
-        l = list(source.visit(rec=syncer.filter, fil=syncer.filter))
-        assert len(l) == 3
-        basenames = [x.basename for x in l]
+        files = list(source.visit(rec=syncer.filter, fil=syncer.filter))
+        assert len(files) == 3
+        basenames = [x.basename for x in files]
         assert 'dir' in basenames
         assert 'file.txt' in basenames
         assert 'somedir' in basenames
@@ -174,7 +174,7 @@ class TestNodeManager:
         assert p.join("dir1").check()
         assert p.join("dir1", "file1").check()
 
-    def test_popen_rsync_subdir(self, testdir, mysetup, slavecontroller):
+    def test_popen_rsync_subdir(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir1 = mysetup.source.mkdir("dir1")
         dir2 = dir1.mkdir("dir2")
@@ -192,7 +192,7 @@ class TestNodeManager:
             assert dest.join("dir1", "dir2", 'hello').check()
             nodemanager.teardown_nodes()
 
-    def test_init_rsync_roots(self, testdir, mysetup, slavecontroller):
+    def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)
         source.ensure("dir1", "somefile", dir=1)
@@ -209,7 +209,7 @@ class TestNodeManager:
         assert not dest.join("dir1").check()
         assert not dest.join("bogus").check()
 
-    def test_rsyncignore(self, testdir, mysetup, slavecontroller):
+    def test_rsyncignore(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)
         source.ensure("dir5", "dir6", "bogus")
@@ -233,7 +233,7 @@ class TestNodeManager:
         assert not dest.join('foo').check()
         assert not dest.join('bar').check()
 
-    def test_optimise_popen(self, testdir, mysetup, slavecontroller):
+    def test_optimise_popen(self, testdir, mysetup, workercontroller):
         source = mysetup.source
         specs = ["popen"] * 3
         source.join("conftest.py").write("rsyncdirs = ['a']")

tox.ini

@@ -1,9 +1,9 @@
 [tox]
 # if you change the envlist, please update .travis.yml file as well
 envlist=
-  py{26,27,34}-pytest2{6,7,8,9}
-  py35-pytest2{7,8,9}
-  py{27,35}-pytest28-pexpect
+  py{27,34,35,36}-pytest{30,31,32,33}
+  py{27,36}-pytest{30,31,32,33}-pexpect
+  py{27,36}-pytest{master,features}
   flakes
   readme
 
@@ -15,10 +15,12 @@ deps =
   pycmd
   # to avoid .eggs
   setuptools_scm
-  pytest26: pytest~=2.6.1
-  pytest27: pytest~=2.7.2
-  pytest28: pytest~=2.8.7
-  pytest29: pytest~=2.9.1
+  pytest30: pytest~=3.0.5
+  pytest31: pytest~=3.1.0
+  pytest32: pytest~=3.2.0
+  pytest33: pytest~=3.3.0
+  pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
+  pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect
 platform=
   pexpect: linux|darwin

xdist/boxed.py

@@ -1,60 +0,0 @@
-
-import py
-
-
-def pytest_addoption(parser):
-    group = parser.getgroup("xdist", "distributed and subprocess testing")
-    group.addoption(
-        '--boxed',
-        action="store_true", dest="boxed", default=False,
-        help="box each test run in a separate process (unix)")
-
-
-def pytest_runtest_protocol(item):
-    if item.config.getvalue("boxed"):
-        reports = forked_run_report(item)
-        for rep in reports:
-            item.ihook.pytest_runtest_logreport(report=rep)
-        return True
-
-
-def forked_run_report(item):
-    # for now, we run setup/teardown in the subprocess
-    # XXX optionally allow sharing of setup/teardown
-    from _pytest.runner import runtestprotocol
-    EXITSTATUS_TESTEXIT = 4
-    import marshal
-    from xdist.remote import serialize_report
-    from xdist.slavemanage import unserialize_report
-
-    def runforked():
-        try:
-            reports = runtestprotocol(item, log=False)
-        except KeyboardInterrupt:
-            py.std.os._exit(EXITSTATUS_TESTEXIT)
-        return marshal.dumps([serialize_report(x) for x in reports])
-
-    ff = py.process.ForkedFunc(runforked)
-    result = ff.waitfinish()
-    if result.retval is not None:
-        report_dumps = marshal.loads(result.retval)
-        return [unserialize_report("testreport", x) for x in report_dumps]
-    else:
-        if result.exitstatus == EXITSTATUS_TESTEXIT:
-            py.test.exit("forked test item %s raised Exit" % (item,))
-        return [report_process_crash(item, result)]
-
-
-def report_process_crash(item, result):
-    path, lineno = item._getfslineno()
-    info = ("%s:%s: running the test CRASHED with signal %d" %
-            (path, lineno, result.signal))
-    from _pytest import runner
-    call = runner.CallInfo(lambda: 0/0, "???")
-    call.excinfo = info
-    rep = runner.pytest_runtest_makereport(item, call)
-    if result.out:
-        rep.sections.append(("captured stdout", result.out))
-    if result.err:
-        rep.sections.append(("captured stderr", result.err))
-    return rep

xdist/dsession.py

@@ -1,449 +1,18 @@
-import difflib
-import itertools
-from _pytest.runner import CollectReport
-
-import pytest
 import py
-from xdist.slavemanage import NodeManager
+import pytest
+
+from xdist.workermanage import NodeManager
+from xdist.scheduler import (
+    EachScheduling,
+    LoadScheduling,
+    LoadScopeScheduling,
+    LoadFileScheduling,
+)
 
 
 queue = py.builtin._tryimport('queue', 'Queue')
 
 
-class EachScheduling:
-    """Implement scheduling of test items on all nodes
-
-    If a node gets added after the test run is started then it is
-    assumed to replace a node which got removed before it finished
-    its collection.  In this case it will only be used if a node
-    with the same spec got removed earlier.
-
-    Any nodes added after the run is started will only get items
-    assigned if a node with a matching spec was removed before it
-    finished all its pending items.  The new node will then be
-    assigned the remaining items from the removed node.
-    """
-
-    def __init__(self, numnodes, log=None):
-        self.numnodes = numnodes
-        self.node2collection = {}
-        self.node2pending = {}
-        self._started = []
-        self._removed2pending = {}
-        if log is None:
-            self.log = py.log.Producer("eachsched")
-        else:
-            self.log = log.eachsched
-        self.collection_is_completed = False
-
-    @property
-    def nodes(self):
-        """A list of all nodes in the scheduler."""
-        return list(self.node2pending.keys())
-
-    def hasnodes(self):
-        return bool(self.node2pending)
-
-    def haspending(self):
-        """Return True if there are pending test items
-
-        This indicates that collection has finished and nodes are
-        still processing test items, so this can be thought of as
-        "the scheduler is active".
-        """
-        for pending in self.node2pending.values():
-            if pending:
-                return True
-        return False
-
-    def addnode(self, node):
-        assert node not in self.node2pending
-        self.node2pending[node] = []
-
-    def tests_finished(self):
-        if not self.collection_is_completed:
-            return False
-        if self._removed2pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        """Add the collected test items from a node
-
-        Collection is complete once all nodes have submitted their
-        collection.  In this case its pending list is set to an empty
-        list.  When the collection is already completed this
-        submission is from a node which was restarted to replace a
-        dead node.  In this case we already assign the pending items
-        here.  In either case ``.init_distribute()`` will instruct the
-        node to start running the required tests.
-        """
-        assert node in self.node2pending
-        if not self.collection_is_completed:
-            self.node2collection[node] = list(collection)
-            self.node2pending[node] = []
-            if len(self.node2collection) >= self.numnodes:
-                self.collection_is_completed = True
-        elif self._removed2pending:
-            for deadnode in self._removed2pending:
-                if deadnode.gateway.spec == node.gateway.spec:
-                    dead_collection = self.node2collection[deadnode]
-                    if collection != dead_collection:
-                        msg = report_collection_diff(dead_collection,
-                                                     collection,
-                                                     deadnode.gateway.id,
-                                                     node.gateway.id)
-                        self.log(msg)
-                        return
-                    pending = self._removed2pending.pop(deadnode)
-                    self.node2pending[node] = pending
-                    break
-
-    def remove_item(self, node, item_index, duration=0):
-        self.node2pending[node].remove(item_index)
-
-    def remove_node(self, node):
-        # KeyError if we didn't get an addnode() yet
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-        crashitem = self.node2collection[node][pending.pop(0)]
-        if pending:
-            self._removed2pending[node] = pending
-        return crashitem
-
-    def init_distribute(self):
-        """Schedule the test items on the nodes
-
-        If the node's pending list is empty it is a new node which
-        needs to run all the tests.  If the pending list is already
-        populated (by ``.addnode_collection()``) then it replaces a
-        dead node and we only need to run those tests.
-        """
-        assert self.collection_is_completed
-        for node, pending in self.node2pending.items():
-            if node in self._started:
-                continue
-            if not pending:
-                pending[:] = range(len(self.node2collection[node]))
-                node.send_runtest_all()
-            else:
-                node.send_runtest_some(pending)
-            self._started.append(node)
-
-
-class LoadScheduling:
-    """Implement load scheduling across nodes.
-
-    This distributes the tests collected across all nodes so each test
-    is run just once.  All nodes collect and submit the test suite and
-    when all collections are received it is verified they are
-    identical collections.  Then the collection gets divided up in
-    chunks and chunks get submitted to nodes.  Whenever a node finishes
-    an item, it calls ``.remove_item()`` which will trigger the
-    scheduler to assign more tests if the number of pending tests for
-    the node falls below a low-watermark.
-
-    When created, ``numnodes`` defines how many nodes are expected to
-    submit a collection. This is used to know when all nodes have
-    finished collection or how large the chunks need to be created.
-
-    Attributes:
-
-    :numnodes: The expected number of nodes taking part.  The actual
-       number of nodes will vary during the scheduler's lifetime as
-       nodes are added by the DSession as they are brought up and
-       removed either because of a dead node or normal shutdown.  This
-       number is primarily used to know when the initial collection is
-       completed.
-
-    :node2collection: Map of nodes and their test collection.  All
-       collections should always be identical.
-
-    :node2pending: Map of nodes and the indices of their pending
-       tests.  The indices are an index into ``.pending`` (which is
-       identical to their own collection stored in
-       ``.node2collection``).
-
-    :collection: The one collection once it is validated to be
-       identical between all the nodes.  It is initialised to None
-       until ``.init_distribute()`` is called.
-
-    :pending: List of indices of globally pending tests.  These are
-       tests which have not yet been allocated to a chunk for a node
-       to process.
-
-    :log: A py.log.Producer instance.
-
-    :config: Config object, used for handling hooks.
-    """
-
-    def __init__(self, numnodes, log=None, config=None):
-        self.numnodes = numnodes
-        self.node2collection = {}
-        self.node2pending = {}
-        self.pending = []
-        self.collection = None
-        if log is None:
-            self.log = py.log.Producer("loadsched")
-        else:
-            self.log = log.loadsched
-        self.config = config
-
-    @property
-    def nodes(self):
-        """A list of all nodes in the scheduler."""
-        return list(self.node2pending.keys())
-
-    @property
-    def collection_is_completed(self):
-        """Boolean indication initial test collection is complete.
-
-        This is a boolean indicating all initial participating nodes
-        have finished collection.  The required number of initial
-        nodes is defined by ``.numnodes``.
-        """
-        return len(self.node2collection) >= self.numnodes
-
-    def haspending(self):
-        """Return True if there are pending test items
-
-        This indicates that collection has finished and nodes are
-        still processing test items, so this can be thought of as
-        "the scheduler is active".
-        """
-        if self.pending:
-            return True
-        for pending in self.node2pending.values():
-            if pending:
-                return True
-        return False
-
-    def hasnodes(self):
-        """Return True if nodes exist in the scheduler."""
-        return bool(self.node2pending)
-
-    def addnode(self, node):
-        """Add a new node to the scheduler.
-
-        From now on the node will be allocated chunks of tests to
-        execute.
-
-        Called by the ``DSession.slave_slaveready`` hook when it
-        successfully bootstraps a new node.
-        """
-        assert node not in self.node2pending
-        self.node2pending[node] = []
-
-    def tests_finished(self):
-        """Return True if all tests have been executed by the nodes."""
-        if not self.collection_is_completed:
-            return False
-        if self.pending:
-            return False
-        for pending in self.node2pending.values():
-            if len(pending) >= 2:
-                return False
-        return True
-
-    def addnode_collection(self, node, collection):
-        """Add the collected test items from a node
-
-        The collection is stored in the ``.node2collection`` map.
-        Called by the ``DSession.slave_collectionfinish`` hook.
-        """
-        assert node in self.node2pending
-        if self.collection_is_completed:
-            # A new node has been added later, perhaps an original one died.
-            # .init_distribute() should have
-            # been called by now
-            assert self.collection
-            if collection != self.collection:
-                other_node = next(iter(self.node2collection.keys()))
-                msg = report_collection_diff(self.collection,
-                                             collection,
-                                             other_node.gateway.id,
-                                             node.gateway.id)
-                self.log(msg)
-                return
-        self.node2collection[node] = list(collection)
-
-    def remove_item(self, node, item_index, duration=0):
-        """Mark test item as completed by node
-
-        The duration it took to execute the item is used as a hint to
-        the scheduler.
-
-        This is called by the ``DSession.slave_testreport`` hook.
-        """
-        self.node2pending[node].remove(item_index)
-        self.check_schedule(node, duration=duration)
-
-    def check_schedule(self, node, duration=0):
-        """Maybe schedule new items on the node
-
-        If there are any globally pending nodes left then this will
-        check if the given node should be given any more tests.  The
-        ``duration`` of the last test is optionally used as a
-        heuristic to influence how many tests the node is assigned.
-        """
-        if node.shutting_down:
-            return
-
-        if self.pending:
-            # how many nodes do we have?
-            num_nodes = len(self.node2pending)
-            # if our node goes below a heuristic minimum, fill it out to
-            # heuristic maximum
-            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
-            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
-            node_pending = self.node2pending[node]
-            if len(node_pending) < items_per_node_min:
-                if duration >= 0.1 and len(node_pending) >= 2:
-                    # seems the node is doing long-running tests
-                    # and has enough items to continue
-                    # so let's rather wait with sending new items
-                    return
-                num_send = items_per_node_max - len(node_pending)
-                self._send_tests(node, num_send)
-        self.log("num items waiting for node:", len(self.pending))
-
-    def remove_node(self, node):
-        """Remove a node from the scheduler
-
-        This should be called either when the node crashed or at
-        shutdown time.  In the former case any pending items assigned
-        to the node will be re-scheduled.  Called by the
-        ``DSession.slave_slavefinished`` and
-        ``DSession.slave_errordown`` hooks.
-
-        Return the item which was being executing while the node
-        crashed or None if the node has no more pending items.
-
-        """
-        pending = self.node2pending.pop(node)
-        if not pending:
-            return
-
-        # The node crashed, reassing pending items
-        crashitem = self.collection[pending.pop(0)]
-        self.pending.extend(pending)
-        for node in self.node2pending:
-            self.check_schedule(node)
-        return crashitem
-
-    def init_distribute(self):
-        """Initiate distribution of the test collection
-
-        Initiate scheduling of the items across the nodes.  If this
-        gets called again later it behaves the same as calling
-        ``.check_schedule()`` on all nodes so that newly added nodes
-        will start to be used.
-
-        This is called by the ``DSession.slave_collectionfinish`` hook
-        if ``.collection_is_completed`` is True.
-
-        XXX Perhaps this method should have been called ".schedule()".
-        """
-        assert self.collection_is_completed
-
-        # Initial distribution already happened, reschedule on all nodes
-        if self.collection is not None:
-            for node in self.nodes:
-                self.check_schedule(node)
-            return
-
-        # XXX allow nodes to have different collections
-        if not self._check_nodes_have_same_collection():
-            self.log('**Different tests collected, aborting run**')
-            return
-
-        # Collections are identical, create the index of pending items.
-        self.collection = list(self.node2collection.values())[0]
-        self.pending[:] = range(len(self.collection))
-        if not self.collection:
-            return
-
-        # Send a batch of tests to run. If we don't have at least two
-        # tests per node, we have to send them all so that we can send
-        # shutdown signals and get all nodes working.
-        initial_batch = max(len(self.pending) // 4,
-                            2 * len(self.nodes))
-
-        # distribute tests round-robin up to the batch size
-        # (or until we run out)
-        nodes = itertools.cycle(self.nodes)
-        for i in range(initial_batch):
-            self._send_tests(next(nodes), 1)
-
-        if not self.pending:
-            # initial distribution sent all tests, start node shutdown
-            for node in self.nodes:
-                node.shutdown()
-
-    def _send_tests(self, node, num):
-        tests_per_node = self.pending[:num]
-        if tests_per_node:
-            del self.pending[:num]
-            self.node2pending[node].extend(tests_per_node)
-            node.send_runtest_some(tests_per_node)
-
-    def _check_nodes_have_same_collection(self):
-        """Return True if all nodes have collected the same items.
-
-        If collections differ, this method returns False while logging
-        the collection differences and posting collection errors to
-        pytest_collectreport hook.
-        """
-        node_collection_items = list(self.node2collection.items())
-        first_node, col = node_collection_items[0]
-        same_collection = True
-        for node, collection in node_collection_items[1:]:
-            msg = report_collection_diff(
-                col,
-                collection,
-                first_node.gateway.id,
-                node.gateway.id,
-            )
-            if msg:
-                same_collection = False
-                self.log(msg)
-                if self.config is not None:
-                    rep = CollectReport(
-                        node.gateway.id, 'failed',
-                        longrepr=msg, result=[])
-                    self.config.hook.pytest_collectreport(report=rep)
-
-        return same_collection
-
-
-def report_collection_diff(from_collection, to_collection, from_id, to_id):
-    """Report the collected test difference between two nodes.
-
-    :returns: detailed message describing the difference between the given
-    collections, or None if they are equal.
-    """
-    if from_collection == to_collection:
-        return None
-
-    diff = difflib.unified_diff(
-        from_collection,
-        to_collection,
-        fromfile=from_id,
-        tofile=to_id,
-    )
-    error_message = py.builtin._totext(
-        'Different tests were collected between {from_id} and {to_id}. '
-        'The difference is:\n'
-        '{diff}'
-    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
-    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
-    return msg
-
-
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
 
@@ -453,7 +22,7 @@ class DSession:
 
     At the beginning of the test session this creates a NodeManager
     instance which creates and starts all nodes.  Nodes then emit
-    events processed in the pytest_runtestloop hook using the slave_*
+    events processed in the pytest_runtestloop hook using the worker_*
     methods.
 
     Once a node is started it will automatically start running the
@@ -461,11 +30,14 @@ class DSession:
     automatically starts collecting tests.  Once tests are collected
     it will wait for instructions.
     """
+
     def __init__(self, config):
         self.config = config
         self.log = py.log.Producer("dsession")
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
+        self.nodemanager = None
+        self.sched = None
         self.shuttingdown = False
         self.countfailures = 0
         self.maxfail = config.getvalue("maxfail")
@@ -474,9 +46,9 @@ class DSession:
         self._failed_collection_errors = {}
         self._active_nodes = set()
         self._failed_nodes_count = 0
-        self._max_slave_restart = self.config.getoption('max_slave_restart')
-        if self._max_slave_restart is not None:
-            self._max_slave_restart = int(self._max_slave_restart)
+        self._max_worker_restart = self.config.option.maxworkerrestart
+        if self._max_worker_restart is not None:
+            self._max_worker_restart = int(self._max_worker_restart)
         try:
             self.terminal = config.pluginmanager.getplugin("terminalreporter")
         except KeyError:
@@ -503,7 +75,7 @@ class DSession:
         """Creates and starts the nodes.
 
         The nodes are setup to put their events onto self.queue.  As
-        soon as nodes start they will emit the slave_slaveready event.
+        soon as nodes start they will emit the worker_workerready event.
         """
         self.nodemanager = NodeManager(self.config)
         nodes = self.nodemanager.setup_nodes(putevent=self.queue.put)
@@ -521,16 +93,24 @@ class DSession:
         # prohibit collection of test items in master process
         return True
 
+    @pytest.mark.trylast
+    def pytest_xdist_make_scheduler(self, config, log):
+        dist = config.getvalue('dist')
+        schedulers = {
+            'each': EachScheduling,
+            'load': LoadScheduling,
+            'loadscope': LoadScopeScheduling,
+            'loadfile': LoadFileScheduling,
+        }
+        return schedulers[dist](config, log)
+
     def pytest_runtestloop(self):
-        numnodes = len(self.nodemanager.specs)
-        dist = self.config.getvalue("dist")
-        if dist == "load":
-            self.sched = LoadScheduling(numnodes, log=self.log,
-                                        config=self.config)
-        elif dist == "each":
-            self.sched = EachScheduling(numnodes, log=self.log)
-        else:
-            assert 0, dist
+        self.sched = self.config.hook.pytest_xdist_make_scheduler(
+            config=self.config,
+            log=self.log
+        )
+        assert self.sched is not None
+
         self.shouldstop = False
         while not self.session_finished:
             self.loop_once()
@@ -540,8 +120,12 @@ class DSession:
         return True
 
     def loop_once(self):
-        """Process one callback from one of the slaves."""
+        """Process one callback from one of the workers."""
         while 1:
+            if not self._active_nodes:
+                # If everything has died stop looping
+                self.triggershutdown()
+                raise RuntimeError("Unexpectedly no active workers available")
             try:
                 eventcall = self.queue.get(timeout=2.0)
                 break
@@ -549,52 +133,56 @@ class DSession:
                 continue
         callname, kwargs = eventcall
         assert callname, kwargs
-        method = "slave_" + callname
+        method = "worker_" + callname
         call = getattr(self, method)
         self.log("calling method", method, kwargs)
         call(**kwargs)
-        if self.sched.tests_finished():
+        if self.sched.tests_finished:
             self.triggershutdown()
 
     #
-    # callbacks for processing events from slaves
+    # callbacks for processing events from workers
     #
 
-    def slave_slaveready(self, node, slaveinfo):
+    def worker_workerready(self, node, workerinfo):
         """Emitted when a node first starts up.
 
         This adds the node to the scheduler, nodes continue with
         collection without any further input.
         """
-        node.slaveinfo = slaveinfo
-        node.slaveinfo['id'] = node.gateway.id
-        node.slaveinfo['spec'] = node.gateway.spec
+        node.workerinfo = workerinfo
+        node.workerinfo['id'] = node.gateway.id
+        node.workerinfo['spec'] = node.gateway.spec
+
+        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
+        node.slaveinfo = node.workerinfo
+
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
         else:
-            self.sched.addnode(node)
+            self.sched.add_node(node)
 
-    def slave_slavefinished(self, node):
+    def worker_workerfinished(self, node):
         """Emitted when node executes its pytest_sessionfinish hook.
 
         Removes the node from the scheduler.
 
         The node might not be in the scheduler if it had not emitted
-        slaveready before shutdown was triggered.
+        workerready before shutdown was triggered.
         """
         self.config.hook.pytest_testnodedown(node=node, error=None)
-        if node.slaveoutput['exitstatus'] == 2:  # keyboard-interrupt
+        if node.workeroutput['exitstatus'] == 2:  # keyboard-interrupt
             self.shouldstop = "%s received keyboard-interrupt" % (node,)
-            self.slave_errordown(node, "keyboard-interrupt")
+            self.worker_errordown(node, "keyboard-interrupt")
             return
         if node in self.sched.nodes:
             crashitem = self.sched.remove_node(node)
             assert not crashitem, (crashitem, node)
         self._active_nodes.remove(node)
 
-    def slave_errordown(self, node, error):
-        """Emitted by the SlaveController when a node dies."""
+    def worker_errordown(self, node, error):
+        """Emitted by the WorkerController when a node dies."""
         self.config.hook.pytest_testnodedown(node=node, error=error)
         try:
             crashitem = self.sched.remove_node(node)
@@ -605,22 +193,22 @@ class DSession:
                 self.handle_crashitem(crashitem, node)
 
         self._failed_nodes_count += 1
-        maximum_reached = (self._max_slave_restart is not None and
-                           self._failed_nodes_count > self._max_slave_restart)
+        maximum_reached = (self._max_worker_restart is not None and
+                           self._failed_nodes_count > self._max_worker_restart)
         if maximum_reached:
-            if self._max_slave_restart == 0:
-                msg = 'Slave restarting disabled'
+            if self._max_worker_restart == 0:
+                msg = 'Worker restarting disabled'
             else:
-                msg = "Maximum crashed slaves reached: %d" % \
-                      self._max_slave_restart
+                msg = "Maximum crashed workers reached: %d" % \
+                      self._max_worker_restart
             self.report_line(msg)
         else:
-            self.report_line("Replacing crashed slave %s" % node.gateway.id)
+            self.report_line("Replacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
-    def slave_collectionfinish(self, node, ids):
-        """Slave has finished test collection.
+    def worker_collectionfinish(self, node, ids):
+        """worker has finished test collection.
 
         This adds the collection for this node to the scheduler.  If
         the scheduler indicates collection is finished (i.e. all
@@ -635,39 +223,50 @@ class DSession:
         # tell session which items were effectively collected otherwise
         # the master node will finish the session with EXIT_NOTESTSCOLLECTED
         self._session.testscollected = len(ids)
-        self.sched.addnode_collection(node, ids)
+        self.sched.add_node_collection(node, ids)
         if self.terminal:
             self.trdist.setstatus(node.gateway.spec, "[%d]" % (len(ids)))
         if self.sched.collection_is_completed:
-            if self.terminal and not self.sched.haspending():
+            if self.terminal and not self.sched.has_pending:
                 self.trdist.ensure_show_status()
                 self.terminal.write_line("")
                 self.terminal.write_line("scheduling tests via %s" % (
                     self.sched.__class__.__name__))
-            self.sched.init_distribute()
+            self.sched.schedule()
 
-    def slave_logstart(self, node, nodeid, location):
+    def worker_logstart(self, node, nodeid, location):
         """Emitted when a node calls the pytest_runtest_logstart hook."""
         self.config.hook.pytest_runtest_logstart(
             nodeid=nodeid, location=location)
 
-    def slave_testreport(self, node, rep):
-        """Emitted when a node calls the pytest_runtest_logreport hook.
+    def worker_logfinish(self, node, nodeid, location):
+        """Emitted when a node calls the pytest_runtest_logfinish hook."""
+        self.config.hook.pytest_runtest_logfinish(
+            nodeid=nodeid, location=location)
 
-        If the node indicates it is finished with a test item, remove
-        the item from the pending list in the scheduler.
-        """
-        if rep.when == "call" or (rep.when == "setup" and not rep.passed):
-            self.sched.remove_item(node, rep.item_index, rep.duration)
-        # self.report_line("testreport %s: %s" %(rep.id, rep.status))
+    def worker_testreport(self, node, rep):
+        """Emitted when a node calls the pytest_runtest_logreport hook."""
         rep.node = node
         self.config.hook.pytest_runtest_logreport(report=rep)
         self._handlefailures(rep)
 
-    def slave_collectreport(self, node, rep):
+    def worker_runtest_protocol_complete(self, node, item_index, duration):
+        """
+        Emitted when a node fires the 'runtest_protocol_complete' event,
+        signalling that a test has completed the runtestprotocol and should be
+        removed from the pending list in the scheduler.
+        """
+        self.sched.mark_test_complete(node, item_index, duration)
+
+    def worker_collectreport(self, node, rep):
         """Emitted when a node calls the pytest_collectreport hook."""
         if rep.failed:
-            self._failed_slave_collectreport(node, rep)
+            self._failed_worker_collectreport(node, rep)
+
+    def worker_logwarning(self, message, code, nodeid, fslocation):
+        """Emitted when a node calls the pytest_logwarning hook."""
+        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
+        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
     def _clone_node(self, node):
         """Return new node based on an existing one.
@@ -675,7 +274,7 @@ class DSession:
         This is normally for when a node dies, this will copy the spec
         of the existing node and create a new one with a new id.  The
         new node will have been setup so it will start calling the
-        "slave_*" hooks and do work soon.
+        "worker_*" hooks and do work soon.
         """
         spec = node.gateway.spec
         spec.id = None
@@ -684,9 +283,9 @@ class DSession:
         self._active_nodes.add(node)
         return node
 
-    def _failed_slave_collectreport(self, node, rep):
+    def _failed_worker_collectreport(self, node, rep):
         # Check we haven't already seen this report (from
-        # another slave).
+        # another worker).
         if rep.longrepr not in self._failed_collection_errors:
             self._failed_collection_errors[rep.longrepr] = True
             self.config.hook.pytest_collectreport(report=rep)
@@ -705,15 +304,15 @@ class DSession:
         for node in self.sched.nodes:
             node.shutdown()
 
-    def handle_crashitem(self, nodeid, slave):
+    def handle_crashitem(self, nodeid, worker):
         # XXX get more reporting info by recording pytest_runtest_logstart?
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Slave %r crashed while running %r" % (slave.gateway.id, nodeid)
+        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
         rep = runner.TestReport(nodeid, (fspath, None, fspath),
                                 (), "failed", msg, "???")
-        rep.node = slave
+        rep.node = worker
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
@@ -743,7 +342,7 @@ class TerminalDistReporter:
         return " / ".join(parts)
 
     def rewrite(self, line, newline=False):
-        pline = line + " " * max(self._lastlen-len(line), 0)
+        pline = line + " " * max(self._lastlen - len(line), 0)
         if newline:
             self._lastlen = 0
             pline += "\n"
@@ -769,7 +368,7 @@ class TerminalDistReporter:
 
     def pytest_testnodeready(self, node):
         if self.config.option.verbose > 0:
-            d = node.slaveinfo
+            d = node.workerinfo
             infoline = "[%s] Python %s" % (
                 d['id'],
                 d['version'].replace('\n', ' -- '),)

xdist/looponfail.py

@@ -69,10 +69,10 @@ class RemoteControl(object):
             out = py.io.TerminalWriter()
         if hasattr(self, 'gateway'):
             raise ValueError("already have gateway %r" % self.gateway)
-        self.trace("setting up slave session")
+        self.trace("setting up worker session")
         self.gateway = self.initgateway()
         self.channel = channel = self.gateway.remote_exec(
-            init_slave_session,
+            init_worker_session,
             args=self.config.args,
             option_dict=vars(self.config.option),
         )
@@ -134,7 +134,7 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
         tr.line("### Watching:   %s" % (rootdir,), bold=True)
 
 
-def init_slave_session(channel, args, option_dict):
+def init_worker_session(channel, args, option_dict):
     import os
     import sys
     outchannel = channel.gateway.newchannel()
@@ -153,11 +153,11 @@ def init_slave_session(channel, args, option_dict):
     from _pytest.config import Config
     config = Config.fromdictargs(option_dict, list(args))
     config.args = args
-    from xdist.looponfail import SlaveFailSession
-    SlaveFailSession(config, channel).main()
+    from xdist.looponfail import WorkerFailSession
+    WorkerFailSession(config, channel).main()
 
 
-class SlaveFailSession:
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -194,11 +194,11 @@ class SlaveFailSession:
             self.collection_failed = True
 
     def main(self):
-        self.DEBUG("SLAVE: received configuration, waiting for command trails")
+        self.DEBUG("WORKER: received configuration, waiting for command trails")
         try:
             command = self.channel.receive()
         except KeyboardInterrupt:
-            return  # in the slave we can't do much about this
+            return  # in the worker we can't do much about this
         self.DEBUG("received", command)
         self.current_command = command
         self.config.hook.pytest_cmdline_main(config=self.config)

xdist/newhooks.py

@@ -9,8 +9,9 @@ with the worker instance that executed the hook originally:
 You can use this hooks just as you would use normal pytest hooks, but some care
 must be taken in plugins in case ``xdist`` is not installed. Please see:
 
-    http://pytest.org/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
+    http://pytest.org/en/latest/writing_plugins.html#optionally-using-hooks-from-3rd-party-plugins
 """
+import pytest
 
 
 def pytest_xdist_setupnodes(config, specs):
@@ -44,3 +45,8 @@ def pytest_testnodedown(node, error):
 def pytest_xdist_node_collection_finished(node, ids):
     """called by the master node when a node finishes collecting.
     """
+
+
+@pytest.mark.firstresult
+def pytest_xdist_make_scheduler(config, log):
+    """ return a node scheduler implementation """

xdist/plugin.py

@@ -20,24 +20,32 @@ def parse_numprocesses(s):
 def pytest_addoption(parser):
     group = parser.getgroup("xdist", "distributed and subprocess testing")
     group._addoption(
-        '-n', dest="numprocesses", metavar="numprocesses",
+        '-n', '--numprocesses', dest="numprocesses", metavar="numprocesses",
         action="store",
         type=parse_numprocesses,
         help="shortcut for '--dist=load --tx=NUM*popen', "
              "you can use 'auto' here for auto detection CPUs number on "
              "host system")
-    group._addoption('--max-slave-restart', action="store", default=None,
-                     help="maximum number of slaves that can be restarted "
-                          "when crashed (set to zero to disable this feature)")
-    group._addoption(
+    group.addoption('--max-worker-restart', '--max-slave-restart', action="store", default=None,
+                    dest="maxworkerrestart",
+                    help="maximum number of workers that can be restarted "
+                         "when crashed (set to zero to disable this feature)\n"
+                         "'--max-slave-restart' option is deprecated and will be removed in "
+                         "a future release")
+    group.addoption(
         '--dist', metavar="distmode",
-        action="store", choices=['load', 'each', 'no'],
-        type="choice", dest="dist", default="no",
+        action="store", choices=['each', 'load', 'loadscope', 'loadfile', 'no'],
+        dest="dist", default="no",
         help=("set mode for distributing tests to exec environments.\n\n"
-              "each: send each test to each available environment.\n\n"
-              "load: send each test to available environment.\n\n"
+              "each: send each test to all available environments.\n\n"
+              "load: load balance by sending any pending test to any"
+              " available environment.\n\n"
+              "loadscope: load balance by sending pending groups of tests in"
+              " the same scope to any available environment.\n\n"
+              "loadfile: load balance by sending test grouped by file"
+              " to any available environment.\n\n"
               "(default) no: run tests inprocess, don't distribute."))
-    group._addoption(
+    group.addoption(
         '--tx', dest="tx", action="append", default=[],
         metavar="xspec",
         help=("add a test execution environment. some examples: "
@@ -54,6 +62,9 @@ def pytest_addoption(parser):
         '--rsyncignore', action="append", default=[], metavar="GLOB",
         help="add expression for ignores when rsyncing to remote tx nodes.")
 
+    group.addoption(
+        "--boxed", action="store_true",
+        help="backward compatibility alias for pytest-forked --forked")
     parser.addini(
         'rsyncdirs', 'list of (relative) paths to be rsynced for'
         ' remote distributed testing.', type="pathlist")
@@ -64,6 +75,7 @@ def pytest_addoption(parser):
         "looponfailroots", type="pathlist",
         help="directories to check for changes", default=[py.path.local()])
 
+
 # -------------------------------------------------------------------------
 # distributed testing hooks
 # -------------------------------------------------------------------------
@@ -84,18 +96,21 @@ def pytest_addhooks(pluginmanager):
 
 @pytest.mark.trylast
 def pytest_configure(config):
-    if config.getoption("dist") != "no":
+    if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
         from xdist.dsession import DSession
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
         tr = config.pluginmanager.getplugin("terminalreporter")
         tr.showfspath = False
+    if config.getoption("boxed"):
+        config.option.forked = True
 
 
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
     if config.option.numprocesses:
-        config.option.dist = "load"
+        if config.option.dist == 'no':
+            config.option.dist = "load"
         config.option.tx = ['popen'] * config.option.numprocesses
     if config.option.distload:
         config.option.dist = "load"
@@ -105,7 +120,7 @@ def pytest_cmdline_main(config):
         if val("dist") != "no":
             if usepdb:
                 raise pytest.UsageError(
-                    "--pdb incompatible with distributing tests.")
+                    "--pdb is incompatible with distributing tests; try using -n0.")  # noqa: E501
 
 # -------------------------------------------------------------------------
 # fixtures
@@ -114,7 +129,10 @@ def pytest_cmdline_main(config):
 
 @pytest.fixture(scope="session")
 def worker_id(request):
-    if hasattr(request.config, 'slaveinput'):
-        return request.config.slaveinput['slaveid']
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the master node.
+    """
+    if hasattr(request.config, 'workerinput'):
+        return request.config.workerinput['workerid']
     else:
         return 'master'

xdist/remote.py

@@ -8,13 +8,17 @@
 
 import sys
 import os
+import time
+
+import _pytest.hookspec
+import pytest
 
 
-class SlaveInteractor:
+class WorkerInteractor:
     def __init__(self, config, channel):
         self.config = config
-        self.slaveid = config.slaveinput.get('slaveid', "?")
-        self.log = py.log.Producer("slave-%s" % self.slaveid)
+        self.workerid = config.workerinput.get('workerid', "?")
+        self.log = py.log.Producer("worker-%s" % self.workerid)
         if not config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
         self.channel = channel
@@ -30,14 +34,14 @@ class SlaveInteractor:
 
     def pytest_sessionstart(self, session):
         self.session = session
-        slaveinfo = getinfodict()
-        self.sendevent("slaveready", slaveinfo=slaveinfo)
+        workerinfo = getinfodict()
+        self.sendevent("workerready", workerinfo=workerinfo)
 
-    def pytest_sessionfinish(self, __multicall__, exitstatus):
-        self.config.slaveoutput['exitstatus'] = exitstatus
-        res = __multicall__.execute()
-        self.sendevent("slavefinished", slaveoutput=self.config.slaveoutput)
-        return res
+    @pytest.hookimpl(hookwrapper=True)
+    def pytest_sessionfinish(self, exitstatus):
+        self.config.workeroutput['exitstatus'] = exitstatus
+        yield
+        self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
     def pytest_collection(self, session):
         self.sendevent("collectionstart")
@@ -58,23 +62,29 @@ class SlaveInteractor:
             self.log("items to run:", torun)
             # only run if we have an item and a next item
             while len(torun) >= 2:
-                self.run_tests(torun)
+                self.run_one_test(torun)
             if name == "shutdown":
                 if torun:
-                    self.run_tests(torun)
+                    self.run_one_test(torun)
                 break
         return True
 
-    def run_tests(self, torun):
+    def run_one_test(self, torun):
         items = self.session.items
         self.item_index = torun.pop(0)
+        item = items[self.item_index]
         if torun:
             nextitem = items[torun[0]]
         else:
             nextitem = None
+
+        start = time.time()
         self.config.hook.pytest_runtest_protocol(
-            item=items[self.item_index],
+            item=item,
             nextitem=nextitem)
+        duration = time.time() - start
+        self.sendevent("runtest_protocol_complete", item_index=self.item_index,
+                       duration=duration)
 
     def pytest_collection_finish(self, session):
         self.sendevent(
@@ -85,9 +95,15 @@ class SlaveInteractor:
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
+    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
+    if hasattr(_pytest.hookspec, 'pytest_runtest_logfinish'):
+        def pytest_runtest_logfinish(self, nodeid, location):
+            self.sendevent("logfinish", nodeid=nodeid, location=location)
+
     def pytest_runtest_logreport(self, report):
         data = serialize_report(report)
         data["item_index"] = self.item_index
+        data["worker_id"] = self.workerid
         assert self.session.items[self.item_index].nodeid == report.nodeid
         self.sendevent("testreport", data=data)
 
@@ -95,12 +111,43 @@ class SlaveInteractor:
         data = serialize_report(report)
         self.sendevent("collectreport", data=data)
 
+    def pytest_logwarning(self, message, code, nodeid, fslocation):
+        self.sendevent("logwarning", message=message, code=code, nodeid=nodeid,
+                       fslocation=str(fslocation))
+
 
 def serialize_report(rep):
+    def disassembled_report(rep):
+        reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
+        reprcrash = rep.longrepr.reprcrash.__dict__.copy()
+
+        new_entries = []
+        for entry in reprtraceback['reprentries']:
+            entry_data = {
+                'type': type(entry).__name__,
+                'data': entry.__dict__.copy(),
+            }
+            for key, value in entry_data['data'].items():
+                if hasattr(value, '__dict__'):
+                    entry_data['data'][key] = value.__dict__.copy()
+            new_entries.append(entry_data)
+
+        reprtraceback['reprentries'] = new_entries
+
+        return {
+            'reprcrash': reprcrash,
+            'reprtraceback': reprtraceback,
+            'sections': rep.longrepr.sections
+        }
+
     import py
     d = rep.__dict__.copy()
     if hasattr(rep.longrepr, 'toterminal'):
-        d['longrepr'] = str(rep.longrepr)
+        if hasattr(rep.longrepr, 'reprtraceback') \
+                and hasattr(rep.longrepr, 'reprcrash'):
+            d['longrepr'] = disassembled_report(rep)
+        else:
+            d['longrepr'] = str(rep.longrepr)
     else:
         d['longrepr'] = rep.longrepr
     for name in d:
@@ -138,22 +185,21 @@ def remote_initconfig(option_dict, args):
 
 if __name__ == '__channelexec__':
     channel = channel  # noqa
-    # python3.2 is not concurrent import safe, so let's play it safe
-    # https://bitbucket.org/hpk42/pytest/issue/347/pytest-xdist-and-python-32
-    if sys.version_info[:2] == (3, 2):
-        os.environ["PYTHONDONTWRITEBYTECODE"] = "1"
-    slaveinput, args, option_dict = channel.receive()
+    workerinput, args, option_dict = channel.receive()
     importpath = os.getcwd()
     sys.path.insert(0, importpath)  # XXX only for remote situations
     os.environ['PYTHONPATH'] = (
         importpath + os.pathsep +
         os.environ.get('PYTHONPATH', ''))
-    os.environ['PYTEST_XDIST_WORKER'] = slaveinput['slaveid']
-    os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(slaveinput['slavecount'])
+    os.environ['PYTEST_XDIST_WORKER'] = workerinput['workerid']
+    os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(workerinput['workercount'])
     # os.environ['PYTHONPATH'] = importpath
     import py
     config = remote_initconfig(option_dict, args)
-    config.slaveinput = slaveinput
-    config.slaveoutput = {}
-    interactor = SlaveInteractor(config, channel)
+    config.workerinput = workerinput
+    config.workeroutput = {}
+    # TODO: deprecated name, backward compatibility only. Remove it in future
+    config.slaveinput = config.workerinput
+    config.slaveoutput = config.workeroutput
+    interactor = WorkerInteractor(config, channel)
     config.hook.pytest_cmdline_main(config=config)

xdist/report.py

@@ -0,0 +1,26 @@
+import py
+from difflib import unified_diff
+
+
+def report_collection_diff(from_collection, to_collection, from_id, to_id):
+    """Report the collected test difference between two nodes.
+
+    :returns: detailed message describing the difference between the given
+    collections, or None if they are equal.
+    """
+    if from_collection == to_collection:
+        return None
+
+    diff = unified_diff(
+        from_collection,
+        to_collection,
+        fromfile=from_id,
+        tofile=to_id,
+    )
+    error_message = py.builtin._totext(
+        'Different tests were collected between {from_id} and {to_id}. '
+        'The difference is:\n'
+        '{diff}'
+    ).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
+    msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
+    return msg

xdist/scheduler/__init__.py

@@ -0,0 +1,4 @@
+from xdist.scheduler.each import EachScheduling  # noqa
+from xdist.scheduler.load import LoadScheduling  # noqa
+from xdist.scheduler.loadscope import LoadScopeScheduling  # noqa
+from xdist.scheduler.filescope import LoadFileScheduling  # noqa

xdist/scheduler/each.py

@@ -0,0 +1,129 @@
+from py.log import Producer
+
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class EachScheduling:
+    """Implement scheduling of test items on all nodes
+
+    If a node gets added after the test run is started then it is
+    assumed to replace a node which got removed before it finished
+    its collection.  In this case it will only be used if a node
+    with the same spec got removed earlier.
+
+    Any nodes added after the run is started will only get items
+    assigned if a node with a matching spec was removed before it
+    finished all its pending items.  The new node will then be
+    assigned the remaining items from the removed node.
+    """
+
+    def __init__(self, config, log=None):
+        self.config = config
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self._started = []
+        self._removed2pending = {}
+        if log is None:
+            self.log = Producer("eachsched")
+        else:
+            self.log = log.eachsched
+        self.collection_is_completed = False
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def tests_finished(self):
+        if not self.collection_is_completed:
+            return False
+        if self._removed2pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        Collection is complete once all nodes have submitted their
+        collection.  In this case its pending list is set to an empty
+        list.  When the collection is already completed this
+        submission is from a node which was restarted to replace a
+        dead node.  In this case we already assign the pending items
+        here.  In either case ``.schedule()`` will instruct the
+        node to start running the required tests.
+        """
+        assert node in self.node2pending
+        if not self.collection_is_completed:
+            self.node2collection[node] = list(collection)
+            self.node2pending[node] = []
+            if len(self.node2collection) >= self.numnodes:
+                self.collection_is_completed = True
+        elif self._removed2pending:
+            for deadnode in self._removed2pending:
+                if deadnode.gateway.spec == node.gateway.spec:
+                    dead_collection = self.node2collection[deadnode]
+                    if collection != dead_collection:
+                        msg = report_collection_diff(dead_collection,
+                                                     collection,
+                                                     deadnode.gateway.id,
+                                                     node.gateway.id)
+                        self.log(msg)
+                        return
+                    pending = self._removed2pending.pop(deadnode)
+                    self.node2pending[node] = pending
+                    break
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        self.node2pending[node].remove(item_index)
+
+    def remove_node(self, node):
+        # KeyError if we didn't get an add_node() yet
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+        crashitem = self.node2collection[node][pending.pop(0)]
+        if pending:
+            self._removed2pending[node] = pending
+        return crashitem
+
+    def schedule(self):
+        """Schedule the test items on the nodes
+
+        If the node's pending list is empty it is a new node which
+        needs to run all the tests.  If the pending list is already
+        populated (by ``.add_node_collection()``) then it replaces a
+        dead node and we only need to run those tests.
+        """
+        assert self.collection_is_completed
+        for node, pending in self.node2pending.items():
+            if node in self._started:
+                continue
+            if not pending:
+                pending[:] = range(len(self.node2collection[node]))
+                node.send_runtest_all()
+            else:
+                node.send_runtest_some(pending)
+            self._started.append(node)

xdist/scheduler/filescope.py

@@ -0,0 +1,52 @@
+from . import LoadScopeScheduling
+from py.log import Producer
+
+
+class LoadFileScheduling(LoadScopeScheduling):
+    """Implement load scheduling across nodes, but grouping test test file.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test file,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    This class behaves very much like LoadScopeScheduling, but with a file-level scope.
+    """
+
+    def __init(self, config, log=None):
+        super(LoadFileScheduling, self).__init__(config, log)
+        if log is None:
+            self.log = Producer('loadfilesched')
+        else:
+            self.log = log.loadfilesched
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the left. That is, test will be grouped in a
+        single work unit when they reside in the same file.
+         In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.split('::', 1)[0]

xdist/scheduler/load.py

@@ -0,0 +1,288 @@
+from itertools import cycle
+
+from py.log import Producer
+from _pytest.runner import CollectReport
+
+from xdist.workermanage import parse_spec_config
+from xdist.report import report_collection_diff
+
+
+class LoadScheduling:
+    """Implement load scheduling across nodes.
+
+    This distributes the tests collected across all nodes so each test
+    is run just once.  All nodes collect and submit the test suite and
+    when all collections are received it is verified they are
+    identical collections.  Then the collection gets divided up in
+    chunks and chunks get submitted to nodes.  Whenever a node finishes
+    an item, it calls ``.mark_test_complete()`` which will trigger the
+    scheduler to assign more tests if the number of pending tests for
+    the node falls below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to
+    submit a collection. This is used to know when all nodes have
+    finished collection or how large the chunks need to be created.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual
+       number of nodes will vary during the scheduler's lifetime as
+       nodes are added by the DSession as they are brought up and
+       removed either because of a dead node or normal shutdown.  This
+       number is primarily used to know when the initial collection is
+       completed.
+
+    :node2collection: Map of nodes and their test collection.  All
+       collections should always be identical.
+
+    :node2pending: Map of nodes and the indices of their pending
+       tests.  The indices are an index into ``.pending`` (which is
+       identical to their own collection stored in
+       ``.node2collection``).
+
+    :collection: The one collection once it is validated to be
+       identical between all the nodes.  It is initialised to None
+       until ``.schedule()`` is called.
+
+    :pending: List of indices of globally pending tests.  These are
+       tests which have not yet been allocated to a chunk for a node
+       to process.
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.node2collection = {}
+        self.node2pending = {}
+        self.pending = []
+        self.collection = None
+        if log is None:
+            self.log = Producer("loadsched")
+        else:
+            self.log = log.loadsched
+        self.config = config
+
+    @property
+    def nodes(self):
+        """A list of all nodes in the scheduler."""
+        return list(self.node2pending.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes
+        have finished collection.  The required number of initial
+        nodes is defined by ``.numnodes``.
+        """
+        return len(self.node2collection) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+        if self.pending:
+            return False
+        for pending in self.node2pending.values():
+            if len(pending) >= 2:
+                return False
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items
+
+        This indicates that collection has finished and nodes are
+        still processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.pending:
+            return True
+        for pending in self.node2pending.values():
+            if pending:
+                return True
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be allocated chunks of tests to
+        execute.
+
+        Called by the ``DSession.worker_workerready`` hook when it
+        successfully bootstraps a new node.
+        """
+        assert node not in self.node2pending
+        self.node2pending[node] = []
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node
+
+        The collection is stored in the ``.node2collection`` map.
+        Called by the ``DSession.worker_collectionfinish`` hook.
+        """
+        assert node in self.node2pending
+        if self.collection_is_completed:
+            # A new node has been added later, perhaps an original one died.
+            # .schedule() should have
+            # been called by now
+            assert self.collection
+            if collection != self.collection:
+                other_node = next(iter(self.node2collection.keys()))
+                msg = report_collection_diff(self.collection,
+                                             collection,
+                                             other_node.gateway.id,
+                                             node.gateway.id)
+                self.log(msg)
+                return
+        self.node2collection[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node
+
+        The duration it took to execute the item is used as a hint to
+        the scheduler.
+
+        This is called by the ``DSession.worker_testreport`` hook.
+        """
+        self.node2pending[node].remove(item_index)
+        self.check_schedule(node, duration=duration)
+
+    def check_schedule(self, node, duration=0):
+        """Maybe schedule new items on the node
+
+        If there are any globally pending nodes left then this will
+        check if the given node should be given any more tests.  The
+        ``duration`` of the last test is optionally used as a
+        heuristic to influence how many tests the node is assigned.
+        """
+        if node.shutting_down:
+            return
+
+        if self.pending:
+            # how many nodes do we have?
+            num_nodes = len(self.node2pending)
+            # if our node goes below a heuristic minimum, fill it out to
+            # heuristic maximum
+            items_per_node_min = max(2, len(self.pending) // num_nodes // 4)
+            items_per_node_max = max(2, len(self.pending) // num_nodes // 2)
+            node_pending = self.node2pending[node]
+            if len(node_pending) < items_per_node_min:
+                if duration >= 0.1 and len(node_pending) >= 2:
+                    # seems the node is doing long-running tests
+                    # and has enough items to continue
+                    # so let's rather wait with sending new items
+                    return
+                num_send = items_per_node_max - len(node_pending)
+                self._send_tests(node, num_send)
+        self.log("num items waiting for node:", len(self.pending))
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler
+
+        This should be called either when the node crashed or at
+        shutdown time.  In the former case any pending items assigned
+        to the node will be re-scheduled.  Called by the
+        ``DSession.worker_workerfinished`` and
+        ``DSession.worker_errordown`` hooks.
+
+        Return the item which was being executing while the node
+        crashed or None if the node has no more pending items.
+
+        """
+        pending = self.node2pending.pop(node)
+        if not pending:
+            return
+
+        # The node crashed, reassing pending items
+        crashitem = self.collection[pending.pop(0)]
+        self.pending.extend(pending)
+        for node in self.node2pending:
+            self.check_schedule(node)
+        return crashitem
+
+    def schedule(self):
+        """Initiate distribution of the test collection
+
+        Initiate scheduling of the items across the nodes.  If this
+        gets called again later it behaves the same as calling
+        ``.check_schedule()`` on all nodes so that newly added nodes
+        will start to be used.
+
+        This is called by the ``DSession.worker_collectionfinish`` hook
+        if ``.collection_is_completed`` is True.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self.check_schedule(node)
+            return
+
+        # XXX allow nodes to have different collections
+        if not self._check_nodes_have_same_collection():
+            self.log('**Different tests collected, aborting run**')
+            return
+
+        # Collections are identical, create the index of pending items.
+        self.collection = list(self.node2collection.values())[0]
+        self.pending[:] = range(len(self.collection))
+        if not self.collection:
+            return
+
+        # Send a batch of tests to run. If we don't have at least two
+        # tests per node, we have to send them all so that we can send
+        # shutdown signals and get all nodes working.
+        initial_batch = max(len(self.pending) // 4,
+                            2 * len(self.nodes))
+
+        # distribute tests round-robin up to the batch size
+        # (or until we run out)
+        nodes = cycle(self.nodes)
+        for i in range(initial_batch):
+            self._send_tests(next(nodes), 1)
+
+        if not self.pending:
+            # initial distribution sent all tests, start node shutdown
+            for node in self.nodes:
+                node.shutdown()
+
+    def _send_tests(self, node, num):
+        tests_per_node = self.pending[:num]
+        if tests_per_node:
+            del self.pending[:num]
+            self.node2pending[node].extend(tests_per_node)
+            node.send_runtest_some(tests_per_node)
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.node2collection.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+            if msg:
+                same_collection = False
+                self.log(msg)
+                if self.config is not None:
+                    rep = CollectReport(
+                        node.gateway.id, 'failed',
+                        longrepr=msg, result=[])
+                    self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

xdist/scheduler/loadscope.py

@@ -0,0 +1,427 @@
+from collections import OrderedDict
+
+from _pytest.runner import CollectReport
+from py.log import Producer
+from xdist.report import report_collection_diff
+from xdist.workermanage import parse_spec_config
+
+
+class LoadScopeScheduling:
+    """Implement load scheduling across nodes, but grouping test by scope.
+
+    This distributes the tests collected across all nodes so each test is run
+    just once.  All nodes collect and submit the list of tests and when all
+    collections are received it is verified they are identical collections.
+    Then the collection gets divided up in work units, grouped by test scope,
+    and those work units get submitted to nodes.  Whenever a node finishes an
+    item, it calls ``.mark_test_complete()`` which will trigger the scheduler
+    to assign more work units if the number of pending tests for the node falls
+    below a low-watermark.
+
+    When created, ``numnodes`` defines how many nodes are expected to submit a
+    collection. This is used to know when all nodes have finished collection.
+
+    Attributes:
+
+    :numnodes: The expected number of nodes taking part.  The actual number of
+       nodes will vary during the scheduler's lifetime as nodes are added by
+       the DSession as they are brought up and removed either because of a dead
+       node or normal shutdown.  This number is primarily used to know when the
+       initial collection is completed.
+
+    :collection: The final list of tests collected by all nodes once it is
+       validated to be identical between all the nodes.  It is initialised to
+       None until ``.schedule()`` is called.
+
+    :workqueue: Ordered dictionary that maps all available scopes with their
+       associated tests (nodeid). Nodeids are in turn associated with their
+       completion status. One entry of the workqueue is called a work unit.
+       In turn, a collection of work unit is called a workload.
+
+       ::
+
+            workqueue = {
+                '<full>/<path>/<to>/test_module.py': {
+                    '<full>/<path>/<to>/test_module.py::test_case1': False,
+                    '<full>/<path>/<to>/test_module.py::test_case2': False,
+                    (...)
+                },
+                (...)
+            }
+
+    :assigned_work: Ordered dictionary that maps worker nodes with their
+       assigned work units.
+
+       ::
+
+            assigned_work = {
+                '<worker node A>': {
+                    '<full>/<path>/<to>/test_module.py': {
+                        '<full>/<path>/<to>/test_module.py::test_case1': False,
+                        '<full>/<path>/<to>/test_module.py::test_case2': False,
+                        (...)
+                    },
+                    (...)
+                },
+                (...)
+            }
+
+    :registered_collections: Ordered dictionary that maps worker nodes with
+       their collection of tests gathered during test discovery.
+
+       ::
+
+            registered_collections = {
+                '<worker node A>': [
+                    '<full>/<path>/<to>/test_module.py::test_case1',
+                    '<full>/<path>/<to>/test_module.py::test_case2',
+                ],
+                (...)
+            }
+
+    :log: A py.log.Producer instance.
+
+    :config: Config object, used for handling hooks.
+    """
+
+    def __init__(self, config, log=None):
+        self.numnodes = len(parse_spec_config(config))
+        self.collection = None
+
+        self.workqueue = OrderedDict()
+        self.assigned_work = OrderedDict()
+        self.registered_collections = OrderedDict()
+
+        if log is None:
+            self.log = Producer('loadscopesched')
+        else:
+            self.log = log.loadscopesched
+
+        self.config = config
+
+    @property
+    def nodes(self):
+        """A list of all active nodes in the scheduler."""
+        return list(self.assigned_work.keys())
+
+    @property
+    def collection_is_completed(self):
+        """Boolean indication initial test collection is complete.
+
+        This is a boolean indicating all initial participating nodes have
+        finished collection.  The required number of initial nodes is defined
+        by ``.numnodes``.
+        """
+        return len(self.registered_collections) >= self.numnodes
+
+    @property
+    def tests_finished(self):
+        """Return True if all tests have been executed by the nodes."""
+        if not self.collection_is_completed:
+            return False
+
+        if self.workqueue:
+            return False
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) >= 2:
+                return False
+
+        return True
+
+    @property
+    def has_pending(self):
+        """Return True if there are pending test items.
+
+        This indicates that collection has finished and nodes are still
+        processing test items, so this can be thought of as
+        "the scheduler is active".
+        """
+        if self.workqueue:
+            return True
+
+        for assigned_unit in self.assigned_work.values():
+            if self._pending_of(assigned_unit) > 0:
+                return True
+
+        return False
+
+    def add_node(self, node):
+        """Add a new node to the scheduler.
+
+        From now on the node will be assigned work units to be executed.
+
+        Called by the ``DSession.worker_workerready`` hook when it successfully
+        bootstraps a new node.
+        """
+        assert node not in self.assigned_work
+        self.assigned_work[node] = OrderedDict()
+
+    def remove_node(self, node):
+        """Remove a node from the scheduler.
+
+        This should be called either when the node crashed or at shutdown time.
+        In the former case any pending items assigned to the node will be
+        re-scheduled.
+
+        Called by the hooks:
+
+        - ``DSession.worker_workerfinished``.
+        - ``DSession.worker_errordown``.
+
+        Return the item being executed while the node crashed or None if the
+        node has no more pending items.
+        """
+        workload = self.assigned_work.pop(node)
+        if not self._pending_of(workload):
+            return None
+
+        # The node crashed, identify test that crashed
+        for work_unit in workload.values():
+            for nodeid, completed in work_unit.items():
+                if not completed:
+                    crashitem = nodeid
+                    break
+            else:
+                continue
+            break
+        else:
+            raise RuntimeError(
+                'Unable to identify crashitem on a workload with '
+                'pending items'
+            )
+
+        # Made uncompleted work unit available again
+        self.workqueue.update(workload)
+
+        for node in self.assigned_work:
+            self._reschedule(node)
+
+        return crashitem
+
+    def add_node_collection(self, node, collection):
+        """Add the collected test items from a node.
+
+        The collection is stored in the ``.registered_collections`` dictionary.
+
+        Called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+
+        # Check that add_node() was called on the node before
+        assert node in self.assigned_work
+
+        # A new node has been added later, perhaps an original one died.
+        if self.collection_is_completed:
+
+            # Assert that .schedule() should have been called by now
+            assert self.collection
+
+            # Check that the new collection matches the official collection
+            if collection != self.collection:
+
+                other_node = next(iter(self.registered_collections.keys()))
+
+                msg = report_collection_diff(
+                    self.collection,
+                    collection,
+                    other_node.gateway.id,
+                    node.gateway.id
+                )
+                self.log(msg)
+                return
+
+        self.registered_collections[node] = list(collection)
+
+    def mark_test_complete(self, node, item_index, duration=0):
+        """Mark test item as completed by node.
+
+        Called by the hook:
+
+        - ``DSession.worker_testreport``.
+        """
+        nodeid = self.registered_collections[node][item_index]
+        scope = self._split_scope(nodeid)
+
+        self.assigned_work[node][scope][nodeid] = True
+        self._reschedule(node)
+
+    def _assign_work_unit(self, node):
+        """Assign a work unit to a node."""
+        assert self.workqueue
+
+        # Grab a unit of work
+        scope, work_unit = self.workqueue.popitem(last=False)
+
+        # Keep track of the assigned work
+        assigned_to_node = self.assigned_work.setdefault(
+            node, default=OrderedDict()
+        )
+        assigned_to_node[scope] = work_unit
+
+        # Ask the node to execute the workload
+        worker_collection = self.registered_collections[node]
+        nodeids_indexes = [
+            worker_collection.index(nodeid)
+            for nodeid, completed in work_unit.items()
+            if not completed
+        ]
+
+        node.send_runtest_some(nodeids_indexes)
+
+    def _split_scope(self, nodeid):
+        """Determine the scope (grouping) of a nodeid.
+
+        There are usually 3 cases for a nodeid::
+
+            example/loadsuite/test/test_beta.py::test_beta0
+            example/loadsuite/test/test_delta.py::Delta1::test_delta0
+            example/loadsuite/epsilon/__init__.py::epsilon.epsilon
+
+        #. Function in a test module.
+        #. Method of a class in a test module.
+        #. Doctest in a function in a package.
+
+        This function will group tests with the scope determined by splitting
+        the first ``::`` from the right. That is, classes will be grouped in a
+        single work unit, and functions from a test module will be grouped by
+        their module. In the above example, scopes will be::
+
+            example/loadsuite/test/test_beta.py
+            example/loadsuite/test/test_delta.py::Delta1
+            example/loadsuite/epsilon/__init__.py
+        """
+        return nodeid.rsplit('::', 1)[0]
+
+    def _pending_of(self, workload):
+        """Return the number of pending tests in a workload."""
+        pending = sum(
+            list(scope.values()).count(False)
+            for scope in workload.values()
+        )
+        return pending
+
+    def _reschedule(self, node):
+        """Maybe schedule new items on the node.
+
+        If there are any globally pending work units left then this will check
+        if the given node should be given any more tests.
+        """
+
+        # Do not add more work to a node shutting down
+        if node.shutting_down:
+            return
+
+        # Check that more work is available
+        if not self.workqueue:
+            return
+
+        self.log('Number of units waiting for node:', len(self.workqueue))
+
+        # Check that the node is almost depleted of work
+        # 2: Heuristic of minimum tests to enqueue more work
+        if self._pending_of(self.assigned_work[node]) > 2:
+            return
+
+        # Pop one unit of work and assign it
+        self._assign_work_unit(node)
+
+    def schedule(self):
+        """Initiate distribution of the test collection.
+
+        Initiate scheduling of the items across the nodes.  If this gets called
+        again later it behaves the same as calling ``._reschedule()`` on all
+        nodes so that newly added nodes will start to be used.
+
+        If ``.collection_is_completed`` is True, this is called by the hook:
+
+        - ``DSession.worker_collectionfinish``.
+        """
+        assert self.collection_is_completed
+
+        # Initial distribution already happened, reschedule on all nodes
+        if self.collection is not None:
+            for node in self.nodes:
+                self._reschedule(node)
+            return
+
+        # Check that all nodes collected the same tests
+        if not self._check_nodes_have_same_collection():
+            self.log('**Different tests collected, aborting run**')
+            return
+
+        # Collections are identical, create the final list of items
+        self.collection = list(
+            next(iter(self.registered_collections.values()))
+        )
+        if not self.collection:
+            return
+
+        # Determine chunks of work (scopes)
+        for nodeid in self.collection:
+            scope = self._split_scope(nodeid)
+            work_unit = self.workqueue.setdefault(scope, default=OrderedDict())
+            work_unit[nodeid] = False
+
+        # Avoid having more workers than work
+        extra_nodes = len(self.nodes) - len(self.workqueue)
+
+        if extra_nodes > 0:
+            self.log('Shuting down {0} nodes'.format(extra_nodes))
+
+            for _ in range(extra_nodes):
+                unused_node, assigned = self.assigned_work.popitem(last=True)
+
+                self.log('Shuting down unused node {0}'.format(unused_node))
+                unused_node.shutdown()
+
+        # Assign initial workload
+        for node in self.nodes:
+            self._assign_work_unit(node)
+
+        # Ensure nodes start with at least two work units if possible (#277)
+        for node in self.nodes:
+            self._reschedule(node)
+
+        # Initial distribution sent all tests, start node shutdown
+        if not self.workqueue:
+            for node in self.nodes:
+                node.shutdown()
+
+    def _check_nodes_have_same_collection(self):
+        """Return True if all nodes have collected the same items.
+
+        If collections differ, this method returns False while logging
+        the collection differences and posting collection errors to
+        pytest_collectreport hook.
+        """
+        node_collection_items = list(self.registered_collections.items())
+        first_node, col = node_collection_items[0]
+        same_collection = True
+
+        for node, collection in node_collection_items[1:]:
+            msg = report_collection_diff(
+                col,
+                collection,
+                first_node.gateway.id,
+                node.gateway.id,
+            )
+            if not msg:
+                continue
+
+            same_collection = False
+            self.log(msg)
+
+            if self.config is None:
+                continue
+
+            rep = CollectReport(
+                node.gateway.id,
+                'failed',
+                longrepr=msg,
+                result=[]
+            )
+            self.config.hook.pytest_collectreport(report=rep)
+
+        return same_collection

xdist/workermanage.py

@@ -10,6 +10,22 @@ import xdist.remote
 from _pytest import runner  # XXX load dynamically
 
 
+def parse_spec_config(config):
+    xspeclist = []
+    for xspec in config.getvalue("tx"):
+        i = xspec.find("*")
+        try:
+            num = int(xspec[:i])
+        except ValueError:
+            xspeclist.append(xspec)
+        else:
+            xspeclist.extend([xspec[i + 1:]] * num)
+    if not xspeclist:
+        raise pytest.UsageError(
+            "MISSING test execution (tx) nodes: please specify --tx")
+    return xspeclist
+
+
 class NodeManager(object):
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
@@ -52,7 +68,7 @@ class NodeManager(object):
         gw = self.group.makegateway(spec)
         self.config.hook.pytest_xdist_newgateway(gateway=gw)
         self.rsync_roots(gw)
-        node = SlaveController(self, gw, self.config, putevent)
+        node = WorkerController(self, gw, self.config, putevent)
         gw.node = node          # keep the node alive
         node.setup()
         self.trace("started node %r" % node)
@@ -62,19 +78,7 @@ class NodeManager(object):
         self.group.terminate(self.EXIT_TIMEOUT)
 
     def _getxspecs(self):
-        xspeclist = []
-        for xspec in self.config.getvalue("tx"):
-            i = xspec.find("*")
-            try:
-                num = int(xspec[:i])
-            except ValueError:
-                xspeclist.append(xspec)
-            else:
-                xspeclist.extend([xspec[i+1:]] * num)
-        if not xspeclist:
-            raise pytest.UsageError(
-                "MISSING test execution (tx) nodes: please specify --tx")
-        return [execnet.XSpec(x) for x in xspeclist]
+        return [execnet.XSpec(x) for x in parse_spec_config(self.config)]
 
     def _getrsyncdirs(self):
         for spec in self.specs:
@@ -182,7 +186,7 @@ class HostRSync(execnet.RSync):
 def make_reltoroot(roots, args):
     # XXX introduce/use public API for splitting py.test args
     splitcode = "::"
-    l = []
+    result = []
     for arg in args:
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
@@ -193,11 +197,11 @@ def make_reltoroot(roots, args):
                 break
         else:
             raise ValueError("arg %s not relative to an rsync root" % (arg,))
-        l.append(splitcode.join(parts))
-    return l
+        result.append(splitcode.join(parts))
+    return result
 
 
-class SlaveController(object):
+class WorkerController(object):
     ENDMARK = -1
 
     def __init__(self, nodemanager, gateway, config, putevent):
@@ -205,11 +209,16 @@ class SlaveController(object):
         self.putevent = putevent
         self.gateway = gateway
         self.config = config
-        self.slaveinput = {'slaveid': gateway.id,
-                           'slavecount': len(nodemanager.specs)}
+        self.workerinput = {'workerid': gateway.id,
+                            'workercount': len(nodemanager.specs),
+                            'slaveid': gateway.id,
+                            'slavecount': len(nodemanager.specs)
+                            }
+        # TODO: deprecated name, backward compatibility only. Remove it in future
+        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
-        self.log = py.log.Producer("slavectl-%s" % gateway.id)
+        self.log = py.log.Producer("workerctl-%s" % gateway.id)
         if not self.config.option.debug:
             py.log.setconsumer(self.log._keywords, None)
 
@@ -221,7 +230,7 @@ class SlaveController(object):
         return self._down or self._shutdown_sent
 
     def setup(self):
-        self.log("setting up slave session")
+        self.log("setting up worker session")
         spec = self.gateway.spec
         args = self.config.args
         if not spec.popen or spec.chdir:
@@ -234,7 +243,7 @@ class SlaveController(object):
                 option_dict['basetemp'] = str(basetemp.join(name))
         self.config.hook.pytest_configure_node(node=self)
         self.channel = self.gateway.remote_exec(xdist.remote)
-        self.channel.send((self.slaveinput, args, option_dict))
+        self.channel.send((self.workerinput, args, option_dict))
         if self.putevent:
             self.channel.setcallback(
                 self.process_from_remote,
@@ -292,15 +301,15 @@ class SlaveController(object):
                     self._down = True
                 return
             eventname, kwargs = eventcall
-            if eventname in ("collectionstart"):
+            if eventname in ("collectionstart",):
                 self.log("ignoring %s(%s)" % (eventname, kwargs))
-            elif eventname == "slaveready":
+            elif eventname == "workerready":
                 self.notify_inproc(eventname, node=self, **kwargs)
-            elif eventname == "slavefinished":
+            elif eventname == "workerfinished":
                 self._down = True
-                self.slaveoutput = kwargs['slaveoutput']
-                self.notify_inproc("slavefinished", node=self)
-            elif eventname == "logstart":
+                self.workeroutput = kwargs['workeroutput']
+                self.notify_inproc("workerfinished", node=self)
+            elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in (
                     "testreport", "collectreport", "teardownreport"):
@@ -311,19 +320,101 @@ class SlaveController(object):
                 self.notify_inproc(eventname, node=self, rep=rep)
             elif eventname == "collectionfinish":
                 self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
+            elif eventname == "runtest_protocol_complete":
+                self.notify_inproc(eventname, node=self, **kwargs)
+            elif eventname == "logwarning":
+                self.notify_inproc(eventname, message=kwargs['message'],
+                                   code=kwargs['code'], nodeid=kwargs['nodeid'],
+                                   fslocation=kwargs['nodeid'])
             else:
                 raise ValueError("unknown event: %s" % (eventname,))
         except KeyboardInterrupt:
             # should not land in receiver-thread
             raise
-        except:
+        except:  # noqa
             excinfo = py.code.ExceptionInfo()
             py.builtin.print_("!" * 20, excinfo)
-            self.config.pluginmanager.notify_exception(excinfo)
+            self.config.notify_exception(excinfo)
+            self.shutdown()
+            self.notify_inproc("errordown", node=self, error=excinfo)
 
 
 def unserialize_report(name, reportdict):
+    def assembled_report(reportdict):
+        from _pytest._code.code import (
+            ReprEntry,
+            ReprEntryNative,
+            ReprExceptionInfo,
+            ReprFileLocation,
+            ReprFuncArgs,
+            ReprLocals,
+            ReprTraceback
+        )
+        if reportdict['longrepr']:
+            if 'reprcrash' in reportdict['longrepr'] and 'reprtraceback' in reportdict['longrepr']:
+
+                reprtraceback = reportdict['longrepr']['reprtraceback']
+                reprcrash = reportdict['longrepr']['reprcrash']
+
+                unserialized_entries = []
+                reprentry = None
+                for entry_data in reprtraceback['reprentries']:
+                    data = entry_data['data']
+                    entry_type = entry_data['type']
+                    if entry_type == 'ReprEntry':
+                        reprfuncargs = None
+                        reprfileloc = None
+                        reprlocals = None
+                        if data['reprfuncargs']:
+                            reprfuncargs = ReprFuncArgs(
+                                **data['reprfuncargs'])
+                        if data['reprfileloc']:
+                            reprfileloc = ReprFileLocation(
+                                **data['reprfileloc'])
+                        if data['reprlocals']:
+                            reprlocals = ReprLocals(
+                                data['reprlocals']['lines'])
+
+                        reprentry = ReprEntry(
+                            lines=data['lines'],
+                            reprfuncargs=reprfuncargs,
+                            reprlocals=reprlocals,
+                            filelocrepr=reprfileloc,
+                            style=data['style']
+                        )
+                    elif entry_type == 'ReprEntryNative':
+                        reprentry = ReprEntryNative(data['lines'])
+                    else:
+                        report_unserialization_failure(
+                            entry_type, name, reportdict)
+                    unserialized_entries.append(reprentry)
+                reprtraceback['reprentries'] = unserialized_entries
+
+                exception_info = ReprExceptionInfo(
+                    reprtraceback=ReprTraceback(**reprtraceback),
+                    reprcrash=ReprFileLocation(**reprcrash),
+                )
+
+                for section in reportdict['longrepr']['sections']:
+                    exception_info.addsection(*section)
+                reportdict['longrepr'] = exception_info
+        return reportdict
+
     if name == "testreport":
-        return runner.TestReport(**reportdict)
+        return runner.TestReport(**assembled_report(reportdict))
     elif name == "collectreport":
-        return runner.CollectReport(**reportdict)
+        return runner.CollectReport(**assembled_report(reportdict))
+
+
+def report_unserialization_failure(type_name, report_name, reportdict):
+    from pprint import pprint
+    url = 'https://github.com/pytest-dev/pytest-xdist/issues'
+    stream = py.io.TextIO()
+    pprint('-' * 100, stream=stream)
+    pprint('INTERNALERROR: Unknown entry type returned: %s' % type_name,
+           stream=stream)
+    pprint('report_name: %s' % report_name, stream=stream)
+    pprint(reportdict, stream=stream)
+    pprint('Please report this bug at %s' % url, stream=stream)
+    pprint('-' * 100, stream=stream)
+    assert 0, stream.getvalue()