Prepare release 2.0.0

Co-authored-by: Bruno Oliveira <nicoddemus@gmail.com>

.appveyor.yml

@@ -1,6 +1,5 @@
 environment:
   matrix:
-  - TOXENV: "py27-pytestlatest"
   - TOXENV: "py35-pytestlatest"
   - TOXENV: "py36-pytestlatest"
   - TOXENV: "py37-pytestlatest"

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 19.10b0
     hooks:
     -   id: black
-        args: [--safe, --quiet]
+        args: [--safe, --quiet, --target-version, py35]
         language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.4.0
@@ -13,6 +13,11 @@ repos:
     -   id: check-yaml
     -   id: debug-statements
     -   id: flake8
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
+    hooks:
+    -   id: pyupgrade
+        args: [--py3-plus]
 -   repo: local
     hooks:
     -   id: rst

.travis.yml

@@ -32,8 +32,6 @@ jobs:
           - $HOME/.cache/pre-commit
     - python: '3.8'
       env: TOXENV=py38-pytestlatest
-    - python: '2.7'
-      env: TOXENV=py27-pytestlatest
 
     - stage: test
       python: "3.5"
@@ -42,6 +40,8 @@ jobs:
       env: TOXENV=py36-pytestlatest
     - python: "3.7"
       env: TOXENV=py37-pytestlatest
+    - python: "3.9-dev"
+      env: TOXENV=py39-pytestlatest
     - python: "3.8"
       env: TOXENV=py38-pytestmaster
 

CHANGELOG.rst

@@ -1,3 +1,68 @@
+pytest-xdist 2.0.0 (2020-08-12)
+===============================
+
+Deprecations and Removals
+-------------------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: Drop backward-compatibility "slave" aliases related to worker nodes.  We deliberately moved away from this terminology years ago, and it seems like the right time to finish the deprecation and removal process.
+
+- `#569 <https://github.com/pytest-dev/pytest-xdist/issues/569>`_: ``pytest-xdist`` no longer supports Python 2.7.
+
+
+Features
+--------
+
+- `#504 <https://github.com/pytest-dev/pytest-xdist/issues/504>`_: New functions ``xdist.is_xdist_worker``, ``xdist.is_xdist_master``, ``xdist.get_xdist_worker_id``, to easily identify the current node.
+
+
+Bug Fixes
+---------
+
+- `#471 <https://github.com/pytest-dev/pytest-xdist/issues/471>`_: Fix issue with Rsync reporting in quiet mode.
+
+- `#553 <https://github.com/pytest-dev/pytest-xdist/issues/553>`_: When using ``-n auto``, count the number of physical CPU cores instead of logical ones.
+
+
+Trivial Changes
+---------------
+
+- `#541 <https://github.com/pytest-dev/pytest-xdist/issues/541>`_: ``pytest-xdist`` now requires ``pytest>=6.0``.
+
+
+pytest-xdist 1.34.0 (2020-07-27)
+================================
+
+Features
+--------
+
+- `#549 <https://github.com/pytest-dev/pytest-xdist/issues/549>`_: Make ``--pdb`` imply ``--dist no``, as the two options cannot really work together at the moment.
+
+
+Bug Fixes
+---------
+
+- `#478 <https://github.com/pytest-dev/pytest-xdist/issues/478>`_: Fix regression with duplicated arguments via $PYTEST_ADDOPTS in 1.30.0.
+
+- `#558 <https://github.com/pytest-dev/pytest-xdist/issues/558>`_: Fix ``rsyncdirs`` usage with pytest 6.0.
+
+- `#562 <https://github.com/pytest-dev/pytest-xdist/issues/562>`_: Do not trigger the deprecated ``pytest_warning_captured`` in pytest 6.0+.
+
+
+pytest-xdist 1.33.0 (2020-07-09)
+================================
+
+Features
+--------
+
+- `#554 <https://github.com/pytest-dev/pytest-xdist/issues/554>`_: Fix warnings support for upcoming pytest 6.0.
+
+
+Trivial Changes
+---------------
+
+- `#548 <https://github.com/pytest-dev/pytest-xdist/issues/548>`_: SCM and CI files are no longer included in the source distribution.
+
+
 pytest-xdist 1.32.0 (2020-05-03)
 ================================
 

ISSUES.txt

@@ -1,31 +0,0 @@
-next release critical
------------------------------------------------
-tag: bug
-
-miserably fails: --dist=each --tx popen --tx socket=...
-
-
-rename / hooks
------------------------------------------------
-tag: bug
-
-node -> slave
-transition for hooks?
-configure_node -> configure_slave
-
-allow to remotely run xdist tests with xdist
------------------------------------------------
-tag: feature
-
-allow to run xdist own tests using its own mechanism.
-currently this doesn't work because the remote side
-has no pytest plugin.   How to configure/do
-register "xdist.plugin" on the remote side?
-
-see to avoid any "from _pytest" internal imports
------------------------------------------------
-tag: feature
-
-currently tests and even xdist core code imports
-names from the internal _pytest namespace.
-See to avoid it.

MANIFEST.in

@@ -0,0 +1,5 @@
+exclude .appveyor.yml
+exclude .gitignore
+exclude .pre-commit-config.yaml
+exclude .travis.yml
+prune .github

README.rst

@@ -67,33 +67,35 @@ a checkout of the `pytest-xdist repository`_ ::
 Speed up test runs by sending tests to multiple CPUs
 ----------------------------------------------------
 
-To send tests to multiple CPUs, type::
+To send tests to multiple CPUs, use the ``-n`` (or ``-numprocesses``) option::
 
-    pytest -n NUM
+    pytest -n NUMCPUS
 
-Especially for longer running tests or tests requiring
-a lot of I/O this can lead to considerable speed ups. This option can
-also be set to ``auto`` for automatic detection of the number of CPUs.
+Pass ``-n auto`` to use as many processes as your computer has CPU cores. This
+can lead to considerable speed ups, especially if your test suite takes a
+noticeable amount of time.
 
-If a test crashes the interpreter, pytest-xdist will automatically restart
-that worker and report the failure as usual. You can use the
-``--max-worker-restart`` option to limit the number of workers that can
-be restarted, or disable restarting altogether using ``--max-worker-restart=0``.
+If a test crashes a worker, pytest-xdist will automatically restart that worker
+and report the test’s failure. You can use the ``--max-worker-restart`` option
+to limit the number of worker restarts that are allowed, or disable restarting
+altogether using ``--max-worker-restart 0``.
 
-By default, the ``-n`` option will send pending tests to any worker that is available, without
-any guaranteed order, but you can control this with these options:
+By default, using ``--numprocesses`` will send pending tests to any worker that
+is available, without any guaranteed order. You can change the test
+distribution algorithm this with the ``--dist`` option. It takes these values:
 
-* ``--dist=loadscope``: tests will be grouped by **module** for *test functions* and
-  by **class** for *test methods*, then each group will be sent to an available worker,
-  guaranteeing that all tests in a group run in the same process. This can be useful if you have
-  expensive module-level or class-level fixtures. Currently the groupings can't be customized,
-  with grouping by class takes priority over grouping by module.
-  This feature was added in version ``1.19``.
+* ``--dist no``: The default algorithm, distributing one test at a time.
 
-* ``--dist=loadfile``: tests will be grouped by file name, and then will be sent to an available
-  worker, guaranteeing that all tests in a group run in the same worker. This feature was added
-  in version ``1.21``.
+* ``--dist loadscope``: Tests are grouped by **module** for *test functions*
+  and by **class** for *test methods*. Groups are distributed to available
+  workers as whole units. This guarantees that all tests in a group run in the
+  same process. This can be useful if you have expensive module-level or
+  class-level fixtures. Grouping by class takes priority over grouping by
+  module.
 
+* ``--dist loadfile``: Tests are grouped by their containing file. Groups are
+  distributed to available workers as whole units. This guarantees that all
+  tests in a file run in the same worker.
 
 Making session-scoped fixtures execute only once
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -266,7 +268,7 @@ a test or fixture, you may use the ``worker_id`` fixture to do so:
 When ``xdist`` is disabled (running with ``-n0`` for example), then
 ``worker_id`` will return ``"master"``.
 
-Additionally, worker processes have the following environment variables
+Worker processes also have the following environment variables
 defined:
 
 * ``PYTEST_XDIST_WORKER``: the name of the worker, e.g., ``"gw2"``.
@@ -276,6 +278,34 @@ defined:
 The information about the worker_id in a test is stored in the ``TestReport`` as
 well, under the ``worker_id`` attribute.
 
+Since version 2.0, the following functions are also available in the ``xdist`` module:
+
+.. code-block:: python
+
+    def is_xdist_worker(request_or_session) -> bool:
+        """Return `True` if this is an xdist worker, `False` otherwise
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def is_xdist_master(request_or_session) -> bool:
+        """Return `True` if this is the xdist master, `False` otherwise
+
+        Note: this method also returns `False` when distribution has not been
+        activated at all.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
+    def get_xdist_worker_id(request_or_session) -> str:
+        """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+        if running on the 'master' node.
+
+        If not distributing tests (for example passing `-n0` or not passing `-n` at all) also return 'master'.
+
+        :param request_or_session: the `pytest` `request` or `session` object
+        """
+
 
 Uniquely identifying the current test run
 -----------------------------------------

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-install_requires = ["execnet>=1.1", "pytest>=4.4.0", "pytest-forked", "six"]
+install_requires = ["execnet>=1.1", "psutil>=3.0.0", "pytest>=6.0.0", "pytest-forked"]
 
 
 with open("README.rst") as f:
@@ -9,8 +9,7 @@ with open("README.rst") as f:
 setup(
     name="pytest-xdist",
     use_scm_version={"write_to": "src/xdist/_version.py"},
-    description="pytest xdist plugin for distributed testing"
-    " and loop-on-failing modes",
+    description="pytest xdist plugin for distributed testing and loop-on-failing modes",
     long_description=long_description,
     license="MIT",
     author="holger krekel and contributors",
@@ -24,7 +23,7 @@ setup(
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },
     zip_safe=False,
-    python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*",
+    python_requires=">=3.5",
     install_requires=install_requires,
     setup_requires=["setuptools_scm"],
     classifiers=[
@@ -39,9 +38,8 @@ setup(
         "Topic :: Software Development :: Quality Assurance",
         "Topic :: Utilities",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",

src/xdist/__init__.py

@@ -1,3 +1,4 @@
+from xdist.plugin import is_xdist_worker, is_xdist_master, get_xdist_worker_id
 from xdist._version import version as __version__
 
-__all__ = ["__version__"]
+__all__ = ["__version__", "is_xdist_worker", "is_xdist_master", "get_xdist_worker_id"]

src/xdist/dsession.py

@@ -10,14 +10,14 @@ from xdist.scheduler import (
 )
 
 
-from six.moves.queue import Empty, Queue
+from queue import Empty, Queue
 
 
 class Interrupted(KeyboardInterrupt):
     """ signals an immediate interruption. """
 
 
-class DSession(object):
+class DSession:
     """A pytest plugin which runs a distributed test session
 
     At the beginning of the test session this creates a NodeManager
@@ -150,9 +150,6 @@ class DSession(object):
         node.workerinfo["id"] = node.gateway.id
         node.workerinfo["spec"] = node.gateway.spec
 
-        # TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
-        node.slaveinfo = node.workerinfo
-
         self.config.hook.pytest_testnodeready(node=node)
         if self.shuttingdown:
             node.shutdown()
@@ -271,16 +268,18 @@ class DSession(object):
         assert not rep.passed
         self._failed_worker_collectreport(node, rep)
 
-    def worker_logwarning(self, message, code, nodeid, fslocation):
-        """Emitted when a node calls the pytest_logwarning hook."""
-        kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
-        self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
-
     def worker_warning_captured(self, warning_message, when, item):
-        """Emitted when a node calls the pytest_logwarning hook."""
+        """Emitted when a node calls the pytest_warning_captured hook (deprecated in 6.0)."""
         kwargs = dict(warning_message=warning_message, when=when, item=item)
         self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
 
+    def worker_warning_recorded(self, warning_message, when, nodeid, location):
+        """Emitted when a node calls the pytest_warning_recorded hook."""
+        kwargs = dict(
+            warning_message=warning_message, when=when, nodeid=nodeid, location=location
+        )
+        self.config.hook.pytest_warning_recorded.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 
@@ -329,7 +328,7 @@ class DSession(object):
         self.config.hook.pytest_runtest_logreport(report=rep)
 
 
-class TerminalDistReporter(object):
+class TerminalDistReporter:
     def __init__(self, config):
         self.config = config
         self.tr = config.pluginmanager.getplugin("terminalreporter")

src/xdist/looponfail.py

@@ -6,7 +6,6 @@
     processes) otherwise changes to source code can crash
     the controlling process which should best never happen.
 """
-from __future__ import print_function
 import py
 import pytest
 import sys
@@ -55,7 +54,7 @@ def looponfail_main(config):
         print()
 
 
-class RemoteControl(object):
+class RemoteControl:
     def __init__(self, config):
         self.config = config
         self.failures = []
@@ -165,7 +164,7 @@ def init_worker_session(channel, args, option_dict):
     WorkerFailSession(config, channel).main()
 
 
-class WorkerFailSession(object):
+class WorkerFailSession:
     def __init__(self, config, channel):
         self.config = config
         self.channel = channel
@@ -220,7 +219,7 @@ class WorkerFailSession(object):
         self.channel.send((trails, failreports, self.collection_failed))
 
 
-class StatRecorder(object):
+class StatRecorder:
     def __init__(self, rootdirlist):
         self.rootdirlist = rootdirlist
         self.statcache = {}

src/xdist/plugin.py

@@ -1,31 +1,12 @@
-import os
 import uuid
 
+import psutil
 import py
 import pytest
 
 
 def auto_detect_cpus():
-    try:
-        from os import sched_getaffinity
-    except ImportError:
-        if os.environ.get("TRAVIS") == "true":
-            # workaround https://bitbucket.org/pypy/pypy/issues/2375
-            return 2
-        try:
-            from os import cpu_count
-        except ImportError:
-            from multiprocessing import cpu_count
-    else:
-
-        def cpu_count():
-            return len(sched_getaffinity(0))
-
-    try:
-        n = cpu_count()
-    except NotImplementedError:
-        return 1
-    return n if n else 1
+    return psutil.cpu_count(logical=False) or psutil.cpu_count() or 1
 
 
 class AutoInt(int):
@@ -62,14 +43,11 @@ def pytest_addoption(parser):
     )
     group.addoption(
         "--max-worker-restart",
-        "--max-slave-restart",
         action="store",
         default=None,
         dest="maxworkerrestart",
         help="maximum number of workers that can be restarted "
-        "when crashed (set to zero to disable this feature)\n"
-        "'--max-slave-restart' option is deprecated and will be removed in "
-        "a future release",
+        "when crashed (set to zero to disable this feature)",
     )
     group.addoption(
         "--dist",
@@ -191,7 +169,11 @@ def pytest_configure(config):
 def pytest_cmdline_main(config):
     usepdb = config.getoption("usepdb", False)  # a core option
     if isinstance(config.option.numprocesses, AutoInt):
-        config.option.numprocesses = 0 if usepdb else int(config.option.numprocesses)
+        if usepdb:
+            config.option.numprocesses = 0
+            config.option.dist = "no"
+        else:
+            config.option.numprocesses = int(config.option.numprocesses)
 
     if config.option.numprocesses:
         if config.option.dist == "no":
@@ -203,28 +185,60 @@ def pytest_cmdline_main(config):
     if config.option.distload:
         config.option.dist = "load"
     val = config.getvalue
-    if not val("collectonly"):
-        if val("dist") != "no":
-            if usepdb:
-                raise pytest.UsageError(
-                    "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
-                )  # noqa: E501
+    if not val("collectonly") and val("dist") != "no" and usepdb:
+        raise pytest.UsageError(
+            "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
+        )  # noqa: E501
 
 
 # -------------------------------------------------------------------------
-# fixtures
+# fixtures and API to easily know the role of current node
 # -------------------------------------------------------------------------
 
 
+def is_xdist_worker(request_or_session) -> bool:
+    """Return `True` if this is an xdist worker, `False` otherwise
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return hasattr(request_or_session.config, "workerinput")
+
+
+def is_xdist_master(request_or_session) -> bool:
+    """Return `True` if this is the xdist master, `False` otherwise
+
+    Note: this method also returns `False` when distribution has not been
+    activated at all.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    return (
+        not is_xdist_worker(request_or_session)
+        and request_or_session.config.option.dist != "no"
+    )
+
+
+def get_xdist_worker_id(request_or_session) -> str:
+    """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
+    if running on the 'master' node.
+
+    If not distributing tests (for example passing `-n0` or not passing `-n` at all)
+    also return 'master'.
+
+    :param request_or_session: the `pytest` `request` or `session` object
+    """
+    if hasattr(request_or_session.config, "workerinput"):
+        return request_or_session.config.workerinput["workerid"]
+    else:
+        return "master"
+
+
 @pytest.fixture(scope="session")
 def worker_id(request):
     """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
     if running on the master node.
     """
-    if hasattr(request.config, "workerinput"):
-        return request.config.workerinput["workerid"]
-    else:
-        return "master"
+    return get_xdist_worker_id(request)
 
 
 @pytest.fixture(scope="session")

src/xdist/remote.py

@@ -11,14 +11,13 @@ import os
 import time
 
 import py
-import _pytest.hookspec
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
 from _pytest.config import _prepareconfig, Config
 
 
-class WorkerInteractor(object):
+class WorkerInteractor:
     def __init__(self, config, channel):
         self.config = config
         self.workerid = config.workerinput.get("workerid", "?")
@@ -101,11 +100,8 @@ class WorkerInteractor(object):
     def pytest_runtest_logstart(self, nodeid, location):
         self.sendevent("logstart", nodeid=nodeid, location=location)
 
-    # the pytest_runtest_logfinish hook was introduced in pytest 3.4
-    if hasattr(_pytest.hookspec, "pytest_runtest_logfinish"):
-
-        def pytest_runtest_logfinish(self, nodeid, location):
-            self.sendevent("logfinish", nodeid=nodeid, location=location)
+    def pytest_runtest_logfinish(self, nodeid, location):
+        self.sendevent("logfinish", nodeid=nodeid, location=location)
 
     def pytest_runtest_logreport(self, report):
         data = self.config.hook.pytest_report_to_serializable(
@@ -125,31 +121,14 @@ class WorkerInteractor(object):
             )
             self.sendevent("collectreport", data=data)
 
-    # the pytest_logwarning hook was deprecated since pytest 4.0
-    if hasattr(
-        _pytest.hookspec, "pytest_logwarning"
-    ) and not _pytest.hookspec.pytest_logwarning.pytest_spec.get("warn_on_impl"):
-
-        def pytest_logwarning(self, message, code, nodeid, fslocation):
-            self.sendevent(
-                "logwarning",
-                message=message,
-                code=code,
-                nodeid=nodeid,
-                fslocation=str(fslocation),
-            )
-
-    # the pytest_warning_captured hook was introduced in pytest 3.8
-    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
-
-        def pytest_warning_captured(self, warning_message, when, item):
-            self.sendevent(
-                "warning_captured",
-                warning_message_data=serialize_warning_message(warning_message),
-                when=when,
-                # item cannot be serialized and will always be None when used with xdist
-                item=None,
-            )
+    def pytest_warning_recorded(self, warning_message, when, nodeid, location):
+        self.sendevent(
+            "warning_recorded",
+            warning_message_data=serialize_warning_message(warning_message),
+            when=when,
+            nodeid=nodeid,
+            location=location,
+        )
 
 
 def serialize_warning_message(warning_message):
@@ -254,8 +233,5 @@ if __name__ == "__channelexec__":
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
     config.workerinput = workerinput
     config.workeroutput = {}
-    # TODO: deprecated name, backward compatibility only. Remove it in future
-    config.slaveinput = config.workerinput
-    config.slaveoutput = config.workeroutput
     interactor = WorkerInteractor(config, channel)
     config.hook.pytest_cmdline_main(config=config)

src/xdist/report.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 from difflib import unified_diff
 
 
@@ -13,9 +12,9 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
 
     diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
     error_message = (
-        u"Different tests were collected between {from_id} and {to_id}. "
-        u"The difference is:\n"
-        u"{diff}"
+        "Different tests were collected between {from_id} and {to_id}. "
+        "The difference is:\n"
+        "{diff}"
     ).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
     msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
     return msg

src/xdist/scheduler/each.py

@@ -4,7 +4,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class EachScheduling(object):
+class EachScheduling:
     """Implement scheduling of test items on all nodes
 
     If a node gets added after the test run is started then it is

src/xdist/scheduler/load.py

@@ -7,7 +7,7 @@ from xdist.workermanage import parse_spec_config
 from xdist.report import report_collection_diff
 
 
-class LoadScheduling(object):
+class LoadScheduling:
     """Implement load scheduling across nodes.
 
     This distributes the tests collected across all nodes so each test

src/xdist/scheduler/loadfile.py

@@ -21,7 +21,7 @@ class LoadFileScheduling(LoadScopeScheduling):
     """
 
     def __init__(self, config, log=None):
-        super(LoadFileScheduling, self).__init__(config, log)
+        super().__init__(config, log)
         if log is None:
             self.log = Producer("loadfilesched")
         else:

src/xdist/scheduler/loadscope.py

@@ -6,7 +6,7 @@ from xdist.report import report_collection_diff
 from xdist.workermanage import parse_spec_config
 
 
-class LoadScopeScheduling(object):
+class LoadScopeScheduling:
     """Implement load scheduling across nodes, but grouping test by scope.
 
     This distributes the tests collected across all nodes so each test is run

src/xdist/workermanage.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import fnmatch
 import os
 import re
@@ -29,7 +28,7 @@ def parse_spec_config(config):
     return xspeclist
 
 
-class NodeManager(object):
+class NodeManager:
     EXIT_TIMEOUT = 10
     DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
 
@@ -63,10 +62,7 @@ class NodeManager(object):
     def setup_nodes(self, putevent):
         self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
         self.trace("setting up nodes")
-        nodes = []
-        for spec in self.specs:
-            nodes.append(self.setup_node(spec, putevent))
-        return nodes
+        return [self.setup_node(spec, putevent) for spec in self.specs]
 
     def setup_node(self, spec, putevent):
         gw = self.group.makegateway(spec)
@@ -93,8 +89,16 @@ class NodeManager(object):
         import pytest
         import _pytest
 
-        pytestpath = pytest.__file__.rstrip("co")
-        pytestdir = py.path.local(_pytest.__file__).dirpath()
+        def get_dir(p):
+            """Return the directory path if p is a package or the path to the .py file otherwise."""
+            stripped = p.rstrip("co")
+            if os.path.basename(stripped) == "__init__.py":
+                return os.path.dirname(p)
+            else:
+                return stripped
+
+        pytestpath = get_dir(pytest.__file__)
+        pytestdir = get_dir(_pytest.__file__)
         config = self.config
         candidates = [py._pydir, pytestpath, pytestdir]
         candidates += config.option.rsyncdir
@@ -118,7 +122,7 @@ class NodeManager(object):
 
         return {
             "ignores": ignores,
-            "verbose": getattr(self.config.option, "verbose", False),
+            "verbose": getattr(self.config.option, "verbose", 0),
         }
 
     def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
@@ -158,12 +162,11 @@ class HostRSync(execnet.RSync):
 
     def __init__(self, sourcedir, *args, **kwargs):
         self._synced = {}
-        self._ignores = []
         ignores = kwargs.pop("ignores", None) or []
-        for x in ignores:
-            x = getattr(x, "strpath", x)
-            self._ignores.append(re.compile(fnmatch.translate(x)))
-        super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
+        self._ignores = [
+            re.compile(fnmatch.translate(getattr(x, "strpath", x))) for x in ignores
+        ]
+        super().__init__(sourcedir=sourcedir, **kwargs)
 
     def filter(self, path):
         path = py.path.local(path)
@@ -175,12 +178,10 @@ class HostRSync(execnet.RSync):
 
     def add_target_host(self, gateway, finished=None):
         remotepath = os.path.basename(self._sourcedir)
-        super(HostRSync, self).add_target(
-            gateway, remotepath, finishedcallback=finished, delete=True
-        )
+        super().add_target(gateway, remotepath, finishedcallback=finished, delete=True)
 
     def _report_send_file(self, gateway, modified_rel_path):
-        if self._verbose:
+        if self._verbose > 0:
             path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
             remotepath = gateway.spec.chdir
             print("{}:{} <= {}".format(gateway.spec, remotepath, path))
@@ -207,7 +208,7 @@ def make_reltoroot(roots, args):
     return result
 
 
-class WorkerController(object):
+class WorkerController:
     ENDMARK = -1
 
     class RemoteHook:
@@ -224,13 +225,9 @@ class WorkerController(object):
         self.workerinput = {
             "workerid": gateway.id,
             "workercount": len(nodemanager.specs),
-            "slaveid": gateway.id,
-            "slavecount": len(nodemanager.specs),
             "testrunuid": nodemanager.testrunuid,
             "mainargv": sys.argv,
         }
-        # TODO: deprecated name, backward compatibility only. Remove it in future
-        self.slaveinput = self.workerinput
         self._down = False
         self._shutdown_sent = False
         self.log = py.log.Producer("workerctl-%s" % gateway.id)
@@ -292,7 +289,7 @@ class WorkerController(object):
         if not self._down:
             try:
                 self.sendcommand("shutdown")
-            except (IOError, OSError):
+            except OSError:
                 pass
             self._shutdown_sent = True
 
@@ -329,7 +326,7 @@ class WorkerController(object):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname == "workerfinished":
                 self._down = True
-                self.workeroutput = self.slaveoutput = kwargs["workeroutput"]
+                self.workeroutput = kwargs["workeroutput"]
                 self.notify_inproc("workerfinished", node=self)
             elif eventname in ("logstart", "logfinish"):
                 self.notify_inproc(eventname, node=self, **kwargs)
@@ -363,6 +360,17 @@ class WorkerController(object):
                     when=kwargs["when"],
                     item=kwargs["item"],
                 )
+            elif eventname == "warning_recorded":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    nodeid=kwargs["nodeid"],
+                    location=kwargs["location"],
+                )
             else:
                 raise ValueError("unknown event: {}".format(eventname))
         except KeyboardInterrupt:
@@ -371,11 +379,7 @@ class WorkerController(object):
         except:  # noqa
             from _pytest._code import ExceptionInfo
 
-            # ExceptionInfo API changed in pytest 4.1
-            if hasattr(ExceptionInfo, "from_current"):
-                excinfo = ExceptionInfo.from_current()
-            else:
-                excinfo = ExceptionInfo()
+            excinfo = ExceptionInfo.from_current()
             print("!" * 20, excinfo)
             self.config.notify_exception(excinfo)
             self.shutdown()

testing/acceptance_test.py

@@ -1,10 +1,9 @@
 import os
 import re
-import sys
-import textwrap
 
 import py
 import pytest
+import xdist
 
 
 class TestDistribution:
@@ -115,9 +114,6 @@ class TestDistribution:
         assert result.ret == 2
         result.stdout.fnmatch_lines(["*Interrupted: stopping*1*", "*1 failed*"])
 
-    @pytest.mark.xfail(
-        reason="#527: Python 3.8 failure in pytest where testdir.tmpdir returns an unexpected value"
-    )
     def test_basetemp_in_subprocesses(self, testdir):
         p1 = testdir.makepyfile(
             """
@@ -189,7 +185,6 @@ class TestDistribution:
         )
         assert result.ret == 1
 
-    @pytest.mark.xfail(reason="#527: Ignore Python 3.8 failure for the time being")
     def test_distribution_rsyncdirs_example(self, testdir, monkeypatch):
         # use a custom plugin that has a custom command-line option to ensure
         # this is propagated to workers (see #491)
@@ -246,22 +241,6 @@ class TestDistribution:
         result.stderr.fnmatch_lines(["--foobar=123 active! *"])
         assert dest.join(subdir.basename).check(dir=1)
 
-    def test_backward_compatibility_worker_terminology(self, testdir):
-        """Ensure that we still support "config.slaveinput" for backward compatibility (#234).
-
-        Keep in mind that removing this compatibility will break a ton of plugins and user code.
-        """
-        testdir.makepyfile(
-            """
-            def test(pytestconfig):
-                assert hasattr(pytestconfig, 'slaveinput')
-                assert hasattr(pytestconfig, 'workerinput')
-        """
-        )
-        result = testdir.runpytest("-n1")
-        result.stdout.fnmatch_lines("*1 passed*")
-        assert result.ret == 0
-
     def test_data_exchange(self, testdir):
         testdir.makeconftest(
             """
@@ -332,7 +311,7 @@ class TestDistribution:
                 time.sleep(10)
         """
         )
-        child = testdir.spawn_pytest("-n1 -v")
+        child = testdir.spawn_pytest("-n1 -v", expect_timeout=30.0)
         child.expect(".*test_sleep.*")
         child.kill(2)  # keyboard interrupt
         child.expect(".*KeyboardInterrupt.*")
@@ -594,16 +573,13 @@ def test_fixture_teardown_failure(testdir):
     assert result.ret
 
 
-def test_config_initialization(testdir, pytestconfig):
+def test_config_initialization(testdir, monkeypatch, pytestconfig):
     """Ensure workers and master are initialized consistently. Integration test for #445"""
-    if not hasattr(pytestconfig, "invocation_params"):
-        pytest.skip(
-            "requires pytest >=5.1 (config has no attribute 'invocation_params')"
-        )
     testdir.makepyfile(
         **{
             "dir_a/test_foo.py": """
-                def test_1(): pass
+                def test_1(request):
+                    assert request.config.option.verbose == 2
         """
         }
     )
@@ -614,8 +590,10 @@ def test_config_initialization(testdir, pytestconfig):
         testpaths=dir_a
     """,
     )
+    monkeypatch.setenv("PYTEST_ADDOPTS", "-v")
     result = testdir.runpytest("-n2", "-c", "myconfig.ini", "-v")
-    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*"])
+    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*", "*= 1 passed in *"])
+    assert result.ret == 0
 
 
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
@@ -693,32 +671,6 @@ def test_skipping(testdir):
     result.stdout.fnmatch_lines(["*hello*", "*1 skipped*"])
 
 
-def test_issue34_pluginloading_in_subprocess(testdir):
-    import _pytest.hookspec
-
-    if not hasattr(_pytest.hookspec, "pytest_namespace"):
-        pytest.skip("this pytest version no longer supports pytest_namespace()")
-
-    testdir.tmpdir.join("plugin123.py").write(
-        textwrap.dedent(
-            """
-        def pytest_namespace():
-            return {'sample_variable': 'testing'}
-    """
-        )
-    )
-    testdir.makepyfile(
-        """
-        import pytest
-        def test_hello():
-            assert pytest.sample_variable == "testing"
-    """
-    )
-    result = testdir.runpytest_subprocess("-n1", "-p", "plugin123")
-    assert result.ret == 0
-    result.stdout.fnmatch_lines(["*1 passed*"])
-
-
 def test_fixture_scope_caching_issue503(testdir):
     p1 = testdir.makepyfile(
         """
@@ -796,31 +748,45 @@ def test_sub_plugins_disabled(testdir, plugin):
 
 class TestWarnings:
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
-    @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
-    def test_warnings(self, testdir, n, request, warn_type):
-        if warn_type == "builtin":
-            warn_code = """warnings.warn(UserWarning('this is a warning'))"""
-        elif warn_type == "pytest":
-            if not hasattr(request.config, "warn"):
-                pytest.skip("config.warn has been removed in pytest 4.1")
-            warn_code = """request.config.warn('', 'this is a warning',
-                           fslocation=py.path.local())"""
-        else:
-            assert False
+    def test_warnings(self, testdir, n):
         testdir.makepyfile(
             """
             import warnings, py, pytest
 
             @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
             def test_func(request):
-                {warn_code}
-        """.format(
-                warn_code=warn_code
-            )
+                warnings.warn(UserWarning('this is a warning'))
+            """
         )
         result = testdir.runpytest(n)
         result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warning*"])
 
+    def test_warning_captured_deprecated_in_pytest_6(self, testdir):
+        """
+        Do not trigger the deprecated pytest_warning_captured hook in pytest 6+ (#562)
+        """
+        import _pytest.hookspec
+
+        if not hasattr(_pytest.hookspec, "pytest_warning_recorded"):
+            pytest.skip("test requires pytest 6.0+")
+
+        testdir.makeconftest(
+            """
+            def pytest_warning_captured():
+                assert False, "this hook should not be called in this version"
+        """
+        )
+        testdir.makepyfile(
+            """
+            import warnings
+            def test():
+                warnings.warn("custom warning")
+        """
+        )
+        result = testdir.runpytest("-n1")
+        result.stdout.fnmatch_lines(["* 1 passed in *"])
+        result.stdout.no_fnmatch_line("*this hook should not be called in this version")
+
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     def test_custom_subclass(self, testdir, n):
         """Check that warning subclasses that don't honor the args attribute don't break
@@ -867,9 +833,6 @@ class TestWarnings:
         """Check that warnings with unserializable _WARNING_DETAILS are
         handled correctly (#379).
         """
-        if sys.version_info[0] < 3:
-            # The issue is only present in Python 3 warnings
-            return
         testdir.makepyfile(
             """
             import warnings, pytest
@@ -1385,7 +1348,7 @@ def parse_tests_and_workers_from_output(lines):
             r"""
             \[(gw\d)\]  # worker
             \s*
-            (?:\[\s*\d+%\])? # progress indicator (pytest >=3.3)
+            (?:\[\s*\d+%\])? # progress indicator
             \s(.*?)     # status string ("PASSED")
             \s(.*::.*)  # nodeid
         """,
@@ -1404,3 +1367,39 @@ def get_workers_and_test_count_by_prefix(prefix, lines, expected_status="PASSED"
         if expected_status == status and nodeid.startswith(prefix):
             result[worker] = result.get(worker, 0) + 1
     return result
+
+
+class TestAPI:
+    @pytest.fixture
+    def fake_request(self):
+        class FakeOption:
+            def __init__(self):
+                self.dist = "load"
+
+        class FakeConfig:
+            def __init__(self):
+                self.workerinput = {"workerid": "gw5"}
+                self.option = FakeOption()
+
+        class FakeRequest:
+            def __init__(self):
+                self.config = FakeConfig()
+
+        return FakeRequest()
+
+    def test_is_xdist_worker(self, fake_request):
+        assert xdist.is_xdist_worker(fake_request)
+        del fake_request.config.workerinput
+        assert not xdist.is_xdist_worker(fake_request)
+
+    def test_is_xdist_master(self, fake_request):
+        assert not xdist.is_xdist_master(fake_request)
+        del fake_request.config.workerinput
+        assert xdist.is_xdist_master(fake_request)
+        fake_request.config.option.dist = "no"
+        assert not xdist.is_xdist_master(fake_request)
+
+    def test_get_xdist_worker_id(self, fake_request):
+        assert xdist.get_xdist_worker_id(fake_request) == "gw5"
+        del fake_request.config.workerinput
+        assert xdist.get_xdist_worker_id(fake_request) == "master"

testing/conftest.py

@@ -1,20 +1,9 @@
-import six
 import py
 import pytest
 import execnet
 
 pytest_plugins = "pytester"
 
-if six.PY2:
-
-    @pytest.fixture(scope="session", autouse=True)
-    def _ensure_imports():
-        # we import some modules because pytest-2.8's testdir fixture
-        # will unload all modules after each test and this cause
-        # (unknown) problems with execnet.Group()
-        execnet.Group
-        execnet.makegateway
-
 
 @pytest.fixture(autouse=True)
 def _divert_atexit(request, monkeypatch):
@@ -22,12 +11,16 @@ def _divert_atexit(request, monkeypatch):
 
     finalizers = []
 
-    def finish():
-        while finalizers:
-            finalizers.pop()()
+    def fake_register(func, *args, **kwargs):
+        finalizers.append((func, args, kwargs))
 
-    monkeypatch.setattr(atexit, "register", finalizers.append)
-    request.addfinalizer(finish)
+    monkeypatch.setattr(atexit, "register", fake_register)
+
+    yield
+
+    while finalizers:
+        func, args, kwargs = finalizers.pop()
+        func(*args, **kwargs)
 
 
 def pytest_addoption(parser):

testing/test_dsession.py

@@ -203,7 +203,7 @@ class TestLoadScheduling:
         different test ids are collected by workers.
         """
 
-        class CollectHook(object):
+        class CollectHook:
             """
             Dummy hook that stores collection reports.
             """

testing/test_looponfail.py

@@ -284,7 +284,7 @@ class TestFunctional:
         )
         # p = testdir.mkdir("sub").join(p1.basename)
         # p1.move(p)
-        child = testdir.spawn_pytest("-f %s --traceconfig" % p)
+        child = testdir.spawn_pytest("-f %s --traceconfig" % p, expect_timeout=30.0)
         child.expect("def test_one")
         child.expect("x == 1")
         child.expect("1 failed")
@@ -311,7 +311,7 @@ class TestFunctional:
                 pass
         """
         )
-        child = testdir.spawn_pytest("-f %s" % p)
+        child = testdir.spawn_pytest("-f %s" % p, expect_timeout=30.0)
         child.expect("1 xpass")
         # child.expect("### LOOPONFAILING ####")
         child.expect("waiting for changes")

testing/test_plugin.py

@@ -35,17 +35,10 @@ def test_dist_options(testdir):
 
 
 def test_auto_detect_cpus(testdir, monkeypatch):
-    import os
+    import psutil
     from xdist.plugin import pytest_cmdline_main as check_options
 
-    if hasattr(os, "sched_getaffinity"):
-        monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
-    elif hasattr(os, "cpu_count"):
-        monkeypatch.setattr(os, "cpu_count", lambda: 99)
-    else:
-        import multiprocessing
-
-        monkeypatch.setattr(multiprocessing, "cpu_count", lambda: 99)
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: 99)
 
     config = testdir.parseconfigure("-n2")
     assert config.getoption("numprocesses") == 2
@@ -57,11 +50,11 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     check_options(config)
     assert config.getoption("usepdb")
     assert config.getoption("numprocesses") == 0
+    assert config.getoption("dist") == "no"
 
-    monkeypatch.delattr(os, "sched_getaffinity", raising=False)
-    monkeypatch.setenv("TRAVIS", "true")
+    monkeypatch.setattr(psutil, "cpu_count", lambda logical=True: None)
     config = testdir.parseconfigure("-nauto")
-    assert config.getoption("numprocesses") == 2
+    assert config.getoption("numprocesses") == 1
 
 
 def test_boxed_with_collect_only(testdir):

testing/test_remote.py

@@ -8,7 +8,7 @@ from xdist.workermanage import WorkerController
 import execnet
 import marshal
 
-from six.moves.queue import Queue
+from queue import Queue
 
 WAIT_TIMEOUT = 10.0
 

testing/test_workermanage.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import py
 import pytest
 import textwrap
@@ -35,7 +34,7 @@ def mysetup(tmpdir):
 
 @pytest.fixture
 def workercontroller(monkeypatch):
-    class MockController(object):
+    class MockController:
         def __init__(self, *args):
             pass
 
@@ -197,6 +196,25 @@ class TestNodeManager:
             assert dest.join("dir1", "dir2", "hello").check()
             nodemanager.teardown_nodes()
 
+    @pytest.mark.parametrize(
+        "flag, expects_report", [("-q", False), ("", False), ("-v", True)]
+    )
+    def test_rsync_report(
+        self, testdir, mysetup, workercontroller, capsys, flag, expects_report
+    ):
+        source, dest = mysetup.source, mysetup.dest
+        dir1 = mysetup.source.mkdir("dir1")
+        args = "--tx", "popen//chdir=%s" % dest, "--rsyncdir", dir1, source
+        if flag:
+            args += (flag,)
+        nodemanager = NodeManager(testdir.parseconfig(*args))
+        nodemanager.setup_nodes(None)  # calls .rsync_roots()
+        out, _ = capsys.readouterr()
+        if expects_report:
+            assert "<= pytest/__init__.py" in out
+        else:
+            assert "<= pytest/__init__.py" not in out
+
     def test_init_rsync_roots(self, testdir, mysetup, workercontroller):
         source, dest = mysetup.source, mysetup.dest
         dir2 = source.ensure("dir1", "dir2", dir=1)

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 envlist=
   linting
-  py{27,35,36,37,38}-pytestlatest
+  py{35,36,37,38,39}-pytestlatest
   py38-pytestmaster
 
 [testenv]