prepare 1.30.0 release

Fix issues related to running xdist with the terminal plugin disabled

.gitignore

@@ -23,7 +23,7 @@ include/
 lib/
 bin/
 env/
-xdist/_version.py*
+src/xdist/_version.py*
 pytest_xdist.egg-info
 issue/
 3rdparty/

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
     -   id: black
         args: [--safe, --quiet]
-        language_version: python3.6
+        language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v1.3.0
     hooks:
@@ -21,4 +21,4 @@ repos:
         files: ^(CHANGELOG.rst|HOWTORELEASE.rst|README.rst|changelog/.*)$
         language: python
         additional_dependencies: [pygments, restructuredtext_lint]
-        python_version: python3.6
+        python_version: python3.7

.travis.yml

@@ -1,10 +1,10 @@
-sudo: false
+dist: xenial
 language: python
 
 notifications:
   irc:
     channels:
-    - 'chat.freenode.net#pytest'
+      - 'chat.freenode.net#pytest'
     on_success: change
     on_failure: change
     skip_join: true
@@ -12,23 +12,26 @@ notifications:
   - pytest-commit@python.org
 
 install:
-  - pip install -U pip
+  - pip install -U pip setuptools
   - pip install tox setuptools_scm
 script: tox
 
 stages:
-- baseline
-- test
-- name: deploy
-  if: repo = pytest-dev/pytest-xdist AND tag IS present
+  - baseline
+  - test
+  - name: deploy
+    if: repo = pytest-dev/pytest-xdist AND tag IS present
 
 jobs:
   include:
     - stage: baseline
-      python: '3.6'
+      python: '3.7'
       env: TOXENV=linting
-    - python: '3.6'
-      env: TOXENV=py36-pytestlatest
+      cache:
+        directories:
+          - $HOME/.cache/pre-commit
+    - python: '3.7'
+      env: TOXENV=py37-pytestlatest
     - python: '2.7'
       env: TOXENV=py27-pytestlatest
 
@@ -37,21 +40,15 @@ jobs:
       env: TOXENV=py34-pytestlatest
     - python: "3.5"
       env: TOXENV=py35-pytestlatest
+    - python: "3.6"
+      env: TOXENV=py36-pytestlatest
     - python: "3.7"
-      env: TOXENV=py37-pytestlatest
-      sudo: required
-      dist: xenial
-    - python: "2.7"
-      env: TOXENV=py27-pytestmaster
-    - python: "2.7"
-      env: TOXENV=py27-pytestfeatures
-    - python: "3.6"
-      env: TOXENV=py36-pytestmaster
-    - python: "3.6"
-      env: TOXENV=py36-pytestfeatures
+      env: TOXENV=py37-pytestmaster
+    - python: "3.7"
+      env: TOXENV=py37-pytestfeatures
 
     - stage: deploy
-      python: '3.6'
+      python: '3.7'
       env:
       install: pip install -U setuptools setuptools_scm
       script: skip

CHANGELOG.rst

@@ -1,3 +1,48 @@
+pytest-xdist 1.29.0 (2019-06-14)
+================================
+
+Features
+--------
+
+- `#226 <https://github.com/pytest-dev/pytest-xdist/issues/226>`_: ``--max-worker-restart`` now assumes a more reasonable value (4 times the number of
+  nodes) when not given explicitly. This prevents test suites from running forever when the suite crashes during collection.
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: When the test session is interrupted due to running out of workers, the reason is shown in the test summary
+  for easier viewing.
+
+- `#442 <https://github.com/pytest-dev/pytest-xdist/issues/442>`_: Compatibility fix for upcoming pytest 5.0: ``session.exitstatus`` is now an ``IntEnum`` object.
+
+
+Bug Fixes
+---------
+
+- `#435 <https://github.com/pytest-dev/pytest-xdist/issues/435>`_: No longer show an internal error when we run out of workers due to crashes.
+
+
+pytest-xdist 1.28.0 (2019-04-02)
+================================
+
+Features
+--------
+
+- `#426 <https://github.com/pytest-dev/pytest-xdist/issues/426>`_: ``pytest-xdist`` now uses the new ``pytest_report_to_serializable`` and ``pytest_report_from_serializable``
+  hooks from ``pytest 4.4`` (still experimental). This will make report serialization more reliable and
+  extensible.
+
+  This also means that ``pytest-xdist`` now requires ``pytest>=4.4``.
+
+
+pytest-xdist 1.27.0 (2019-02-15)
+================================
+
+Features
+--------
+
+- `#374 <https://github.com/pytest-dev/pytest-xdist/issues/374>`_: The new ``pytest_xdist_getremotemodule`` hook allows overriding the module run on remote nodes.
+
+- `#415 <https://github.com/pytest-dev/pytest-xdist/issues/415>`_: Improve behavior of ``--numprocesses=auto`` to work well with ``--pdb`` option.
+
+
 pytest-xdist 1.26.1 (2019-01-28)
 ================================
 
@@ -366,6 +411,28 @@ Improved Documentation
 
 .. towncrier release notes start
 
+pytest-xdist 1.30.0 (2019-10-01)
+================================
+
+Features
+--------
+
+- `#448 <https://github.com/pytest-dev/pytest-xdist/issues/448>`_: Initialization between workers and master nodes is now more consistent, which fixes a number of
+  long-standing issues related to startup with the ``-c`` option.
+
+  Issues:
+
+  * `#6 <https://github.com/pytest-dev/pytest-xdist/issues/6>`__: Poor interaction between ``-n#`` and ``-c X.cfg``
+  * `#445 <https://github.com/pytest-dev/pytest-xdist/issues/445>`__: pytest-xdist is not reporting the same nodeid as pytest does
+
+  This however only works with **pytest 5.1 or later**, as it required changes in pytest itself.
+
+
+Bug Fixes
+---------
+
+- `#467 <https://github.com/pytest-dev/pytest-xdist/issues/467>`_: Fix crash issues related to running xdist with the terminal plugin disabled.
+
 
 1.17.0
 ------

appveyor.yml

@@ -5,13 +5,11 @@ environment:
   - TOXENV: "py35-pytestlatest"
   - TOXENV: "py36-pytestlatest"
   - TOXENV: "py37-pytestlatest"
-  - TOXENV: "py27-pytestmaster"
-  - TOXENV: "py36-pytestmaster"
-  - TOXENV: "py27-pytestfeatures"
-  - TOXENV: "py36-pytestfeatures"
+  - TOXENV: "py37-pytestmaster"
+  - TOXENV: "py37-pytestfeatures"
 
 install:
-  - C:\Python37\python -m pip install -U pip
+  - C:\Python37\python -m pip install -U pip setuptools
   - C:\Python37\python -m pip install -U tox setuptools_scm
 
 build: false  # Not a C# project, build stuff at the test step instead.

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-install_requires = ["execnet>=1.1", "pytest>=3.6.0", "pytest-forked", "six"]
+install_requires = ["execnet>=1.1", "pytest>=4.4.0", "pytest-forked", "six"]
 
 
 with open("README.rst") as f:
@@ -8,7 +8,7 @@ with open("README.rst") as f:
 
 setup(
     name="pytest-xdist",
-    use_scm_version={"write_to": "xdist/_version.py"},
+    use_scm_version={"write_to": "src/xdist/_version.py"},
     description="pytest xdist plugin for distributed testing"
     " and loop-on-failing modes",
     long_description=long_description,
@@ -17,7 +17,9 @@ setup(
     author_email="pytest-dev@python.org,holger@merlinux.eu",
     url="https://github.com/pytest-dev/pytest-xdist",
     platforms=["linux", "osx", "win32"],
-    packages=find_packages(exclude=["testing", "example"]),
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    extras_require={"testing": ["filelock"]},
     entry_points={
         "pytest11": ["xdist = xdist.plugin", "xdist.looponfail = xdist.looponfail"]
     },

src/xdist/dsession.py

@@ -46,14 +46,11 @@ class DSession(object):
         self._failed_collection_errors = {}
         self._active_nodes = set()
         self._failed_nodes_count = 0
-        self._max_worker_restart = self.config.option.maxworkerrestart
-        if self._max_worker_restart is not None:
-            self._max_worker_restart = int(self._max_worker_restart)
-        try:
-            self.terminal = config.pluginmanager.getplugin("terminalreporter")
-        except KeyError:
-            self.terminal = None
-        else:
+        self._max_worker_restart = get_default_max_worker_restart(self.config)
+        # summary message to print at the end of the session
+        self._summary_report = None
+        self.terminal = config.pluginmanager.getplugin("terminalreporter")
+        if self.terminal:
             self.trdist = TerminalDistReporter(config)
             config.pluginmanager.register(self.trdist, "terminaldistreporter")
 
@@ -198,15 +195,23 @@ class DSession(object):
         )
         if maximum_reached:
             if self._max_worker_restart == 0:
-                msg = "Worker restarting disabled"
+                msg = "worker {} crashed and worker restarting disabled".format(
+                    node.gateway.id
+                )
             else:
-                msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
-            self.report_line(msg)
+                msg = "maximum crashed workers reached: %d" % self._max_worker_restart
+            self._summary_report = msg
+            self.report_line("\n" + msg)
+            self.triggershutdown()
         else:
-            self.report_line("Replacing crashed worker %s" % node.gateway.id)
+            self.report_line("\nreplacing crashed worker %s" % node.gateway.id)
             self._clone_node(node)
         self._active_nodes.remove(node)
 
+    def pytest_terminal_summary(self, terminalreporter):
+        if self.config.option.verbose >= 0 and self._summary_report:
+            terminalreporter.write_sep("=", "xdist: {}".format(self._summary_report))
+
     def worker_collectionfinish(self, node, ids):
         """worker has finished test collection.
 
@@ -316,7 +321,7 @@ class DSession(object):
         # XXX count no of failures and retry N times
         runner = self.config.pluginmanager.getplugin("runner")
         fspath = nodeid.split("::")[0]
-        msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
+        msg = "worker %r crashed while running %r" % (worker.gateway.id, nodeid)
         rep = runner.TestReport(
             nodeid, (fspath, None, fspath), (), "failed", msg, "???"
         )
@@ -390,10 +395,16 @@ class TerminalDistReporter(object):
             return
         self.write_line("[%s] node down: %s" % (node.gateway.id, error))
 
-    # def pytest_xdist_rsyncstart(self, source, gateways):
-    #    targets = ",".join([gw.id for gw in gateways])
-    #    msg = "[%s] rsyncing: %s" %(targets, source)
-    #    self.write_line(msg)
-    # def pytest_xdist_rsyncfinish(self, source, gateways):
-    #    targets = ", ".join(["[%s]" % gw.id for gw in gateways])
-    #    self.write_line("rsyncfinish: %s -> %s" %(source, targets))
+
+def get_default_max_worker_restart(config):
+    """gets the default value of --max-worker-restart option if it is not provided.
+
+    Use a reasonable default to avoid workers from restarting endlessly due to crashing collections (#226).
+    """
+    result = config.option.maxworkerrestart
+    if result is not None:
+        result = int(result)
+    elif config.option.numprocesses:
+        # if --max-worker-restart was not provided, use a reasonable default (#226)
+        result = config.option.numprocesses * 4
+    return result

src/xdist/looponfail.py

@@ -30,7 +30,7 @@ def pytest_addoption(parser):
 def pytest_cmdline_main(config):
 
     if config.getoption("looponfail"):
-        usepdb = config.getoption("usepdb")  # a core option
+        usepdb = config.getoption("usepdb", False)  # a core option
         if usepdb:
             raise pytest.UsageError("--pdb is incompatible with --looponfail.")
         looponfail_main(config)

src/xdist/newhooks.py

@@ -30,6 +30,11 @@ def pytest_xdist_rsyncfinish(source, gateways):
     """ called after rsyncing a directory to remote gateways takes place. """
 
 
+@pytest.mark.firstresult
+def pytest_xdist_getremotemodule():
+    """ called when creating remote node"""
+
+
 def pytest_configure_node(node):
     """ configure node information before it gets instantiated. """
 

src/xdist/plugin.py

@@ -27,10 +27,14 @@ def auto_detect_cpus():
     return n if n else 1
 
 
+class AutoInt(int):
+    """Mark value as auto-detected."""
+
+
 def parse_numprocesses(s):
     if s == "auto":
-        return auto_detect_cpus()
-    else:
+        return AutoInt(auto_detect_cpus())
+    elif s is not None:
         return int(s)
 
 
@@ -45,7 +49,7 @@ def pytest_addoption(parser):
         type=parse_numprocesses,
         help="shortcut for '--dist=load --tx=NUM*popen', "
         "you can use 'auto' here for auto detection CPUs number on "
-        "host system",
+        "host system and it will be 0 when used with --pdb",
     )
     group.addoption(
         "--maxprocesses",
@@ -150,11 +154,7 @@ def pytest_addoption(parser):
 def pytest_addhooks(pluginmanager):
     from xdist import newhooks
 
-    # avoid warnings with pytest-2.8
-    method = getattr(pluginmanager, "add_hookspecs", None)
-    if method is None:
-        method = pluginmanager.addhooks
-    method(newhooks)
+    pluginmanager.add_hookspecs(newhooks)
 
 
 # -------------------------------------------------------------------------
@@ -170,13 +170,18 @@ def pytest_configure(config):
         session = DSession(config)
         config.pluginmanager.register(session, "dsession")
         tr = config.pluginmanager.getplugin("terminalreporter")
-        tr.showfspath = False
+        if tr:
+            tr.showfspath = False
     if config.getoption("boxed"):
         config.option.forked = True
 
 
 @pytest.mark.tryfirst
 def pytest_cmdline_main(config):
+    usepdb = config.getoption("usepdb", False)  # a core option
+    if isinstance(config.option.numprocesses, AutoInt):
+        config.option.numprocesses = 0 if usepdb else int(config.option.numprocesses)
+
     if config.option.numprocesses:
         if config.option.dist == "no":
             config.option.dist = "load"
@@ -188,11 +193,10 @@ def pytest_cmdline_main(config):
         config.option.dist = "load"
     val = config.getvalue
     if not val("collectonly"):
-        usepdb = config.getoption("usepdb")  # a core option
         if val("dist") != "no":
             if usepdb:
                 raise pytest.UsageError(
-                    "--pdb is incompatible with distributing tests; try using -n0."
+                    "--pdb is incompatible with distributing tests; try using -n0 or -nauto."
                 )  # noqa: E501
 
 

src/xdist/remote.py

@@ -10,10 +10,13 @@ import sys
 import os
 import time
 
+import py
 import _pytest.hookspec
 import pytest
 from execnet.gateway_base import dumps, DumpError
 
+from _pytest.config import _prepareconfig, Config
+
 
 class WorkerInteractor(object):
     def __init__(self, config, channel):
@@ -40,7 +43,8 @@ class WorkerInteractor(object):
 
     @pytest.hookimpl(hookwrapper=True)
     def pytest_sessionfinish(self, exitstatus):
-        self.config.workeroutput["exitstatus"] = exitstatus
+        # in pytest 5.0+, exitstatus is an IntEnum object
+        self.config.workeroutput["exitstatus"] = int(exitstatus)
         yield
         self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
 
@@ -103,7 +107,9 @@ class WorkerInteractor(object):
             self.sendevent("logfinish", nodeid=nodeid, location=location)
 
     def pytest_runtest_logreport(self, report):
-        data = serialize_report(report)
+        data = self.config.hook.pytest_report_to_serializable(
+            config=self.config, report=report
+        )
         data["item_index"] = self.item_index
         data["worker_id"] = self.workerid
         assert self.session.items[self.item_index].nodeid == report.nodeid
@@ -112,7 +118,9 @@ class WorkerInteractor(object):
     def pytest_collectreport(self, report):
         # send only reports that have not passed to master as optimization (#330)
         if not report.passed:
-            data = serialize_report(report)
+            data = self.config.hook.pytest_report_to_serializable(
+                config=self.config, report=report
+            )
             self.sendevent("collectreport", data=data)
 
     # the pytest_logwarning hook was deprecated since pytest 4.0
@@ -142,47 +150,6 @@ class WorkerInteractor(object):
             )
 
 
-def serialize_report(rep):
-    def disassembled_report(rep):
-        reprtraceback = rep.longrepr.reprtraceback.__dict__.copy()
-        reprcrash = rep.longrepr.reprcrash.__dict__.copy()
-
-        new_entries = []
-        for entry in reprtraceback["reprentries"]:
-            entry_data = {"type": type(entry).__name__, "data": entry.__dict__.copy()}
-            for key, value in entry_data["data"].items():
-                if hasattr(value, "__dict__"):
-                    entry_data["data"][key] = value.__dict__.copy()
-            new_entries.append(entry_data)
-
-        reprtraceback["reprentries"] = new_entries
-
-        return {
-            "reprcrash": reprcrash,
-            "reprtraceback": reprtraceback,
-            "sections": rep.longrepr.sections,
-        }
-
-    import py
-
-    d = rep.__dict__.copy()
-    if hasattr(rep.longrepr, "toterminal"):
-        if hasattr(rep.longrepr, "reprtraceback") and hasattr(
-            rep.longrepr, "reprcrash"
-        ):
-            d["longrepr"] = disassembled_report(rep)
-        else:
-            d["longrepr"] = str(rep.longrepr)
-    else:
-        d["longrepr"] = rep.longrepr
-    for name in d:
-        if isinstance(d[name], py.path.local):
-            d[name] = str(d[name])
-        elif name == "result":
-            d[name] = None  # for now
-    return d
-
-
 def serialize_warning_message(warning_message):
     if isinstance(warning_message.message, Warning):
         message_module = type(warning_message.message).__module__
@@ -246,23 +213,21 @@ def getinfodict():
 
 
 def remote_initconfig(option_dict, args):
-    from _pytest.config import Config
-
     option_dict["plugins"].append("no:terminal")
-    config = Config.fromdictargs(option_dict, args)
+    return Config.fromdictargs(option_dict, args)
+
+
+def setup_config(config, basetemp):
     config.option.looponfail = False
     config.option.usepdb = False
     config.option.dist = "no"
     config.option.distload = False
     config.option.numprocesses = None
     config.option.maxprocesses = None
-    config.args = args
-    return config
+    config.option.basetemp = basetemp
 
 
 if __name__ == "__channelexec__":
-    import py
-
     channel = channel  # noqa
     workerinput, args, option_dict, change_sys_path = channel.receive()
 
@@ -276,7 +241,13 @@ if __name__ == "__channelexec__":
     os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
     os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
 
-    config = remote_initconfig(option_dict, args)
+    if hasattr(Config, "InvocationParams"):
+        config = _prepareconfig(args, None)
+    else:
+        config = remote_initconfig(option_dict, args)
+        config.args = args
+
+    setup_config(config, option_dict.get("basetemp"))
     config._parser.prog = os.path.basename(workerinput["mainargv"][0])
     config.workerinput = workerinput
     config.workeroutput = {}

src/xdist/workermanage.py

@@ -3,7 +3,6 @@ import fnmatch
 import os
 import re
 import sys
-import threading
 
 import py
 import pytest
@@ -11,8 +10,6 @@ import execnet
 
 import xdist.remote
 
-from _pytest import runner  # XXX load dynamically
-
 
 def parse_spec_config(config):
     xspeclist = []
@@ -37,7 +34,6 @@ class NodeManager(object):
 
     def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
         self.config = config
-        self._nodesready = threading.Event()
         self.trace = self.config.trace.get("nodemanager")
         self.group = execnet.Group()
         if specs is None:
@@ -116,7 +112,10 @@ class NodeManager(object):
         ignores += self.config.option.rsyncignore
         ignores += self.config.getini("rsyncignore")
 
-        return {"ignores": ignores, "verbose": self.config.option.verbose}
+        return {
+            "ignores": ignores,
+            "verbose": getattr(self.config.option, "verbose", False),
+        }
 
     def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
         """Perform rsync to remote hosts for node."""
@@ -190,6 +189,8 @@ def make_reltoroot(roots, args):
     for arg in args:
         parts = arg.split(splitcode)
         fspath = py.path.local(parts[0])
+        if not fspath.exists():
+            continue
         for root in roots:
             x = fspath.relto(root)
             if x or fspath == root:
@@ -204,7 +205,13 @@ def make_reltoroot(roots, args):
 class WorkerController(object):
     ENDMARK = -1
 
+    class RemoteHook:
+        @pytest.mark.trylast
+        def pytest_xdist_getremotemodule(self):
+            return xdist.remote
+
     def __init__(self, nodemanager, gateway, config, putevent):
+        config.pluginmanager.register(self.RemoteHook())
         self.nodemanager = nodemanager
         self.putevent = putevent
         self.gateway = gateway
@@ -234,20 +241,27 @@ class WorkerController(object):
     def setup(self):
         self.log("setting up worker session")
         spec = self.gateway.spec
-        args = self.config.args
+        if hasattr(self.config, "invocation_params"):
+            args = [str(x) for x in self.config.invocation_params.args or ()]
+            option_dict = {}
+        else:
+            args = self.config.args
+            option_dict = vars(self.config.option)
         if not spec.popen or spec.chdir:
             args = make_reltoroot(self.nodemanager.roots, args)
-        option_dict = vars(self.config.option)
         if spec.popen:
             name = "popen-%s" % self.gateway.id
             if hasattr(self.config, "_tmpdirhandler"):
                 basetemp = self.config._tmpdirhandler.getbasetemp()
                 option_dict["basetemp"] = str(basetemp.join(name))
         self.config.hook.pytest_configure_node(node=self)
-        self.channel = self.gateway.remote_exec(xdist.remote)
+
+        remote_module = self.config.hook.pytest_xdist_getremotemodule()
+        self.channel = self.gateway.remote_exec(remote_module)
         # change sys.path only for remote workers
         change_sys_path = not self.gateway.spec.popen
         self.channel.send((self.workerinput, args, option_dict, change_sys_path))
+
         if self.putevent:
             self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
 
@@ -315,7 +329,9 @@ class WorkerController(object):
                 self.notify_inproc(eventname, node=self, **kwargs)
             elif eventname in ("testreport", "collectreport", "teardownreport"):
                 item_index = kwargs.pop("item_index", None)
-                rep = unserialize_report(eventname, kwargs["data"])
+                rep = self.config.hook.pytest_report_from_serializable(
+                    config=self.config, data=kwargs["data"]
+                )
                 if item_index is not None:
                     rep.item_index = item_index
                 self.notify_inproc(eventname, node=self, rep=rep)
@@ -360,73 +376,6 @@ class WorkerController(object):
             self.notify_inproc("errordown", node=self, error=excinfo)
 
 
-def unserialize_report(name, reportdict):
-    def assembled_report(reportdict):
-        from _pytest._code.code import (
-            ReprEntry,
-            ReprEntryNative,
-            ReprExceptionInfo,
-            ReprFileLocation,
-            ReprFuncArgs,
-            ReprLocals,
-            ReprTraceback,
-        )
-
-        if reportdict["longrepr"]:
-            if (
-                "reprcrash" in reportdict["longrepr"]
-                and "reprtraceback" in reportdict["longrepr"]
-            ):
-
-                reprtraceback = reportdict["longrepr"]["reprtraceback"]
-                reprcrash = reportdict["longrepr"]["reprcrash"]
-
-                unserialized_entries = []
-                reprentry = None
-                for entry_data in reprtraceback["reprentries"]:
-                    data = entry_data["data"]
-                    entry_type = entry_data["type"]
-                    if entry_type == "ReprEntry":
-                        reprfuncargs = None
-                        reprfileloc = None
-                        reprlocals = None
-                        if data["reprfuncargs"]:
-                            reprfuncargs = ReprFuncArgs(**data["reprfuncargs"])
-                        if data["reprfileloc"]:
-                            reprfileloc = ReprFileLocation(**data["reprfileloc"])
-                        if data["reprlocals"]:
-                            reprlocals = ReprLocals(data["reprlocals"]["lines"])
-
-                        reprentry = ReprEntry(
-                            lines=data["lines"],
-                            reprfuncargs=reprfuncargs,
-                            reprlocals=reprlocals,
-                            filelocrepr=reprfileloc,
-                            style=data["style"],
-                        )
-                    elif entry_type == "ReprEntryNative":
-                        reprentry = ReprEntryNative(data["lines"])
-                    else:
-                        report_unserialization_failure(entry_type, name, reportdict)
-                    unserialized_entries.append(reprentry)
-                reprtraceback["reprentries"] = unserialized_entries
-
-                exception_info = ReprExceptionInfo(
-                    reprtraceback=ReprTraceback(**reprtraceback),
-                    reprcrash=ReprFileLocation(**reprcrash),
-                )
-
-                for section in reportdict["longrepr"]["sections"]:
-                    exception_info.addsection(*section)
-                reportdict["longrepr"] = exception_info
-        return reportdict
-
-    if name == "testreport":
-        return runner.TestReport(**assembled_report(reportdict))
-    elif name == "collectreport":
-        return runner.CollectReport(**assembled_report(reportdict))
-
-
 def unserialize_warning_message(data):
     import warnings
     import importlib
@@ -467,17 +416,3 @@ def unserialize_warning_message(data):
         kwargs[attr_name] = data[attr_name]
 
     return warnings.WarningMessage(**kwargs)
-
-
-def report_unserialization_failure(type_name, report_name, reportdict):
-    from pprint import pprint
-
-    url = "https://github.com/pytest-dev/pytest-xdist/issues"
-    stream = py.io.TextIO()
-    pprint("-" * 100, stream=stream)
-    pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream)
-    pprint("report_name: %s" % report_name, stream=stream)
-    pprint(reportdict, stream=stream)
-    pprint("Please report this bug at %s" % url, stream=stream)
-    pprint("-" * 100, stream=stream)
-    assert 0, stream.getvalue()

testing/acceptance_test.py

@@ -559,6 +559,30 @@ def test_fixture_teardown_failure(testdir):
     assert result.ret
 
 
+def test_config_initialization(testdir, pytestconfig):
+    """Ensure workers and master are initialized consistently. Integration test for #445"""
+    if not hasattr(pytestconfig, "invocation_params"):
+        pytest.skip(
+            "requires pytest >=5.1 (config has no attribute 'invocation_params')"
+        )
+    testdir.makepyfile(
+        **{
+            "dir_a/test_foo.py": """
+                def test_1(): pass
+        """
+        }
+    )
+    testdir.makefile(
+        ".ini",
+        myconfig="""
+        [pytest]
+        testpaths=dir_a
+    """,
+    )
+    result = testdir.runpytest("-n2", "-c", "myconfig.ini", "-v")
+    result.stdout.fnmatch_lines(["dir_a/test_foo.py::test_1*"])
+
+
 @pytest.mark.parametrize("when", ["setup", "call", "teardown"])
 def test_crashing_item(testdir, when):
     """Ensure crashing item is correctly reported during all testing stages"""
@@ -849,8 +873,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
-                "*Worker*crashed while running*",
+                "replacing crashed worker gw*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -868,8 +892,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n2")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
-                "*Worker*crashed while running*",
+                "replacing crashed worker gw*",
+                "worker*crashed while running*",
                 "*1 failed*3 passed*",
             ]
         )
@@ -885,8 +909,8 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "--dist=each", "--tx=popen")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
-                "*Worker*crashed while running*",
+                "replacing crashed worker gw*",
+                "worker*crashed while running*",
                 "*1 failed*1 passed*",
             ]
         )
@@ -922,14 +946,35 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=1")
         res.stdout.fnmatch_lines(
             [
-                "*Replacing crashed worker*",
-                "*Maximum crashed workers reached: 1*",
-                "*Worker*crashed while running*",
-                "*Worker*crashed while running*",
+                "replacing crashed worker*",
+                "maximum crashed workers reached: 1*",
+                "worker*crashed while running*",
+                "worker*crashed while running*",
                 "*2 failed*2 passed*",
             ]
         )
 
+    def test_max_worker_restart_tests_queued(self, testdir):
+        f = testdir.makepyfile(
+            """
+            import os, pytest
+            @pytest.mark.parametrize('i', range(10))
+            def test(i): os._exit(1)
+        """
+        )
+        res = testdir.runpytest(f, "-n2", "--max-worker-restart=3")
+        res.stdout.fnmatch_lines(
+            [
+                "replacing crashed worker*",
+                "maximum crashed workers reached: 3*",
+                "worker*crashed while running*",
+                "worker*crashed while running*",
+                "* xdist: maximum crashed workers reached: 3 *",
+                "* 4 failed in *",
+            ]
+        )
+        assert "INTERNALERROR" not in res.stdout.str()
+
     def test_max_worker_restart_die(self, testdir):
         f = testdir.makepyfile(
             """
@@ -939,7 +984,10 @@ class TestNodeFailure:
         )
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
-            ["*Unexpectedly no active workers*", "*INTERNALERROR*"]
+            [
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* no tests ran in *",
+            ]
         )
 
     def test_disable_restart(self, testdir):
@@ -954,9 +1002,10 @@ class TestNodeFailure:
         res = testdir.runpytest(f, "-n4", "--max-worker-restart=0")
         res.stdout.fnmatch_lines(
             [
-                "*Worker restarting disabled*",
-                "*Worker*crashed while running*",
-                "*1 failed*2 passed*",
+                "worker gw* crashed and worker restarting disabled",
+                "*worker*crashed while running*",
+                "* xdist: worker gw* crashed and worker restarting disabled *",
+                "* 1 failed, 2 passed in *",
             ]
         )
 
@@ -1025,6 +1074,22 @@ def test_color_yes_collection_on_non_atty(testdir, request):
     assert "gw0 C / gw1 C" not in result.stdout.str()
 
 
+def test_without_terminal_plugin(testdir, request):
+    """
+    No output when terminal plugin is disabled
+    """
+    testdir.makepyfile(
+        """
+        def test_1():
+            pass
+    """
+    )
+    result = testdir.runpytest("-p", "no:terminal", "-n2")
+    assert result.stdout.str() == ""
+    assert result.stderr.str() == ""
+    assert result.ret == 0
+
+
 def test_internal_error_with_maxfail(testdir):
     """
     Internal error when using --maxfail option (#62, #65).

testing/conftest.py

@@ -1,20 +1,19 @@
+import six
 import py
 import pytest
 import execnet
 
-
-@pytest.fixture(scope="session", autouse=True)
-def _ensure_imports():
-    # we import some modules because pytest-2.8's testdir fixture
-    # will unload all modules after each test and this cause
-    # (unknown) problems with execnet.Group()
-    execnet.Group
-    execnet.makegateway
-
-
 pytest_plugins = "pytester"
 
-# rsyncdirs = ['.', '../xdist', py.path.local(execnet.__file__).dirpath()]
+if six.PY2:
+
+    @pytest.fixture(scope="session", autouse=True)
+    def _ensure_imports():
+        # we import some modules because pytest-2.8's testdir fixture
+        # will unload all modules after each test and this cause
+        # (unknown) problems with execnet.Group()
+        execnet.Group
+        execnet.makegateway
 
 
 @pytest.fixture(autouse=True)
@@ -45,14 +44,6 @@ def specssh(request):
     return getspecssh(request.config)
 
 
-@pytest.fixture
-def testdir(testdir):
-    # pytest before 2.8 did not have a runpytest_subprocess
-    if not hasattr(testdir, "runpytest_subprocess"):
-        testdir.runpytest_subprocess = testdir.runpytest
-    return testdir
-
-
 # configuration information for tests
 def getgspecs(config):
     return [execnet.XSpec(spec) for spec in config.getvalueorskip("gspecs")]

testing/test_dsession.py

@@ -1,4 +1,4 @@
-from xdist.dsession import DSession
+from xdist.dsession import DSession, get_default_max_worker_restart
 from xdist.report import report_collection_diff
 from xdist.scheduler import EachScheduling, LoadScheduling
 
@@ -268,6 +268,24 @@ def test_report_collection_diff_equal():
     assert report_collection_diff(from_collection, to_collection, 1, 2) is None
 
 
+def test_default_max_worker_restart():
+    class config:
+        class option:
+            maxworkerrestart = None
+            numprocesses = 0
+
+    assert get_default_max_worker_restart(config) is None
+
+    config.option.numprocesses = 2
+    assert get_default_max_worker_restart(config) == 8
+
+    config.option.maxworkerrestart = "1"
+    assert get_default_max_worker_restart(config) == 1
+
+    config.option.maxworkerrestart = "0"
+    assert get_default_max_worker_restart(config) == 0
+
+
 def test_report_collection_diff_different():
     """Test reporting of different collections."""
     from_collection = ["aaa", "bbb", "ccc", "YYY"]

testing/test_plugin.py

@@ -36,6 +36,7 @@ def test_dist_options(testdir):
 
 def test_auto_detect_cpus(testdir, monkeypatch):
     import os
+    from xdist.plugin import pytest_cmdline_main as check_options
 
     if hasattr(os, "sched_getaffinity"):
         monkeypatch.setattr(os, "sched_getaffinity", lambda _pid: set(range(99)))
@@ -52,6 +53,11 @@ def test_auto_detect_cpus(testdir, monkeypatch):
     config = testdir.parseconfigure("-nauto")
     assert config.getoption("numprocesses") == 99
 
+    config = testdir.parseconfigure("-nauto", "--pdb")
+    check_options(config)
+    assert config.getoption("usepdb")
+    assert config.getoption("numprocesses") == 0
+
     monkeypatch.delattr(os, "sched_getaffinity", raising=False)
     monkeypatch.setenv("TRAVIS", "true")
     config = testdir.parseconfigure("-nauto")

testing/test_remote.py

@@ -3,8 +3,7 @@ import pprint
 import pytest
 import sys
 
-from xdist.workermanage import WorkerController, unserialize_report
-from xdist.remote import serialize_report
+from xdist.workermanage import WorkerController
 import execnet
 import marshal
 
@@ -81,187 +80,17 @@ def test_remoteinitconfig(testdir):
     assert config2.pluginmanager.getplugin("terminal") in (-1, None)
 
 
-class TestReportSerialization:
-    def test_xdist_longrepr_to_str_issue_241(self, testdir):
-        testdir.makepyfile(
-            """
-            import os
-            def test_a(): assert False
-            def test_b(): pass
-        """
-        )
-        testdir.makeconftest(
-            """
-            def pytest_runtest_logreport(report):
-                print(report.longrepr)
-        """
-        )
-        res = testdir.runpytest("-n1", "-s")
-        res.stdout.fnmatch_lines(["*1 failed, 1 passed *"])
-
-    def test_xdist_report_longrepr_reprcrash_130(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-                    import py
-                    def test_fail(): assert False, 'Expected Message'
-                """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        added_section = ("Failure Metadata", str("metadata metadata"), "*")
-        rep.longrepr.sections.append(added_section)
-        d = serialize_report(rep)
-        check_marshallable(d)
-        a = unserialize_report("testreport", d)
-        # Check assembled == rep
-        assert a.__dict__.keys() == rep.__dict__.keys()
-        for key in rep.__dict__.keys():
-            if key != "longrepr":
-                assert getattr(a, key) == getattr(rep, key)
-        assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno
-        assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message
-        assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path
-        assert rep.longrepr.reprtraceback.entrysep == a.longrepr.reprtraceback.entrysep
-        assert (
-            rep.longrepr.reprtraceback.extraline == a.longrepr.reprtraceback.extraline
-        )
-        assert rep.longrepr.reprtraceback.style == a.longrepr.reprtraceback.style
-        assert rep.longrepr.sections == a.longrepr.sections
-        # Missing section attribute PR171
-        assert added_section in a.longrepr.sections
-
-    def test_reprentries_serialization_170(self, testdir):
-        from _pytest._code.code import ReprEntry
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry():
-                                x = 0
-                                assert x
-                        """,
-            "--showlocals",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntry)
-            assert rep_entries[i].lines == a_entries[i].lines
-            assert rep_entries[i].reprfileloc.lineno == a_entries[i].reprfileloc.lineno
-            assert (
-                rep_entries[i].reprfileloc.message == a_entries[i].reprfileloc.message
-            )
-            assert rep_entries[i].reprfileloc.path == a_entries[i].reprfileloc.path
-            assert rep_entries[i].reprfuncargs.args == a_entries[i].reprfuncargs.args
-            assert rep_entries[i].reprlocals.lines == a_entries[i].reprlocals.lines
-            assert rep_entries[i].style == a_entries[i].style
-
-    def test_reprentries_serialization_196(self, testdir):
-        from _pytest._code.code import ReprEntryNative
-
-        reprec = testdir.inline_runsource(
-            """
-                            def test_repr_entry_native():
-                                x = 0
-                                assert x
-                        """,
-            "--tb=native",
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 3
-        rep = reports[1]
-        d = serialize_report(rep)
-        a = unserialize_report("testreport", d)
-
-        rep_entries = rep.longrepr.reprtraceback.reprentries
-        a_entries = a.longrepr.reprtraceback.reprentries
-        for i in range(len(a_entries)):
-            assert isinstance(rep_entries[i], ReprEntryNative)
-            assert rep_entries[i].lines == a_entries[i].lines
-
-    def test_itemreport_outcomes(self, testdir):
-        reprec = testdir.inline_runsource(
-            """
-            import py
-            def test_pass(): pass
-            def test_fail(): 0/0
-            @py.test.mark.skipif("True")
-            def test_skip(): pass
-            def test_skip_imperative():
-                py.test.skip("hello")
-            @py.test.mark.xfail("True")
-            def test_xfail(): 0/0
-            def test_xfail_imperative():
-                py.test.xfail("hello")
-        """
-        )
-        reports = reprec.getreports("pytest_runtest_logreport")
-        assert len(reports) == 17  # with setup/teardown "passed" reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("testreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if newrep.skipped and not hasattr(newrep, "wasxfail"):
-                assert len(newrep.longrepr) == 3
-            assert newrep.outcome == rep.outcome
-            assert newrep.when == rep.when
-            assert newrep.keywords == rep.keywords
-            if rep.failed:
-                assert newrep.longreprtext == rep.longreprtext
-
-    def test_collectreport_passed(self, testdir):
-        reprec = testdir.inline_runsource("def test_func(): pass")
-        reports = reprec.getreports("pytest_collectreport")
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-
-    def test_collectreport_fail(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-    def test_extended_report_deserialization(self, testdir):
-        reprec = testdir.inline_runsource("qwe abc")
-        reports = reprec.getreports("pytest_collectreport")
-        assert reports
-        for rep in reports:
-            rep.extra = True
-            d = serialize_report(rep)
-            check_marshallable(d)
-            newrep = unserialize_report("collectreport", d)
-            assert newrep.extra
-            assert newrep.passed == rep.passed
-            assert newrep.failed == rep.failed
-            assert newrep.skipped == rep.skipped
-            if rep.failed:
-                assert newrep.longrepr == str(rep.longrepr)
-
-
 class TestWorkerInteractor:
-    def test_basic_collect_and_runtests(self, worker):
+    @pytest.fixture
+    def unserialize_report(self, pytestconfig):
+        def unserialize(data):
+            return pytestconfig.hook.pytest_report_from_serializable(
+                config=pytestconfig, data=data
+            )
+
+        return unserialize
+
+    def test_basic_collect_and_runtests(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             def test_func():
@@ -286,14 +115,14 @@ class TestWorkerInteractor:
         ev = worker.popevent("testreport")  # setup
         ev = worker.popevent("testreport")
         assert ev.name == "testreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.nodeid.endswith("::test_func")
         assert rep.passed
         assert rep.when == "call"
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
 
-    def test_remote_collect_skip(self, worker):
+    def test_remote_collect_skip(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             import pytest
@@ -305,25 +134,25 @@ class TestWorkerInteractor:
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.skipped
         assert rep.longrepr[2] == "Skipped: hello"
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_remote_collect_fail(self, worker):
+    def test_remote_collect_fail(self, worker, unserialize_report):
         worker.testdir.makepyfile("""aasd qwe""")
         worker.setup()
         ev = worker.popevent("collectionstart")
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        rep = unserialize_report(ev.name, ev.kwargs["data"])
+        rep = unserialize_report(ev.kwargs["data"])
         assert rep.failed
         ev = worker.popevent("collectionfinish")
         assert not ev.kwargs["ids"]
 
-    def test_runtests_all(self, worker):
+    def test_runtests_all(self, worker, unserialize_report):
         worker.testdir.makepyfile(
             """
             def test_func(): pass
@@ -345,7 +174,7 @@ class TestWorkerInteractor:
             for i in range(3):  # setup/call/teardown
                 ev = worker.popevent("testreport")
                 assert ev.name == "testreport"
-                rep = unserialize_report(ev.name, ev.kwargs["data"])
+                rep = unserialize_report(ev.kwargs["data"])
                 assert rep.nodeid.endswith(func)
         ev = worker.popevent("workerfinished")
         assert "workeroutput" in ev.kwargs
@@ -457,6 +286,4 @@ def test_remote_usage_prog(testdir, request):
 
     result = testdir.runpytest_subprocess("-n1")
     assert result.ret == 1
-    result.stdout.fnmatch_lines(
-        ["usage: pytest.py *", "pytest.py: error: my_usage_error"]
-    )
+    result.stdout.fnmatch_lines(["*usage: *", "*error: my_usage_error"])

tox.ini

@@ -2,16 +2,15 @@
 envlist=
   linting
   py{27,34,35,36,37}-pytestlatest
-  py{27,36}-pytest{master,features}
+  py37-pytest{master,features}
 
 [testenv]
-changedir=testing
 passenv = USER USERNAME
+extras = testing
 deps =
   pytestlatest: pytest
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
   pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
-  filelock
 commands=
   pytest {posargs}
 
@@ -25,7 +24,7 @@ commands = pre-commit run --all-files --show-diff-on-failure
 [testenv:release]
 changedir=
 decription = do a release, required posarg of the version number
-basepython = python3.6
+basepython = python3.7
 skipsdist = True
 usedevelop = True
 passenv = *
@@ -36,6 +35,7 @@ commands =
 
 [pytest]
 addopts = -ra
+testpaths = testing
 
 [flake8]
 max-line-length = 120