Release 1.23.2

add exception handeling for warning serialization issues

.travis.yml

@@ -22,6 +22,7 @@ env:
 - TOXENV=py-pytest32
 - TOXENV=py-pytest33
 - TOXENV=py-pytest36
+- TOXENV=py-pytest38
 
 install: pip install tox setuptools_scm
 script: tox

CHANGELOG.rst

@@ -1,3 +1,30 @@
+pytest-xdist 1.23.2 (2018-09-28)
+================================
+
+Bug Fixes
+---------
+
+- `#344 <https://github.com/pytest-dev/pytest-xdist/issues/344>`_: Fix issue where Warnings could cause pytest to fail if they do not set the args attribute correctly.
+
+
+pytest-xdist 1.23.1 (2018-09-25)
+================================
+
+Bug Fixes
+---------
+
+- `#341 <https://github.com/pytest-dev/pytest-xdist/issues/341>`_: Fix warnings transfer between workers and master node with pytest >= 3.8.
+
+
+pytest-xdist 1.23.0 (2018-08-23)
+================================
+
+Features
+--------
+
+- `#330 <https://github.com/pytest-dev/pytest-xdist/issues/330>`_: Improve collection performance by reducing the number of events sent to ``master`` node.
+
+
 pytest-xdist 1.22.5 (2018-07-27)
 ================================
 

appveyor.yml

@@ -6,6 +6,7 @@ environment:
   - TOXENV: "py35-pytest33"
   - TOXENV: "py36-pytest33"
   - TOXENV: "py36-pytest36"
+  - TOXENV: "py36-pytest38"
   - TOXENV: "py27-pytest33-pexpect"
   - TOXENV: "py36-pytest33-pexpect"
 

testing/acceptance_test.py

@@ -402,7 +402,7 @@ class TestTerminalReporting:
 
     @pytest.mark.parametrize("n", ["-n0", "-n1"])
     @pytest.mark.parametrize("warn_type", ["pytest", "builtin"])
-    def test_logwarning(self, testdir, n, warn_type):
+    def test_warnings(self, testdir, n, warn_type):
         from pkg_resources import parse_version
 
         if parse_version(pytest.__version__) < parse_version("3.1"):
@@ -417,7 +417,9 @@ class TestTerminalReporting:
             assert False
         testdir.makepyfile(
             """
-            import warnings, py
+            import warnings, py, pytest
+
+            @pytest.mark.filterwarnings('ignore:config.warn has been deprecated')
             def test_func(request):
                 {warn_code}
         """.format(
@@ -427,6 +429,35 @@ class TestTerminalReporting:
         result = testdir.runpytest(n)
         result.stdout.fnmatch_lines(["*this is a warning*", "*1 passed, 1 warnings*"])
 
+    @pytest.mark.parametrize("n", ["-n0", "-n1"])
+    def test_custom_subclass(self, testdir, n):
+        """Check that warning subclasses that don't honor the args attribute don't break
+        pytest-xdist (#344)
+        """
+        from pkg_resources import parse_version
+
+        if parse_version(pytest.__version__) < parse_version("3.1"):
+            pytest.skip("pytest warnings requires >= 3.1")
+
+        testdir.makepyfile(
+            """
+            import warnings, py, pytest
+
+            class MyWarning(UserWarning):
+
+                def __init__(self, p1, p2):
+                    self.p1 = p1
+                    self.p2 = p2
+                    self.args = ()
+
+            def test_func(request):
+                warnings.warn(MyWarning("foo", 1))
+        """
+        )
+        testdir.syspathinsert()
+        result = testdir.runpytest(n)
+        result.stdout.fnmatch_lines(["*MyWarning*", "*1 passed, 1 warnings*"])
+
     def test_logfinish_hook(self, testdir):
         """Ensure the pytest_runtest_logfinish hook is being properly handled"""
         from _pytest import hookspec

testing/test_remote.py

@@ -321,8 +321,6 @@ class TestWorkerInteractor:
         assert not ev.kwargs
         ev = worker.popevent()
         assert ev.name == "collectreport"
-        ev = worker.popevent()
-        assert ev.name == "collectreport"
         rep = unserialize_report(ev.name, ev.kwargs["data"])
         assert rep.failed
         ev = worker.popevent("collectionfinish")

tox.ini

@@ -2,7 +2,7 @@
 # if you change the envlist, please update .travis.yml file as well
 envlist=
   linting
-  py{27,34,35,36}-pytest{30,31,32,33,36}
+  py{27,34,35,36}-pytest{30,31,32,33,36,38}
   py{27,36}-pytest36-pexpect
   py{27,36}-pytest{master,features}
 
@@ -19,6 +19,7 @@ deps =
   pytest32: pytest~=3.2.0
   pytest33: pytest~=3.3.0
   pytest36: pytest~=3.6.0
+  pytest38: pytest~=3.8.0
   pytestmaster: git+https://github.com/pytest-dev/pytest.git@master
   pytestfeatures: git+https://github.com/pytest-dev/pytest.git@features
   pexpect: pexpect

xdist/dsession.py

@@ -257,8 +257,12 @@ class DSession(object):
         self.sched.mark_test_complete(node, item_index, duration)
 
     def worker_collectreport(self, node, rep):
-        """Emitted when a node calls the pytest_collectreport hook."""
+        """Emitted when a node calls the pytest_collectreport hook.
-        if rep.failed:
+
+        Because we only need the report when there's a failure, as optimization
+        we only expect to receive failed reports from workers (#330).
+        """
+        assert rep.failed
         self._failed_worker_collectreport(node, rep)
 
     def worker_logwarning(self, message, code, nodeid, fslocation):
@@ -266,6 +270,11 @@ class DSession(object):
         kwargs = dict(message=message, code=code, nodeid=nodeid, fslocation=fslocation)
         self.config.hook.pytest_logwarning.call_historic(kwargs=kwargs)
 
+    def worker_warning_captured(self, warning_message, when, item):
+        """Emitted when a node calls the pytest_logwarning hook."""
+        kwargs = dict(warning_message=warning_message, when=when, item=item)
+        self.config.hook.pytest_warning_captured.call_historic(kwargs=kwargs)
+
     def _clone_node(self, node):
         """Return new node based on an existing one.
 

xdist/remote.py

@@ -109,6 +109,8 @@ class WorkerInteractor(object):
         self.sendevent("testreport", data=data)
 
     def pytest_collectreport(self, report):
+        # master only needs reports that failed, as optimization send only them instead (#330)
+        if report.failed:
             data = serialize_report(report)
             self.sendevent("collectreport", data=data)
 
@@ -121,6 +123,18 @@ class WorkerInteractor(object):
             fslocation=str(fslocation),
         )
 
+    # the pytest_warning_captured hook was introduced in pytest 3.8
+    if hasattr(_pytest.hookspec, "pytest_warning_captured"):
+
+        def pytest_warning_captured(self, warning_message, when, item):
+            self.sendevent(
+                "warning_captured",
+                warning_message_data=serialize_warning_message(warning_message),
+                when=when,
+                # item cannot be serialized and will always be None when used with xdist
+                item=None,
+            )
+
 
 def serialize_report(rep):
     def disassembled_report(rep):
@@ -163,6 +177,40 @@ def serialize_report(rep):
     return d
 
 
+def serialize_warning_message(warning_message):
+    if isinstance(warning_message.message, Warning):
+        message_module = type(warning_message.message).__module__
+        message_class_name = type(warning_message.message).__name__
+        message_args = warning_message.message.args
+        message_str = str(warning_message.message)
+    else:
+        message_str = warning_message.message
+        message_module = None
+        message_class_name = None
+        message_args = None
+    if warning_message.category:
+        category_module = warning_message.category.__module__
+        category_class_name = warning_message.category.__name__
+    else:
+        category_module = None
+        category_class_name = None
+
+    result = {
+        "message_str": message_str,
+        "message_module": message_module,
+        "message_class_name": message_class_name,
+        "message_args": message_args,
+        "category_module": category_module,
+        "category_class_name": category_class_name,
+    }
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warning_message._WARNING_DETAILS:
+        if attr_name in ("message", "category"):
+            continue
+        result[attr_name] = getattr(warning_message, attr_name)
+    return result
+
+
 def getinfodict():
     import platform
 

xdist/workermanage.py

@@ -327,6 +327,16 @@ class WorkerController(object):
                     nodeid=kwargs["nodeid"],
                     fslocation=kwargs["nodeid"],
                 )
+            elif eventname == "warning_captured":
+                warning_message = unserialize_warning_message(
+                    kwargs["warning_message_data"]
+                )
+                self.notify_inproc(
+                    eventname,
+                    warning_message=warning_message,
+                    when=kwargs["when"],
+                    item=kwargs["item"],
+                )
             else:
                 raise ValueError("unknown event: %s" % (eventname,))
         except KeyboardInterrupt:
@@ -409,6 +419,41 @@ def unserialize_report(name, reportdict):
         return runner.CollectReport(**assembled_report(reportdict))
 
 
+def unserialize_warning_message(data):
+    import warnings
+    import importlib
+
+    if data["message_module"]:
+        mod = importlib.import_module(data["message_module"])
+        cls = getattr(mod, data["message_class_name"])
+        try:
+            message = cls(*data["message_args"])
+        except TypeError:
+            message_text = "{mod}.{cls}: {msg}".format(
+                mod=data["message_module"],
+                cls=data["message_class_name"],
+                msg=data["message_str"],
+            )
+            message = Warning(message_text)
+    else:
+        message = data["message_str"]
+
+    if data["category_module"]:
+        mod = importlib.import_module(data["category_module"])
+        category = getattr(mod, data["category_class_name"])
+    else:
+        category = None
+
+    kwargs = {"message": message, "category": category}
+    # access private _WARNING_DETAILS because the attributes vary between Python versions
+    for attr_name in warnings.WarningMessage._WARNING_DETAILS:
+        if attr_name in ("message", "category"):
+            continue
+        kwargs[attr_name] = data[attr_name]
+
+    return warnings.WarningMessage(**kwargs)
+
+
 def report_unserialization_failure(type_name, report_name, reportdict):
     from pprint import pprint