IQ.Lvbs/pytest-xdist
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run pre-commit: black, whitespaces, rst

Browse Source
This commit is contained in:
Bruno Oliveira
2018-07-27 17:50:55 -03:00
parent 1ed3884da9
commit fced1645cb
25 changed files with 1153 additions and 934 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
xdist/__init__.py
View File

@@ -1,3 +1,3 @@
from xdist._version import version as __version__
__all__ = ['__version__']
__all__ = ["__version__"]

71
xdist/dsession.py
View File

@@ -84,7 +84,7 @@ class DSession(object):
def pytest_sessionfinish(self, session):
"""Shutdown all nodes."""
nm = getattr(self, 'nodemanager', None) # if not fully initialized
nm = getattr(self, "nodemanager", None) # if not fully initialized
if nm is not None:
nm.teardown_nodes()
self._session = None
@@ -95,19 +95,18 @@ class DSession(object):
@pytest.mark.trylast
def pytest_xdist_make_scheduler(self, config, log):
dist = config.getvalue('dist')
dist = config.getvalue("dist")
schedulers = {
'each': EachScheduling,
'load': LoadScheduling,
'loadscope': LoadScopeScheduling,
'loadfile': LoadFileScheduling,
"each": EachScheduling,
"load": LoadScheduling,
"loadscope": LoadScopeScheduling,
"loadfile": LoadFileScheduling,
}
return schedulers[dist](config, log)
def pytest_runtestloop(self):
self.sched = self.config.hook.pytest_xdist_make_scheduler(
config=self.config,
log=self.log
config=self.config, log=self.log
)
assert self.sched is not None
@@ -151,8 +150,8 @@ class DSession(object):
collection without any further input.
"""
node.workerinfo = workerinfo
node.workerinfo['id'] = node.gateway.id
node.workerinfo['spec'] = node.gateway.spec
node.workerinfo["id"] = node.gateway.id
node.workerinfo["spec"] = node.gateway.spec
# TODO: (#234 task) needs this for pytest. Remove when refactor in pytest repo
node.slaveinfo = node.workerinfo
@@ -172,7 +171,7 @@ class DSession(object):
workerready before shutdown was triggered.
"""
self.config.hook.pytest_testnodedown(node=node, error=None)
if node.workeroutput['exitstatus'] == 2: # keyboard-interrupt
if node.workeroutput["exitstatus"] == 2: # keyboard-interrupt
self.shouldstop = "%s received keyboard-interrupt" % (node,)
self.worker_errordown(node, "keyboard-interrupt")
return
@@ -193,14 +192,15 @@ class DSession(object):
self.handle_crashitem(crashitem, node)
self._failed_nodes_count += 1
maximum_reached = (self._max_worker_restart is not None and
self._failed_nodes_count > self._max_worker_restart)
maximum_reached = (
self._max_worker_restart is not None
and self._failed_nodes_count > self._max_worker_restart
)
if maximum_reached:
if self._max_worker_restart == 0:
msg = 'Worker restarting disabled'
msg = "Worker restarting disabled"
else:
msg = "Maximum crashed workers reached: %d" % \
self._max_worker_restart
msg = "Maximum crashed workers reached: %d" % self._max_worker_restart
self.report_line(msg)
else:
self.report_line("Replacing crashed worker %s" % node.gateway.id)
@@ -218,8 +218,7 @@ class DSession(object):
"""
if self.shuttingdown:
return
self.config.hook.pytest_xdist_node_collection_finished(node=node,
ids=ids)
self.config.hook.pytest_xdist_node_collection_finished(node=node, ids=ids)
# tell session which items were effectively collected otherwise
# the master node will finish the session with EXIT_NOTESTSCOLLECTED
self._session.testscollected = len(ids)
@@ -230,19 +229,18 @@ class DSession(object):
if self.terminal and not self.sched.has_pending:
self.trdist.ensure_show_status()
self.terminal.write_line("")
self.terminal.write_line("scheduling tests via %s" % (
self.sched.__class__.__name__))
self.terminal.write_line(
"scheduling tests via %s" % (self.sched.__class__.__name__)
)
self.sched.schedule()
def worker_logstart(self, node, nodeid, location):
"""Emitted when a node calls the pytest_runtest_logstart hook."""
self.config.hook.pytest_runtest_logstart(
nodeid=nodeid, location=location)
self.config.hook.pytest_runtest_logstart(nodeid=nodeid, location=location)
def worker_logfinish(self, node, nodeid, location):
"""Emitted when a node calls the pytest_runtest_logfinish hook."""
self.config.hook.pytest_runtest_logfinish(
nodeid=nodeid, location=location)
self.config.hook.pytest_runtest_logfinish(nodeid=nodeid, location=location)
def worker_testreport(self, node, rep):
"""Emitted when a node calls the pytest_runtest_logreport hook."""
@@ -295,8 +293,7 @@ class DSession(object):
if rep.failed:
self.countfailures += 1
if self.maxfail and self.countfailures >= self.maxfail:
self.shouldstop = "stopping after %d failures" % (
self.countfailures)
self.shouldstop = "stopping after %d failures" % (self.countfailures)
def triggershutdown(self):
self.log("triggering shutdown")
@@ -310,8 +307,9 @@ class DSession(object):
runner = self.config.pluginmanager.getplugin("runner")
fspath = nodeid.split("::")[0]
msg = "Worker %r crashed while running %r" % (worker.gateway.id, nodeid)
rep = runner.TestReport(nodeid, (fspath, None, fspath),
(), "failed", msg, "???")
rep = runner.TestReport(
nodeid, (fspath, None, fspath), (), "failed", msg, "???"
)
rep.node = worker
self.config.hook.pytest_runtest_logreport(report=rep)
@@ -322,7 +320,7 @@ class TerminalDistReporter(object):
self.tr = config.pluginmanager.getplugin("terminalreporter")
self._status = {}
self._lastlen = 0
self._isatty = getattr(self.tr, 'isatty', self.tr.hasmarkup)
self._isatty = getattr(self.tr, "isatty", self.tr.hasmarkup)
def write_line(self, msg):
self.tr.write_line(msg)
@@ -337,8 +335,7 @@ class TerminalDistReporter(object):
self.rewrite(self.getstatus())
def getstatus(self):
parts = ["%s %s" % (spec.id, self._status[spec.id])
for spec in self._specs]
parts = ["%s %s" % (spec.id, self._status[spec.id]) for spec in self._specs]
return " / ".join(parts)
def rewrite(self, line, newline=False):
@@ -361,17 +358,17 @@ class TerminalDistReporter(object):
if self.config.option.verbose > 0:
rinfo = gateway._rinfo()
version = "%s.%s.%s" % rinfo.version_info[:3]
self.rewrite("[%s] %s Python %s cwd: %s" % (
gateway.id, rinfo.platform, version, rinfo.cwd),
newline=True)
self.rewrite(
"[%s] %s Python %s cwd: %s"
% (gateway.id, rinfo.platform, version, rinfo.cwd),
newline=True,
)
self.setstatus(gateway.spec, "C")
def pytest_testnodeready(self, node):
if self.config.option.verbose > 0:
d = node.workerinfo
infoline = "[%s] Python %s" % (
d['id'],
d['version'].replace('\n', ' -- '),)
infoline = "[%s] Python %s" % (d["id"], d["version"].replace("\n", " -- "))
self.rewrite(infoline, newline=True)
self.setstatus(node.gateway.spec, "ok")

42
xdist/looponfail.py
View File

@@ -17,19 +17,22 @@ import execnet
def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing")
group._addoption(
'-f', '--looponfail',
action="store_true", dest="looponfail", default=False,
"-f",
"--looponfail",
action="store_true",
dest="looponfail",
default=False,
help="run tests in subprocess, wait for modified files "
"and re-run failing test set until all pass.")
"and re-run failing test set until all pass.",
)
def pytest_cmdline_main(config):
if config.getoption("looponfail"):
usepdb = config.getoption('usepdb') # a core option
usepdb = config.getoption("usepdb") # a core option
if usepdb:
raise pytest.UsageError(
"--pdb incompatible with --looponfail.")
raise pytest.UsageError("--pdb incompatible with --looponfail.")
looponfail_main(config)
return 2 # looponfail only can get stop with ctrl-C anyway
@@ -45,8 +48,8 @@ def looponfail_main(config):
# the last failures passed, let's immediately rerun all
continue
repr_pytest_looponfailinfo(
failreports=remotecontrol.failures,
rootdirs=rootdirs)
failreports=remotecontrol.failures, rootdirs=rootdirs
)
statrecorder.waitonchange(checkinterval=2.0)
except KeyboardInterrupt:
print()
@@ -68,7 +71,7 @@ class RemoteControl(object):
def setup(self, out=None):
if out is None:
out = py.io.TerminalWriter()
if hasattr(self, 'gateway'):
if hasattr(self, "gateway"):
raise ValueError("already have gateway %r" % self.gateway)
self.trace("setting up worker session")
self.gateway = self.initgateway()
@@ -82,15 +85,16 @@ class RemoteControl(object):
def write(s):
out._file.write(s)
out._file.flush()
remote_outchannel.setcallback(write)
def ensure_teardown(self):
if hasattr(self, 'channel'):
if hasattr(self, "channel"):
if not self.channel.isclosed():
self.trace("closing", self.channel)
self.channel.close()
del self.channel
if hasattr(self, 'gateway'):
if hasattr(self, "gateway"):
self.trace("exiting", self.gateway)
self.gateway.exit()
del self.gateway
@@ -138,8 +142,9 @@ def repr_pytest_looponfailinfo(failreports, rootdirs):
def init_worker_session(channel, args, option_dict):
import os
import sys
outchannel = channel.gateway.newchannel()
sys.stdout = sys.stderr = outchannel.makefile('w')
sys.stdout = sys.stderr = outchannel.makefile("w")
channel.send(outchannel)
# prune sys.path to not contain relative paths
newpaths = []
@@ -152,9 +157,11 @@ def init_worker_session(channel, args, option_dict):
# fullwidth, hasmarkup = channel.receive()
from _pytest.config import Config
config = Config.fromdictargs(option_dict, list(args))
config.args = args
from xdist.looponfail import WorkerFailSession
WorkerFailSession(config, channel).main()
@@ -181,7 +188,8 @@ class WorkerFailSession(object):
except pytest.UsageError:
items = session.perform_collect(None)
hook.pytest_collection_modifyitems(
session=session, config=session.config, items=items)
session=session, config=session.config, items=items
)
hook.pytest_collection_finish(session=session)
return True
@@ -207,7 +215,7 @@ class WorkerFailSession(object):
for rep in self.recorded_failures:
trails.append(rep.nodeid)
loc = rep.longrepr
loc = str(getattr(loc, 'reprcrash', loc))
loc = str(getattr(loc, "reprcrash", loc))
failreports.append(loc)
self.channel.send((trails, failreports, self.collection_failed))
@@ -245,8 +253,10 @@ class StatRecorder(object):
changed = True
else:
if oldstat:
if oldstat.mtime != curstat.mtime or \
oldstat.size != curstat.size:
if (
oldstat.mtime != curstat.mtime
or oldstat.size != curstat.size
):
changed = True
print("# MODIFIED", path)
if removepycfiles and path.ext == ".py":

145
xdist/plugin.py
View File

@@ -8,7 +8,7 @@ def auto_detect_cpus():
try:
from os import sched_getaffinity
except ImportError:
if os.environ.get('TRAVIS') == 'true':
if os.environ.get("TRAVIS") == "true":
# workaround https://bitbucket.org/pypy/pypy/issues/2375
return 2
try:
@@ -16,6 +16,7 @@ def auto_detect_cpus():
except ImportError:
from multiprocessing import cpu_count
else:
def cpu_count():
return len(sched_getaffinity(0))
@@ -27,7 +28,7 @@ def auto_detect_cpus():
def parse_numprocesses(s):
if s == 'auto':
if s == "auto":
return auto_detect_cpus()
else:
return int(s)
@@ -36,60 +37,101 @@ def parse_numprocesses(s):
def pytest_addoption(parser):
group = parser.getgroup("xdist", "distributed and subprocess testing")
group._addoption(
'-n', '--numprocesses', dest="numprocesses", metavar="numprocesses",
"-n",
"--numprocesses",
dest="numprocesses",
metavar="numprocesses",
action="store",
type=parse_numprocesses,
help="shortcut for '--dist=load --tx=NUM*popen', "
"you can use 'auto' here for auto detection CPUs number on "
"host system")
group.addoption('--max-worker-restart', '--max-slave-restart', action="store", default=None,
dest="maxworkerrestart",
help="maximum number of workers that can be restarted "
"when crashed (set to zero to disable this feature)\n"
"'--max-slave-restart' option is deprecated and will be removed in "
"a future release")
"you can use 'auto' here for auto detection CPUs number on "
"host system",
)
group.addoption(
'--dist', metavar="distmode",
action="store", choices=['each', 'load', 'loadscope', 'loadfile', 'no'],
dest="dist", default="no",
help=("set mode for distributing tests to exec environments.\n\n"
"each: send each test to all available environments.\n\n"
"load: load balance by sending any pending test to any"
" available environment.\n\n"
"loadscope: load balance by sending pending groups of tests in"
" the same scope to any available environment.\n\n"
"loadfile: load balance by sending test grouped by file"
" to any available environment.\n\n"
"(default) no: run tests inprocess, don't distribute."))
"--max-worker-restart",
"--max-slave-restart",
action="store",
default=None,
dest="maxworkerrestart",
help="maximum number of workers that can be restarted "
"when crashed (set to zero to disable this feature)\n"
"'--max-slave-restart' option is deprecated and will be removed in "
"a future release",
)
group.addoption(
'--tx', dest="tx", action="append", default=[],
"--dist",
metavar="distmode",
action="store",
choices=["each", "load", "loadscope", "loadfile", "no"],
dest="dist",
default="no",
help=(
"set mode for distributing tests to exec environments.\n\n"
"each: send each test to all available environments.\n\n"
"load: load balance by sending any pending test to any"
" available environment.\n\n"
"loadscope: load balance by sending pending groups of tests in"
" the same scope to any available environment.\n\n"
"loadfile: load balance by sending test grouped by file"
" to any available environment.\n\n"
"(default) no: run tests inprocess, don't distribute."
),
)
group.addoption(
"--tx",
dest="tx",
action="append",
default=[],
metavar="xspec",
help=("add a test execution environment. some examples: "
"--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
"--tx ssh=user@codespeak.net//chdir=testcache"))
help=(
"add a test execution environment. some examples: "
"--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
"--tx ssh=user@codespeak.net//chdir=testcache"
),
)
group._addoption(
'-d',
action="store_true", dest="distload", default=False,
help="load-balance tests. shortcut for '--dist=load'")
"-d",
action="store_true",
dest="distload",
default=False,
help="load-balance tests. shortcut for '--dist=load'",
)
group.addoption(
'--rsyncdir', action="append", default=[], metavar="DIR",
help="add directory for rsyncing to remote tx nodes.")
"--rsyncdir",
action="append",
default=[],
metavar="DIR",
help="add directory for rsyncing to remote tx nodes.",
)
group.addoption(
'--rsyncignore', action="append", default=[], metavar="GLOB",
help="add expression for ignores when rsyncing to remote tx nodes.")
"--rsyncignore",
action="append",
default=[],
metavar="GLOB",
help="add expression for ignores when rsyncing to remote tx nodes.",
)
group.addoption(
"--boxed", action="store_true",
help="backward compatibility alias for pytest-forked --forked")
"--boxed",
action="store_true",
help="backward compatibility alias for pytest-forked --forked",
)
parser.addini(
'rsyncdirs', 'list of (relative) paths to be rsynced for'
' remote distributed testing.', type="pathlist")
"rsyncdirs",
"list of (relative) paths to be rsynced for" " remote distributed testing.",
type="pathlist",
)
parser.addini(
'rsyncignore', 'list of (relative) glob-style paths to be ignored '
'for rsyncing.', type="pathlist")
"rsyncignore",
"list of (relative) glob-style paths to be ignored " "for rsyncing.",
type="pathlist",
)
parser.addini(
"looponfailroots", type="pathlist",
help="directories to check for changes", default=[py.path.local()])
"looponfailroots",
type="pathlist",
help="directories to check for changes",
default=[py.path.local()],
)
# -------------------------------------------------------------------------
@@ -99,12 +141,14 @@ def pytest_addoption(parser):
def pytest_addhooks(pluginmanager):
from xdist import newhooks
# avoid warnings with pytest-2.8
method = getattr(pluginmanager, "add_hookspecs", None)
if method is None:
method = pluginmanager.addhooks
method(newhooks)
# -------------------------------------------------------------------------
# distributed testing initialization
# -------------------------------------------------------------------------
@@ -114,6 +158,7 @@ def pytest_addhooks(pluginmanager):
def pytest_configure(config):
if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
from xdist.dsession import DSession
session = DSession(config)
config.pluginmanager.register(session, "dsession")
tr = config.pluginmanager.getplugin("terminalreporter")
@@ -125,18 +170,20 @@ def pytest_configure(config):
@pytest.mark.tryfirst
def pytest_cmdline_main(config):
if config.option.numprocesses:
if config.option.dist == 'no':
if config.option.dist == "no":
config.option.dist = "load"
config.option.tx = ['popen'] * config.option.numprocesses
config.option.tx = ["popen"] * config.option.numprocesses
if config.option.distload:
config.option.dist = "load"
val = config.getvalue
if not val("collectonly"):
usepdb = config.getoption('usepdb') # a core option
usepdb = config.getoption("usepdb") # a core option
if val("dist") != "no":
if usepdb:
raise pytest.UsageError(
"--pdb is incompatible with distributing tests; try using -n0.") # noqa: E501
"--pdb is incompatible with distributing tests; try using -n0."
) # noqa: E501
# -------------------------------------------------------------------------
# fixtures
@@ -148,7 +195,7 @@ def worker_id(request):
"""Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
if running on the master node.
"""
if hasattr(request.config, 'workerinput'):
return request.config.workerinput['workerid']
if hasattr(request.config, "workerinput"):
return request.config.workerinput["workerid"]
else:
return 'master'
return "master"

82
xdist/remote.py
View File

@@ -17,7 +17,7 @@ import pytest
class WorkerInteractor(object):
def __init__(self, config, channel):
self.config = config
self.workerid = config.workerinput.get('workerid', "?")
self.workerid = config.workerinput.get("workerid", "?")
self.log = py.log.Producer("worker-%s" % self.workerid)
if not config.option.debug:
py.log.setconsumer(self.log._keywords, None)
@@ -39,7 +39,7 @@ class WorkerInteractor(object):
@pytest.hookimpl(hookwrapper=True)
def pytest_sessionfinish(self, exitstatus):
self.config.workeroutput['exitstatus'] = exitstatus
self.config.workeroutput["exitstatus"] = exitstatus
yield
self.sendevent("workerfinished", workeroutput=self.config.workeroutput)
@@ -56,7 +56,7 @@ class WorkerInteractor(object):
return True
self.log("received command", name, kwargs)
if name == "runtests":
torun.extend(kwargs['indices'])
torun.extend(kwargs["indices"])
elif name == "runtests_all":
torun.extend(range(len(session.items)))
self.log("items to run:", torun)
@@ -79,24 +79,25 @@ class WorkerInteractor(object):
nextitem = None
start = time.time()
self.config.hook.pytest_runtest_protocol(
item=item,
nextitem=nextitem)
self.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
duration = time.time() - start
self.sendevent("runtest_protocol_complete", item_index=self.item_index,
duration=duration)
self.sendevent(
"runtest_protocol_complete", item_index=self.item_index, duration=duration
)
def pytest_collection_finish(self, session):
self.sendevent(
"collectionfinish",
topdir=str(session.fspath),
ids=[item.nodeid for item in session.items])
ids=[item.nodeid for item in session.items],
)
def pytest_runtest_logstart(self, nodeid, location):
self.sendevent("logstart", nodeid=nodeid, location=location)
# the pytest_runtest_logfinish hook was introduced in pytest 3.4
if hasattr(_pytest.hookspec, 'pytest_runtest_logfinish'):
if hasattr(_pytest.hookspec, "pytest_runtest_logfinish"):
def pytest_runtest_logfinish(self, nodeid, location):
self.sendevent("logfinish", nodeid=nodeid, location=location)
@@ -112,8 +113,13 @@ class WorkerInteractor(object):
self.sendevent("collectreport", data=data)
def pytest_logwarning(self, message, code, nodeid, fslocation):
self.sendevent("logwarning", message=message, code=code, nodeid=nodeid,
fslocation=str(fslocation))
self.sendevent(
"logwarning",
message=message,
code=code,
nodeid=nodeid,
fslocation=str(fslocation),
)
def serialize_report(rep):
@@ -122,34 +128,33 @@ def serialize_report(rep):
reprcrash = rep.longrepr.reprcrash.__dict__.copy()
new_entries = []
for entry in reprtraceback['reprentries']:
entry_data = {
'type': type(entry).__name__,
'data': entry.__dict__.copy(),
}
for key, value in entry_data['data'].items():
if hasattr(value, '__dict__'):
entry_data['data'][key] = value.__dict__.copy()
for entry in reprtraceback["reprentries"]:
entry_data = {"type": type(entry).__name__, "data": entry.__dict__.copy()}
for key, value in entry_data["data"].items():
if hasattr(value, "__dict__"):
entry_data["data"][key] = value.__dict__.copy()
new_entries.append(entry_data)
reprtraceback['reprentries'] = new_entries
reprtraceback["reprentries"] = new_entries
return {
'reprcrash': reprcrash,
'reprtraceback': reprtraceback,
'sections': rep.longrepr.sections
"reprcrash": reprcrash,
"reprtraceback": reprtraceback,
"sections": rep.longrepr.sections,
}
import py
d = rep.__dict__.copy()
if hasattr(rep.longrepr, 'toterminal'):
if hasattr(rep.longrepr, 'reprtraceback') \
and hasattr(rep.longrepr, 'reprcrash'):
d['longrepr'] = disassembled_report(rep)
if hasattr(rep.longrepr, "toterminal"):
if hasattr(rep.longrepr, "reprtraceback") and hasattr(
rep.longrepr, "reprcrash"
):
d["longrepr"] = disassembled_report(rep)
else:
d['longrepr'] = str(rep.longrepr)
d["longrepr"] = str(rep.longrepr)
else:
d['longrepr'] = rep.longrepr
d["longrepr"] = rep.longrepr
for name in d:
if isinstance(d[name], py.path.local):
d[name] = str(d[name])
@@ -160,6 +165,7 @@ def serialize_report(rep):
def getinfodict():
import platform
return dict(
version=sys.version,
version_info=tuple(sys.version_info),
@@ -172,7 +178,8 @@ def getinfodict():
def remote_initconfig(option_dict, args):
from _pytest.config import Config
option_dict['plugins'].append("no:terminal")
option_dict["plugins"].append("no:terminal")
config = Config.fromdictargs(option_dict, args)
config.option.looponfail = False
config.option.usepdb = False
@@ -183,18 +190,19 @@ def remote_initconfig(option_dict, args):
return config
if __name__ == '__channelexec__':
if __name__ == "__channelexec__":
channel = channel # noqa
workerinput, args, option_dict = channel.receive()
importpath = os.getcwd()
sys.path.insert(0, importpath) # XXX only for remote situations
os.environ['PYTHONPATH'] = (
importpath + os.pathsep +
os.environ.get('PYTHONPATH', ''))
os.environ['PYTEST_XDIST_WORKER'] = workerinput['workerid']
os.environ['PYTEST_XDIST_WORKER_COUNT'] = str(workerinput['workercount'])
os.environ["PYTHONPATH"] = (
importpath + os.pathsep + os.environ.get("PYTHONPATH", "")
)
os.environ["PYTEST_XDIST_WORKER"] = workerinput["workerid"]
os.environ["PYTEST_XDIST_WORKER_COUNT"] = str(workerinput["workercount"])
# os.environ['PYTHONPATH'] = importpath
import py
config = remote_initconfig(option_dict, args)
config.workerinput = workerinput
config.workeroutput = {}

15
xdist/report.py
View File

@@ -11,16 +11,11 @@ def report_collection_diff(from_collection, to_collection, from_id, to_id):
if from_collection == to_collection:
return None
diff = unified_diff(
from_collection,
to_collection,
fromfile=from_id,
tofile=to_id,
)
diff = unified_diff(from_collection, to_collection, fromfile=from_id, tofile=to_id)
error_message = (
u'Different tests were collected between {from_id} and {to_id}. '
u'The difference is:\n'
u'{diff}'
).format(from_id=from_id, to_id=to_id, diff='\n'.join(diff))
u"Different tests were collected between {from_id} and {to_id}. "
u"The difference is:\n"
u"{diff}"
).format(from_id=from_id, to_id=to_id, diff="\n".join(diff))
msg = "\n".join([x.rstrip() for x in error_message.split("\n")])
return msg

10
xdist/scheduler/each.py
View File

@@ -86,10 +86,12 @@ class EachScheduling(object):
if deadnode.gateway.spec == node.gateway.spec:
dead_collection = self.node2collection[deadnode]
if collection != dead_collection:
msg = report_collection_diff(dead_collection,
collection,
deadnode.gateway.id,
node.gateway.id)
msg = report_collection_diff(
dead_collection,
collection,
deadnode.gateway.id,
node.gateway.id,
)
self.log(msg)
return
pending = self._removed2pending.pop(deadnode)

4
xdist/scheduler/filescope.py
View File

@@ -23,7 +23,7 @@ class LoadFileScheduling(LoadScopeScheduling):
def __init__(self, config, log=None):
super(LoadFileScheduling, self).__init__(config, log)
if log is None:
self.log = Producer('loadfilesched')
self.log = Producer("loadfilesched")
else:
self.log = log.loadfilesched
@@ -49,4 +49,4 @@ class LoadFileScheduling(LoadScopeScheduling):
example/loadsuite/test/test_delta.py
example/loadsuite/epsilon/__init__.py
"""
return nodeid.split('::', 1)[0]
return nodeid.split("::", 1)[0]

21
xdist/scheduler/load.py
View File

@@ -133,10 +133,9 @@ class LoadScheduling(object):
assert self.collection
if collection != self.collection:
other_node = next(iter(self.node2collection.keys()))
msg = report_collection_diff(self.collection,
collection,
other_node.gateway.id,
node.gateway.id)
msg = report_collection_diff(
self.collection, collection, other_node.gateway.id, node.gateway.id
)
self.log(msg)
return
self.node2collection[node] = list(collection)
@@ -226,7 +225,7 @@ class LoadScheduling(object):
# XXX allow nodes to have different collections
if not self._check_nodes_have_same_collection():
self.log('**Different tests collected, aborting run**')
self.log("**Different tests collected, aborting run**")
return
# Collections are identical, create the index of pending items.
@@ -238,8 +237,7 @@ class LoadScheduling(object):
# Send a batch of tests to run. If we don't have at least two
# tests per node, we have to send them all so that we can send
# shutdown signals and get all nodes working.
initial_batch = max(len(self.pending) // 4,
2 * len(self.nodes))
initial_batch = max(len(self.pending) // 4, 2 * len(self.nodes))
# distribute tests round-robin up to the batch size
# (or until we run out)
@@ -271,18 +269,15 @@ class LoadScheduling(object):
same_collection = True
for node, collection in node_collection_items[1:]:
msg = report_collection_diff(
col,
collection,
first_node.gateway.id,
node.gateway.id,
col, collection, first_node.gateway.id, node.gateway.id
)
if msg:
same_collection = False
self.log(msg)
if self.config is not None:
rep = CollectReport(
node.gateway.id, 'failed',
longrepr=msg, result=[])
node.gateway.id, "failed", longrepr=msg, result=[]
)
self.config.hook.pytest_collectreport(report=rep)
return same_collection

45
xdist/scheduler/loadscope.py
View File

@@ -93,7 +93,7 @@ class LoadScopeScheduling(object):
self.registered_collections = OrderedDict()
if log is None:
self.log = Producer('loadscopesched')
self.log = Producer("loadscopesched")
else:
self.log = log.loadscopesched
@@ -187,8 +187,7 @@ class LoadScopeScheduling(object):
break
else:
raise RuntimeError(
'Unable to identify crashitem on a workload with '
'pending items'
"Unable to identify crashitem on a workload with " "pending items"
)
# Made uncompleted work unit available again
@@ -224,10 +223,7 @@ class LoadScopeScheduling(object):
other_node = next(iter(self.registered_collections.keys()))
msg = report_collection_diff(
self.collection,
collection,
other_node.gateway.id,
node.gateway.id
self.collection, collection, other_node.gateway.id, node.gateway.id
)
self.log(msg)
return
@@ -255,9 +251,7 @@ class LoadScopeScheduling(object):
scope, work_unit = self.workqueue.popitem(last=False)
# Keep track of the assigned work
assigned_to_node = self.assigned_work.setdefault(
node, default=OrderedDict()
)
assigned_to_node = self.assigned_work.setdefault(node, default=OrderedDict())
assigned_to_node[scope] = work_unit
# Ask the node to execute the workload
@@ -292,14 +286,11 @@ class LoadScopeScheduling(object):
example/loadsuite/test/test_delta.py::Delta1
example/loadsuite/epsilon/__init__.py
"""
return nodeid.rsplit('::', 1)[0]
return nodeid.rsplit("::", 1)[0]
def _pending_of(self, workload):
"""Return the number of pending tests in a workload."""
pending = sum(
list(scope.values()).count(False)
for scope in workload.values()
)
pending = sum(list(scope.values()).count(False) for scope in workload.values())
return pending
def _reschedule(self, node):
@@ -317,7 +308,7 @@ class LoadScopeScheduling(object):
if not self.workqueue:
return
self.log('Number of units waiting for node:', len(self.workqueue))
self.log("Number of units waiting for node:", len(self.workqueue))
# Check that the node is almost depleted of work
# 2: Heuristic of minimum tests to enqueue more work
@@ -348,13 +339,11 @@ class LoadScopeScheduling(object):
# Check that all nodes collected the same tests
if not self._check_nodes_have_same_collection():
self.log('**Different tests collected, aborting run**')
self.log("**Different tests collected, aborting run**")
return
# Collections are identical, create the final list of items
self.collection = list(
next(iter(self.registered_collections.values()))
)
self.collection = list(next(iter(self.registered_collections.values())))
if not self.collection:
return
@@ -368,12 +357,12 @@ class LoadScopeScheduling(object):
extra_nodes = len(self.nodes) - len(self.workqueue)
if extra_nodes > 0:
self.log('Shuting down {0} nodes'.format(extra_nodes))
self.log("Shuting down {0} nodes".format(extra_nodes))
for _ in range(extra_nodes):
unused_node, assigned = self.assigned_work.popitem(last=True)
self.log('Shuting down unused node {0}'.format(unused_node))
self.log("Shuting down unused node {0}".format(unused_node))
unused_node.shutdown()
# Assign initial workload
@@ -402,10 +391,7 @@ class LoadScopeScheduling(object):
for node, collection in node_collection_items[1:]:
msg = report_collection_diff(
col,
collection,
first_node.gateway.id,
node.gateway.id,
col, collection, first_node.gateway.id, node.gateway.id
)
if not msg:
continue
@@ -416,12 +402,7 @@ class LoadScopeScheduling(object):
if self.config is None:
continue
rep = CollectReport(
node.gateway.id,
'failed',
longrepr=msg,
result=[]
)
rep = CollectReport(node.gateway.id, "failed", longrepr=msg, result=[])
self.config.hook.pytest_collectreport(report=rep)
return same_collection

164
xdist/workermanage.py
View File

@@ -22,16 +22,17 @@ def parse_spec_config(config):
except ValueError:
xspeclist.append(xspec)
else:
xspeclist.extend([xspec[i + 1:]] * num)
xspeclist.extend([xspec[i + 1 :]] * num)
if not xspeclist:
raise pytest.UsageError(
"MISSING test execution (tx) nodes: please specify --tx")
"MISSING test execution (tx) nodes: please specify --tx"
)
return xspeclist
class NodeManager(object):
EXIT_TIMEOUT = 10
DEFAULT_IGNORES = ['.*', '*.pyc', '*.pyo', '*~']
DEFAULT_IGNORES = [".*", "*.pyc", "*.pyo", "*~"]
def __init__(self, config, specs=None, defaultchdir="pyexecnetcache"):
self.config = config
@@ -59,8 +60,7 @@ class NodeManager(object):
self.rsync(gateway, root, **self.rsyncoptions)
def setup_nodes(self, putevent):
self.config.hook.pytest_xdist_setupnodes(config=self.config,
specs=self.specs)
self.config.hook.pytest_xdist_setupnodes(config=self.config, specs=self.specs)
self.trace("setting up nodes")
nodes = []
for spec in self.specs:
@@ -72,7 +72,7 @@ class NodeManager(object):
self.config.hook.pytest_xdist_newgateway(gateway=gw)
self.rsync_roots(gw)
node = WorkerController(self, gw, self.config, putevent)
gw.node = node # keep the node alive
gw.node = node # keep the node alive
node.setup()
self.trace("started node %r" % node)
return node
@@ -91,6 +91,7 @@ class NodeManager(object):
return []
import pytest
import _pytest
pytestpath = pytest.__file__.rstrip("co")
pytestdir = py.path.local(_pytest.__file__).dirpath()
config = self.config
@@ -114,10 +115,7 @@ class NodeManager(object):
ignores += self.config.option.rsyncignore
ignores += self.config.getini("rsyncignore")
return {
'ignores': ignores,
'verbose': self.config.option.verbose,
}
return {"ignores": ignores, "verbose": self.config.option.verbose}
def rsync(self, gateway, source, notify=None, verbose=False, ignores=None):
"""Perform rsync to remote hosts for node."""
@@ -129,9 +127,12 @@ class NodeManager(object):
if spec.popen and not spec.chdir:
# XXX This assumes that sources are python-packages
# and that adding the basedir does not hurt.
gateway.remote_exec("""
gateway.remote_exec(
"""
import sys ; sys.path.insert(0, %r)
""" % os.path.dirname(str(source))).waitclose()
"""
% os.path.dirname(str(source))
).waitclose()
return
if (spec, source) in self._rsynced_specs:
return
@@ -139,28 +140,24 @@ class NodeManager(object):
def finished():
if notify:
notify("rsyncrootready", spec, source)
rsync.add_target_host(gateway, finished=finished)
self._rsynced_specs.add((spec, source))
self.config.hook.pytest_xdist_rsyncstart(
source=source,
gateways=[gateway],
)
self.config.hook.pytest_xdist_rsyncstart(source=source, gateways=[gateway])
rsync.send()
self.config.hook.pytest_xdist_rsyncfinish(
source=source,
gateways=[gateway],
)
self.config.hook.pytest_xdist_rsyncfinish(source=source, gateways=[gateway])
class HostRSync(execnet.RSync):
""" RSyncer that filters out common files
"""
def __init__(self, sourcedir, *args, **kwargs):
self._synced = {}
self._ignores = []
ignores = kwargs.pop('ignores', None) or []
ignores = kwargs.pop("ignores", None) or []
for x in ignores:
x = getattr(x, 'strpath', x)
x = getattr(x, "strpath", x)
self._ignores.append(re.compile(fnmatch.translate(x)))
super(HostRSync, self).__init__(sourcedir=sourcedir, **kwargs)
@@ -174,15 +171,15 @@ class HostRSync(execnet.RSync):
def add_target_host(self, gateway, finished=None):
remotepath = os.path.basename(self._sourcedir)
super(HostRSync, self).add_target(gateway, remotepath,
finishedcallback=finished,
delete=True,)
super(HostRSync, self).add_target(
gateway, remotepath, finishedcallback=finished, delete=True
)
def _report_send_file(self, gateway, modified_rel_path):
if self._verbose:
path = os.path.basename(self._sourcedir) + "/" + modified_rel_path
remotepath = gateway.spec.chdir
print('%s:%s <= %s' % (gateway.spec, remotepath, path))
print("%s:%s <= %s" % (gateway.spec, remotepath, path))
def make_reltoroot(roots, args):
@@ -211,11 +208,12 @@ class WorkerController(object):
self.putevent = putevent
self.gateway = gateway
self.config = config
self.workerinput = {'workerid': gateway.id,
'workercount': len(nodemanager.specs),
'slaveid': gateway.id,
'slavecount': len(nodemanager.specs)
}
self.workerinput = {
"workerid": gateway.id,
"workercount": len(nodemanager.specs),
"slaveid": gateway.id,
"slavecount": len(nodemanager.specs),
}
# TODO: deprecated name, backward compatibility only. Remove it in future
self.slaveinput = self.workerinput
self._down = False
@@ -225,7 +223,7 @@ class WorkerController(object):
py.log.setconsumer(self.log._keywords, None)
def __repr__(self):
return "<%s %s>" % (self.__class__.__name__, self.gateway.id,)
return "<%s %s>" % (self.__class__.__name__, self.gateway.id)
@property
def shutting_down(self):
@@ -240,24 +238,22 @@ class WorkerController(object):
option_dict = vars(self.config.option)
if spec.popen:
name = "popen-%s" % self.gateway.id
if hasattr(self.config, '_tmpdirhandler'):
if hasattr(self.config, "_tmpdirhandler"):
basetemp = self.config._tmpdirhandler.getbasetemp()
option_dict['basetemp'] = str(basetemp.join(name))
option_dict["basetemp"] = str(basetemp.join(name))
self.config.hook.pytest_configure_node(node=self)
self.channel = self.gateway.remote_exec(xdist.remote)
self.channel.send((self.workerinput, args, option_dict))
if self.putevent:
self.channel.setcallback(
self.process_from_remote,
endmarker=self.ENDMARK)
self.channel.setcallback(self.process_from_remote, endmarker=self.ENDMARK)
def ensure_teardown(self):
if hasattr(self, 'channel'):
if hasattr(self, "channel"):
if not self.channel.isclosed():
self.log("closing", self.channel)
self.channel.close()
# del self.channel
if hasattr(self, 'gateway'):
if hasattr(self, "gateway"):
self.log("exiting", self.gateway)
self.gateway.exit()
# del self.gateway
@@ -266,7 +262,7 @@ class WorkerController(object):
self.sendcommand("runtests", indices=indices)
def send_runtest_all(self):
self.sendcommand("runtests_all",)
self.sendcommand("runtests_all")
def shutdown(self):
if not self._down:
@@ -309,25 +305,28 @@ class WorkerController(object):
self.notify_inproc(eventname, node=self, **kwargs)
elif eventname == "workerfinished":
self._down = True
self.workeroutput = self.slaveoutput = kwargs['workeroutput']
self.workeroutput = self.slaveoutput = kwargs["workeroutput"]
self.notify_inproc("workerfinished", node=self)
elif eventname in ("logstart", "logfinish"):
self.notify_inproc(eventname, node=self, **kwargs)
elif eventname in (
"testreport", "collectreport", "teardownreport"):
elif eventname in ("testreport", "collectreport", "teardownreport"):
item_index = kwargs.pop("item_index", None)
rep = unserialize_report(eventname, kwargs['data'])
rep = unserialize_report(eventname, kwargs["data"])
if item_index is not None:
rep.item_index = item_index
self.notify_inproc(eventname, node=self, rep=rep)
elif eventname == "collectionfinish":
self.notify_inproc(eventname, node=self, ids=kwargs['ids'])
self.notify_inproc(eventname, node=self, ids=kwargs["ids"])
elif eventname == "runtest_protocol_complete":
self.notify_inproc(eventname, node=self, **kwargs)
elif eventname == "logwarning":
self.notify_inproc(eventname, message=kwargs['message'],
code=kwargs['code'], nodeid=kwargs['nodeid'],
fslocation=kwargs['nodeid'])
self.notify_inproc(
eventname,
message=kwargs["message"],
code=kwargs["code"],
nodeid=kwargs["nodeid"],
fslocation=kwargs["nodeid"],
)
else:
raise ValueError("unknown event: %s" % (eventname,))
except KeyboardInterrupt:
@@ -335,6 +334,7 @@ class WorkerController(object):
raise
except: # noqa
from _pytest._code import ExceptionInfo
excinfo = ExceptionInfo()
print("!" * 20, excinfo)
self.config.notify_exception(excinfo)
@@ -351,56 +351,56 @@ def unserialize_report(name, reportdict):
ReprFileLocation,
ReprFuncArgs,
ReprLocals,
ReprTraceback
ReprTraceback,
)
if reportdict['longrepr']:
if 'reprcrash' in reportdict['longrepr'] and 'reprtraceback' in reportdict['longrepr']:
reprtraceback = reportdict['longrepr']['reprtraceback']
reprcrash = reportdict['longrepr']['reprcrash']
if reportdict["longrepr"]:
if (
"reprcrash" in reportdict["longrepr"]
and "reprtraceback" in reportdict["longrepr"]
):
reprtraceback = reportdict["longrepr"]["reprtraceback"]
reprcrash = reportdict["longrepr"]["reprcrash"]
unserialized_entries = []
reprentry = None
for entry_data in reprtraceback['reprentries']:
data = entry_data['data']
entry_type = entry_data['type']
if entry_type == 'ReprEntry':
for entry_data in reprtraceback["reprentries"]:
data = entry_data["data"]
entry_type = entry_data["type"]
if entry_type == "ReprEntry":
reprfuncargs = None
reprfileloc = None
reprlocals = None
if data['reprfuncargs']:
reprfuncargs = ReprFuncArgs(
**data['reprfuncargs'])
if data['reprfileloc']:
reprfileloc = ReprFileLocation(
**data['reprfileloc'])
if data['reprlocals']:
reprlocals = ReprLocals(
data['reprlocals']['lines'])
if data["reprfuncargs"]:
reprfuncargs = ReprFuncArgs(**data["reprfuncargs"])
if data["reprfileloc"]:
reprfileloc = ReprFileLocation(**data["reprfileloc"])
if data["reprlocals"]:
reprlocals = ReprLocals(data["reprlocals"]["lines"])
reprentry = ReprEntry(
lines=data['lines'],
lines=data["lines"],
reprfuncargs=reprfuncargs,
reprlocals=reprlocals,
filelocrepr=reprfileloc,
style=data['style']
style=data["style"],
)
elif entry_type == 'ReprEntryNative':
reprentry = ReprEntryNative(data['lines'])
elif entry_type == "ReprEntryNative":
reprentry = ReprEntryNative(data["lines"])
else:
report_unserialization_failure(
entry_type, name, reportdict)
report_unserialization_failure(entry_type, name, reportdict)
unserialized_entries.append(reprentry)
reprtraceback['reprentries'] = unserialized_entries
reprtraceback["reprentries"] = unserialized_entries
exception_info = ReprExceptionInfo(
reprtraceback=ReprTraceback(**reprtraceback),
reprcrash=ReprFileLocation(**reprcrash),
)
for section in reportdict['longrepr']['sections']:
for section in reportdict["longrepr"]["sections"]:
exception_info.addsection(*section)
reportdict['longrepr'] = exception_info
reportdict["longrepr"] = exception_info
return reportdict
if name == "testreport":
@@ -411,13 +411,13 @@ def unserialize_report(name, reportdict):
def report_unserialization_failure(type_name, report_name, reportdict):
from pprint import pprint
url = 'https://github.com/pytest-dev/pytest-xdist/issues'
url = "https://github.com/pytest-dev/pytest-xdist/issues"
stream = py.io.TextIO()
pprint('-' * 100, stream=stream)
pprint('INTERNALERROR: Unknown entry type returned: %s' % type_name,
stream=stream)
pprint('report_name: %s' % report_name, stream=stream)
pprint("-" * 100, stream=stream)
pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream)
pprint("report_name: %s" % report_name, stream=stream)
pprint(reportdict, stream=stream)
pprint('Please report this bug at %s' % url, stream=stream)
pprint('-' * 100, stream=stream)
pprint("Please report this bug at %s" % url, stream=stream)
pprint("-" * 100, stream=stream)
assert 0, stream.getvalue()