From 895f74547c2c340341f7c41b0667601a6d1e2f7b Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 18 Nov 2013 13:05:56 -0800 Subject: [PATCH] Refactored benchmarks to conform to cpp spec --- benchmark/bin/pycapnp-carsales | 10 ++++ benchmark/bin/pycapnp-catrank | 10 ++++ benchmark/bin/pycapnp-eval | 10 ++++ benchmark/bin/pyproto-carsales | 10 ++++ benchmark/bin/pyproto-catrank | 10 ++++ benchmark/bin/pyproto-eval | 10 ++++ benchmark/bin/pyproto_cpp-carsales | 4 ++ benchmark/bin/pyproto_cpp-catrank | 4 ++ benchmark/bin/pyproto_cpp-eval | 4 ++ benchmark/bin/run_all.py | 75 +++++++++++++++++++++++ benchmark/{ => bin}/runner.py | 42 +++++++------ benchmark/common.py | 13 ++-- benchmark/run_all.py | 95 ------------------------------ 13 files changed, 175 insertions(+), 122 deletions(-) create mode 100755 benchmark/bin/pycapnp-carsales create mode 100755 benchmark/bin/pycapnp-catrank create mode 100755 benchmark/bin/pycapnp-eval create mode 100755 benchmark/bin/pyproto-carsales create mode 100755 benchmark/bin/pyproto-catrank create mode 100755 benchmark/bin/pyproto-eval create mode 100755 benchmark/bin/pyproto_cpp-carsales create mode 100755 benchmark/bin/pyproto_cpp-catrank create mode 100755 benchmark/bin/pyproto_cpp-eval create mode 100755 benchmark/bin/run_all.py rename benchmark/{ => bin}/runner.py (55%) delete mode 100755 benchmark/run_all.py diff --git a/benchmark/bin/pycapnp-carsales b/benchmark/bin/pycapnp-carsales new file mode 100755 index 0000000..670720b --- /dev/null +++ b/benchmark/bin/pycapnp-carsales @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='carsales', suffix='pycapnp', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pycapnp-catrank b/benchmark/bin/pycapnp-catrank new file mode 100755 index 0000000..a11cb27 --- /dev/null +++ b/benchmark/bin/pycapnp-catrank @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='catrank', suffix='pycapnp', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pycapnp-eval b/benchmark/bin/pycapnp-eval new file mode 100755 index 0000000..8bb759a --- /dev/null +++ b/benchmark/bin/pycapnp-eval @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='eval', suffix='pycapnp', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-carsales b/benchmark/bin/pyproto-carsales new file mode 100755 index 0000000..5fb8e54 --- /dev/null +++ b/benchmark/bin/pyproto-carsales @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='carsales', suffix='proto', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-catrank b/benchmark/bin/pyproto-catrank new file mode 100755 index 0000000..e82456c --- /dev/null +++ b/benchmark/bin/pyproto-catrank @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='catrank', suffix='proto', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-eval b/benchmark/bin/pyproto-eval new file mode 100755 index 0000000..270b615 --- /dev/null +++ b/benchmark/bin/pyproto-eval @@ -0,0 +1,10 @@ +#!/usr/bin/env python + +from runner import parse_args_simple, run_test + +def main(): + args = parse_args_simple() + run_test(name='eval', suffix='proto', **vars(args)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/benchmark/bin/pyproto_cpp-carsales b/benchmark/bin/pyproto_cpp-carsales new file mode 100755 index 0000000..2c068fb --- /dev/null +++ b/benchmark/bin/pyproto_cpp-carsales @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp +pyproto-carsales $@ diff --git a/benchmark/bin/pyproto_cpp-catrank b/benchmark/bin/pyproto_cpp-catrank new file mode 100755 index 0000000..eedf594 --- /dev/null +++ b/benchmark/bin/pyproto_cpp-catrank @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp +pyproto-catrank $@ diff --git a/benchmark/bin/pyproto_cpp-eval b/benchmark/bin/pyproto_cpp-eval new file mode 100755 index 0000000..5a81d9b --- /dev/null +++ b/benchmark/bin/pyproto_cpp-eval @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp +pyproto-eval $@ diff --git a/benchmark/bin/run_all.py b/benchmark/bin/run_all.py new file mode 100755 index 0000000..1113766 --- /dev/null +++ b/benchmark/bin/run_all.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python + +from subprocess import Popen, PIPE +import sys +import os +import json +import argparse + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-l', "--langs", help="Add languages to test with the form: pycapnp:pycapnp", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp']) + parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true',) + parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true') + parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0) + + return parser.parse_args() + +def run_cpp(prefix, name, mode, iters, faster, compression): + res_type = prefix + reuse = 'no-reuse' + + if faster: + reuse = 'reuse' + res_type += '_reuse' + if compression != 'none': + res_type += '_' + compression + + p = Popen(["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE) + res = p.communicate()[1] + + data = {} + res = res.strip() + + for line in res.split('\n'): + vals = line.split() + data[vals[0]] = float(vals[1]) + + data['type'] = res_type + data['mode'] = mode + data['name'] = name + data['iters'] = iters + + return data + + +def run_each(name, langs, reuse, compression, iters): + ret = [] + + for lang_name in langs: + ret.append(run_cpp(lang_name, name, 'object', iters, False, 'none')) + ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'none')) + if reuse: + ret.append(run_cpp(lang_name, name, 'object', iters, True, 'none')) + ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'none')) + if compression: + ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'packed')) + if compression: + ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'packed')) + + return ret + +def main(): + args = parse_args() + + del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] + os.environ['PATH'] += ':.' + + data = [] + data += run_each('carsales', args.langs, args.reuse, args.compression, int(2000 * args.scale_iters)) + data += run_each('catrank', args.langs, args.reuse, args.compression, int(100 * args.scale_iters)) + data += run_each('eval', args.langs, args.reuse, args.compression, int(10000 * args.scale_iters)) + json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': ')) + +if __name__ == '__main__': + main() diff --git a/benchmark/runner.py b/benchmark/bin/runner.py similarity index 55% rename from benchmark/runner.py rename to benchmark/bin/runner.py index 09dc908..1627bd8 100755 --- a/benchmark/runner.py +++ b/benchmark/bin/runner.py @@ -1,13 +1,26 @@ #!/usr/bin/env python import argparse -from importlib import import_module -from common import do_benchmark -from timeit import default_timer -import os import sys +import os +from importlib import import_module +from timeit import default_timer import random +_this_dir = os.path.dirname(__file__) +sys.path.append(os.path.join(_this_dir, '..')) +from common import do_benchmark + +def parse_args_simple(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", help="Mode to use for serialization, ie. object or bytes") + parser.add_argument("reuse", help="Currently ignored") + parser.add_argument("compression", help="Valid values are none or packed") + parser.add_argument("iters", help="Number of iterations to run for", type=int) + parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include') + + return parser.parse_args() + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("name", help="Name of the benchmark to run, eg. carsales", nargs='?', default='carsales') @@ -17,31 +30,24 @@ def parse_args(): parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int) parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true') parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include') - parser.add_argument("-S", "--seed", help="Seed to use for random", default=0, type=int) return parser.parse_args() -def run_test(args): +def run_test(name, mode, reuse, compression, iters, suffix, includes): tic = default_timer() - name = args.name - module = import_module(name + '_' + args.suffix) - benchmark = module.Benchmark(compression=args.compression) + name = name + sys.path.append(includes) + module = import_module(name + '_' + suffix) + benchmark = module.Benchmark(compression=compression) - if args.iters is None: - iters = ITERATIONS[name] - else: - iters = args.iters - - do_benchmark(mode=args.mode, benchmark=benchmark, iters=iters, reuse=args.reuse) + do_benchmark(mode=mode, benchmark=benchmark, iters=iters, reuse=reuse) toc = default_timer() return toc - tic def main(): args = parse_args() - sys.path.append(args.includes) - random.seed(args.seed) - run_test(args) + run_test(**vars(args)) if __name__ == '__main__': main() \ No newline at end of file diff --git a/benchmark/common.py b/benchmark/common.py index be68b38..00d839f 100644 --- a/benchmark/common.py +++ b/benchmark/common.py @@ -1,17 +1,12 @@ from random import random +import pyximport +importers = pyximport.install() +from common_fast import rand_int, rand_double, rand_bool +pyximport.uninstall(*importers) WORDS = ["foo ", "bar ", "baz ", "qux ", "quux ", "corge ", "grault ", "garply ", "waldo ", "fred ", "plugh ", "xyzzy ", "thud "] -def rand_int(val): - return int(random() * val) - -def rand_double(val): - return random() * val - -def rand_bool(): - return random() < .5 - def from_bytes_helper(klass): def helper(text): obj = klass() diff --git a/benchmark/run_all.py b/benchmark/run_all.py deleted file mode 100755 index 9890de8..0000000 --- a/benchmark/run_all.py +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env python - -from subprocess import Popen, PIPE -import sys -import os -import json -import random - -seed = int(random.random()*10**6) -def run_one(type, name, mode, iters, faster): - args = [] - res_type = type - if faster: - if type == 'proto': - os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' - res_type = 'proto_cpp' - if type == 'pycapnp': - args += ['-c', 'packed'] - res_type = 'pycapnp_packed' - - p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters), '--seed', str(seed)] + args, stdout=PIPE, stderr=PIPE) - res = p.communicate()[1] - - data = {} - res = res.strip() - - for line in res.split('\n'): - vals = line.split() - data[vals[0]] = float(vals[1]) - - data['type'] = res_type - data['mode'] = mode - data['name'] = name - data['iters'] = iters - - if 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION' in os.environ: - del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] - return data - -def run_cpp(name, mode, iters, faster, compression): - res_type = 'capnp' - reuse = 'no-reuse' - if faster: - reuse = 'reuse' - res_type += '_reuse' - if compression != 'none': - res_type += '_' + compression - - p = Popen(["time", "-p", "capnproto-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE) - res = p.communicate()[1] - - data = {} - res = res.strip() - - for line in res.split('\n'): - vals = line.split() - data[vals[0]] = float(vals[1]) - - data['type'] = res_type - data['mode'] = mode - data['name'] = name - data['iters'] = iters - return data - - -def run_each(name, iters): - ret = [] - - ret.append(run_cpp(name, 'object', iters, False, 'none')) - ret.append(run_cpp(name, 'object', iters, True, 'none')) - ret.append(run_cpp(name, 'bytes', iters, False, 'none')) - ret.append(run_cpp(name, 'bytes', iters, False, 'packed')) - ret.append(run_cpp(name, 'bytes', iters, True, 'none')) - ret.append(run_cpp(name, 'bytes', iters, True, 'packed')) - - ret.append(run_one('pycapnp', name, 'object', iters, False)) - ret.append(run_one('proto', name, 'object', iters, False)) - ret.append(run_one('proto', name, 'object', iters, True)) - - ret.append(run_one('pycapnp', name, 'bytes', iters, False)) - ret.append(run_one('pycapnp', name, 'bytes', iters, True)) - ret.append(run_one('proto', name, 'bytes', iters, False)) - ret.append(run_one('proto', name, 'bytes', iters, True)) - - return ret - -def main(): - data = [] - data += run_each('carsales', 2000) - data += run_each('catrank', 100) - data += run_each('eval', 10000) - json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': ')) - -if __name__ == '__main__': - main()