Refactored benchmarks to conform to cpp spec

benchmark/bin/pycapnp-carsales

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='carsales', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pycapnp-catrank

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='catrank', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pycapnp-eval

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='eval', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-carsales

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='carsales', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-catrank

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='catrank', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-eval

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='eval', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto_cpp-carsales

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-carsales $@

benchmark/bin/pyproto_cpp-catrank

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-catrank $@

benchmark/bin/pyproto_cpp-eval

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-eval $@

benchmark/bin/run_all.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+from subprocess import Popen, PIPE
+import sys
+import os
+import json
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', "--langs", help="Add languages to test with the form: pycapnp:pycapnp", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp'])
+    parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true',)
+    parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true')
+    parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0)
+
+    return parser.parse_args()
+
+def run_cpp(prefix, name, mode, iters, faster, compression):
+    res_type = prefix
+    reuse = 'no-reuse'
+
+    if faster:
+        reuse = 'reuse'
+        res_type += '_reuse'
+    if compression != 'none':
+        res_type += '_' + compression
+
+    p = Popen(["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
+    res = p.communicate()[1]
+
+    data = {}
+    res = res.strip()
+
+    for line in res.split('\n'):
+        vals = line.split()
+        data[vals[0]] = float(vals[1])
+
+    data['type'] = res_type
+    data['mode'] = mode
+    data['name'] = name
+    data['iters'] = iters
+
+    return data
+
+
+def run_each(name, langs, reuse, compression, iters):
+    ret = []
+
+    for lang_name in langs:
+        ret.append(run_cpp(lang_name, name, 'object', iters, False, 'none'))
+        ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'none'))
+        if reuse:
+            ret.append(run_cpp(lang_name, name, 'object', iters, True, 'none'))
+            ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'none'))
+            if compression:
+                ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'packed'))
+        if compression:
+            ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'packed'))
+
+    return ret
+
+def main():
+    args = parse_args()
+
+    del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
+    os.environ['PATH'] += ':.'
+
+    data = []
+    data += run_each('carsales', args.langs, args.reuse, args.compression, int(2000 * args.scale_iters))
+    data += run_each('catrank', args.langs, args.reuse, args.compression, int(100 * args.scale_iters))
+    data += run_each('eval', args.langs, args.reuse, args.compression, int(10000 * args.scale_iters))
+    json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': '))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/runner.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import argparse
+import sys
+import os
+from importlib import import_module
+from timeit import default_timer
+import random
+
+_this_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(_this_dir, '..'))
+from common import do_benchmark
+
+def parse_args_simple():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("mode", help="Mode to use for serialization, ie. object or bytes")
+    parser.add_argument("reuse", help="Currently ignored")
+    parser.add_argument("compression", help="Valid values are none or packed")
+    parser.add_argument("iters", help="Number of iterations to run for", type=int)
+    parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
+
+    return parser.parse_args()
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("name", help="Name of the benchmark to run, eg. carsales", nargs='?', default='carsales')
+    parser.add_argument("-c", "--compression", help="Specify the compression type", default=None)
+    parser.add_argument("-s", "--suffix", help="Choose the protocol type.", default='pycapnp')
+    parser.add_argument("-m", "--mode", help="Specify the mode", default='object')
+    parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int)
+    parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true')
+    parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
+
+    return parser.parse_args()
+
+def run_test(name, mode, reuse, compression, iters, suffix, includes):
+    tic = default_timer()
+
+    name = name
+    sys.path.append(includes)
+    module = import_module(name + '_' + suffix)
+    benchmark = module.Benchmark(compression=compression)
+
+    do_benchmark(mode=mode, benchmark=benchmark, iters=iters, reuse=reuse)
+    toc = default_timer()
+    return toc - tic
+
+def main():
+    args = parse_args()
+    run_test(**vars(args))
+
+if __name__ == '__main__':
+    main()