Refactored benchmarks to conform to cpp spec

benchmark/bin/pycapnp-carsales

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='carsales', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pycapnp-catrank

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='catrank', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pycapnp-eval

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='eval', suffix='pycapnp', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-carsales

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='carsales', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-catrank

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='catrank', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto-eval

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+from runner import parse_args_simple, run_test
+
+def main():
+    args = parse_args_simple()
+    run_test(name='eval', suffix='proto', **vars(args))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/pyproto_cpp-carsales

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-carsales $@

benchmark/bin/pyproto_cpp-catrank

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-catrank $@

benchmark/bin/pyproto_cpp-eval

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
+pyproto-eval $@

benchmark/bin/run_all.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+from subprocess import Popen, PIPE
+import sys
+import os
+import json
+import argparse
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-l', "--langs", help="Add languages to test with the form: pycapnp:pycapnp", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp'])
+    parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true',)
+    parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true')
+    parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0)
+
+    return parser.parse_args()
+
+def run_cpp(prefix, name, mode, iters, faster, compression):
+    res_type = prefix
+    reuse = 'no-reuse'
+
+    if faster:
+        reuse = 'reuse'
+        res_type += '_reuse'
+    if compression != 'none':
+        res_type += '_' + compression
+
+    p = Popen(["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
+    res = p.communicate()[1]
+
+    data = {}
+    res = res.strip()
+
+    for line in res.split('\n'):
+        vals = line.split()
+        data[vals[0]] = float(vals[1])
+
+    data['type'] = res_type
+    data['mode'] = mode
+    data['name'] = name
+    data['iters'] = iters
+
+    return data
+
+
+def run_each(name, langs, reuse, compression, iters):
+    ret = []
+
+    for lang_name in langs:
+        ret.append(run_cpp(lang_name, name, 'object', iters, False, 'none'))
+        ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'none'))
+        if reuse:
+            ret.append(run_cpp(lang_name, name, 'object', iters, True, 'none'))
+            ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'none'))
+            if compression:
+                ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'packed'))
+        if compression:
+            ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'packed'))
+
+    return ret
+
+def main():
+    args = parse_args()
+
+    del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
+    os.environ['PATH'] += ':.'
+
+    data = []
+    data += run_each('carsales', args.langs, args.reuse, args.compression, int(2000 * args.scale_iters))
+    data += run_each('catrank', args.langs, args.reuse, args.compression, int(100 * args.scale_iters))
+    data += run_each('eval', args.langs, args.reuse, args.compression, int(10000 * args.scale_iters))
+    json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': '))
+
+if __name__ == '__main__':
+    main()

benchmark/bin/runner.py

@@ -1,13 +1,26 @@
 #!/usr/bin/env python
 
 import argparse
-from importlib import import_module
-from common import do_benchmark
-from timeit import default_timer
-import os
 import sys
+import os
+from importlib import import_module
+from timeit import default_timer
 import random
 
+_this_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(_this_dir, '..'))
+from common import do_benchmark
+
+def parse_args_simple():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("mode", help="Mode to use for serialization, ie. object or bytes")
+    parser.add_argument("reuse", help="Currently ignored")
+    parser.add_argument("compression", help="Valid values are none or packed")
+    parser.add_argument("iters", help="Number of iterations to run for", type=int)
+    parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
+
+    return parser.parse_args()
+
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument("name", help="Name of the benchmark to run, eg. carsales", nargs='?', default='carsales')
@@ -17,31 +30,24 @@ def parse_args():
     parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int)
     parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true')
     parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
-    parser.add_argument("-S", "--seed", help="Seed to use for random", default=0, type=int)
 
     return parser.parse_args()
 
-def run_test(args):
+def run_test(name, mode, reuse, compression, iters, suffix, includes):
     tic = default_timer()
 
-    name = args.name
+    name = name
-    module = import_module(name + '_' + args.suffix)
+    sys.path.append(includes)
-    benchmark = module.Benchmark(compression=args.compression)
+    module = import_module(name + '_' + suffix)
+    benchmark = module.Benchmark(compression=compression)
 
-    if args.iters is None:
+    do_benchmark(mode=mode, benchmark=benchmark, iters=iters, reuse=reuse)
-        iters = ITERATIONS[name]
-    else:
-        iters = args.iters
-
-    do_benchmark(mode=args.mode, benchmark=benchmark, iters=iters, reuse=args.reuse)
     toc = default_timer()
     return toc - tic
 
 def main():
     args = parse_args()
-    sys.path.append(args.includes)
+    run_test(**vars(args))
-    random.seed(args.seed)
-    run_test(args)
 
 if __name__ == '__main__':
     main()

benchmark/common.py

@@ -1,17 +1,12 @@
 from random import random
+import pyximport
+importers = pyximport.install()
+from common_fast import rand_int, rand_double, rand_bool 
+pyximport.uninstall(*importers)
 
 WORDS = ["foo ", "bar ", "baz ", "qux ", "quux ", "corge ", "grault ", "garply ", "waldo ", "fred ",
     "plugh ", "xyzzy ", "thud "]
 
-def rand_int(val):
-    return int(random() * val)
-
-def rand_double(val):
-    return random() * val
-
-def rand_bool():
-    return random() < .5
-
 def from_bytes_helper(klass):
     def helper(text):
         obj = klass()

benchmark/run_all.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python
-
-from subprocess import Popen, PIPE
-import sys
-import os
-import json
-import random
-
-seed = int(random.random()*10**6)
-def run_one(type, name, mode, iters, faster):
-    args = []
-    res_type = type
-    if faster:
-        if type == 'proto':
-            os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
-            res_type = 'proto_cpp'
-        if type == 'pycapnp':
-            args += ['-c', 'packed']
-            res_type = 'pycapnp_packed'
-
-    p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters), '--seed', str(seed)] + args, stdout=PIPE, stderr=PIPE)
-    res = p.communicate()[1]
-
-    data = {}
-    res = res.strip()
-
-    for line in res.split('\n'):
-        vals = line.split()
-        data[vals[0]] = float(vals[1])
-
-    data['type'] = res_type
-    data['mode'] = mode
-    data['name'] = name
-    data['iters'] = iters
-
-    if 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION' in os.environ:
-        del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
-    return data
-
-def run_cpp(name, mode, iters, faster, compression):
-    res_type = 'capnp'
-    reuse = 'no-reuse'
-    if faster:
-        reuse = 'reuse'
-        res_type += '_reuse'
-    if compression != 'none':
-        res_type += '_' + compression
-
-    p = Popen(["time", "-p", "capnproto-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
-    res = p.communicate()[1]
-
-    data = {}
-    res = res.strip()
-
-    for line in res.split('\n'):
-        vals = line.split()
-        data[vals[0]] = float(vals[1])
-
-    data['type'] = res_type
-    data['mode'] = mode
-    data['name'] = name
-    data['iters'] = iters
-    return data
-
-
-def run_each(name, iters):
-    ret = []
-
-    ret.append(run_cpp(name, 'object', iters, False, 'none'))
-    ret.append(run_cpp(name, 'object', iters, True, 'none'))
-    ret.append(run_cpp(name, 'bytes', iters, False, 'none'))
-    ret.append(run_cpp(name, 'bytes', iters, False, 'packed'))
-    ret.append(run_cpp(name, 'bytes', iters, True, 'none'))
-    ret.append(run_cpp(name, 'bytes', iters, True, 'packed'))
-
-    ret.append(run_one('pycapnp', name, 'object', iters, False))
-    ret.append(run_one('proto', name, 'object', iters, False))
-    ret.append(run_one('proto', name, 'object', iters, True))
-
-    ret.append(run_one('pycapnp', name, 'bytes', iters, False))
-    ret.append(run_one('pycapnp', name, 'bytes', iters, True))
-    ret.append(run_one('proto', name, 'bytes', iters, False))
-    ret.append(run_one('proto', name, 'bytes', iters, True))
-
-    return ret
-
-def main():
-    data = []
-    data += run_each('carsales', 2000)
-    data += run_each('catrank', 100)
-    data += run_each('eval', 10000)
-    json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': '))
-
-if __name__ == '__main__':
-    main()