Change benchmarks to use shared seed

benchmark/common.py

@@ -50,7 +50,7 @@ def do_benchmark(mode, *args, **kwargs):
         pass
     elif mode == "object":
         return pass_by_object(*args, **kwargs)
-    elif mode == "passByBytes":
+    elif mode == "bytes":
         return pass_by_bytes(*args, **kwargs)
     else:
         raise ValueError("Unknown mode: " + str(mode))

benchmark/run_all.py

@@ -4,7 +4,9 @@ from subprocess import Popen, PIPE
 import sys
 import os
 import json
+import random
 
+seed = int(random.random()*10**6)
 def run_one(type, name, mode, iters, faster):
     args = []
     res_type = type
@@ -16,7 +18,7 @@ def run_one(type, name, mode, iters, faster):
             args += ['-c', 'packed']
             res_type = 'pycapnp_packed'
 
-    p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters)] + args, stdout=PIPE, stderr=PIPE)
+    p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters), '--seed', str(seed)] + args, stdout=PIPE, stderr=PIPE)
     res = p.communicate()[1]
 
     data = {}
@@ -35,17 +37,50 @@ def run_one(type, name, mode, iters, faster):
         del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
     return data
 
+def run_cpp(name, mode, iters, faster, compression):
+    res_type = 'capnp'
+    reuse = 'no-reuse'
+    if faster:
+        reuse = 'reuse'
+        res_type += '_reuse'
+    if compression != 'none':
+        res_type += '_' + compression
+
+    p = Popen(["time", "-p", "capnproto-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
+    res = p.communicate()[1]
+
+    data = {}
+    res = res.strip()
+
+    for line in res.split('\n'):
+        vals = line.split()
+        data[vals[0]] = float(vals[1])
+
+    data['type'] = res_type
+    data['mode'] = mode
+    data['name'] = name
+    data['iters'] = iters
+    return data
+
+
 def run_each(name, iters):
     ret = []
 
+    ret.append(run_cpp(name, 'object', iters, False, 'none'))
+    ret.append(run_cpp(name, 'object', iters, True, 'none'))
+    ret.append(run_cpp(name, 'bytes', iters, False, 'none'))
+    ret.append(run_cpp(name, 'bytes', iters, False, 'packed'))
+    ret.append(run_cpp(name, 'bytes', iters, True, 'none'))
+    ret.append(run_cpp(name, 'bytes', iters, True, 'packed'))
+
     ret.append(run_one('pycapnp', name, 'object', iters, False))
     ret.append(run_one('proto', name, 'object', iters, False))
     ret.append(run_one('proto', name, 'object', iters, True))
 
-    ret.append(run_one('pycapnp', name, 'passByBytes', iters, False))
+    ret.append(run_one('pycapnp', name, 'bytes', iters, False))
-    ret.append(run_one('pycapnp', name, 'passByBytes', iters, True))
+    ret.append(run_one('pycapnp', name, 'bytes', iters, True))
-    ret.append(run_one('proto', name, 'passByBytes', iters, False))
+    ret.append(run_one('proto', name, 'bytes', iters, False))
-    ret.append(run_one('proto', name, 'passByBytes', iters, True))
+    ret.append(run_one('proto', name, 'bytes', iters, True))
 
     return ret
 

benchmark/runner.py

@@ -6,6 +6,7 @@ from common import do_benchmark
 from timeit import default_timer
 import os
 import sys
+import random
 
 def parse_args():
     parser = argparse.ArgumentParser()
@@ -16,6 +17,7 @@ def parse_args():
     parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int)
     parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true')
     parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
+    parser.add_argument("-S", "--seed", help="Seed to use for random", default=0, type=int)
 
     return parser.parse_args()
 
@@ -38,6 +40,7 @@ def run_test(args):
 def main():
     args = parse_args()
     sys.path.append(args.includes)
+    random.seed(args.seed)
     run_test(args)
 
 if __name__ == '__main__':