IQ.Lvbs/pycapnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored benchmarks to conform to cpp spec

Browse Source
This commit is contained in:
Jason Paryani
2013-11-18 13:05:56 -08:00
parent 42da9f4101
commit 895f74547c
13 changed files with 175 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
benchmark/bin/pycapnp-carsales Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='carsales', suffix='pycapnp', **vars(args))
if __name__ == '__main__':
main()

10
benchmark/bin/pycapnp-catrank Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='catrank', suffix='pycapnp', **vars(args))
if __name__ == '__main__':
main()

10
benchmark/bin/pycapnp-eval Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='eval', suffix='pycapnp', **vars(args))
if __name__ == '__main__':
main()

10
benchmark/bin/pyproto-carsales Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='carsales', suffix='proto', **vars(args))
if __name__ == '__main__':
main()

10
benchmark/bin/pyproto-catrank Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='catrank', suffix='proto', **vars(args))
if __name__ == '__main__':
main()

10
benchmark/bin/pyproto-eval Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env python
from runner import parse_args_simple, run_test
def main():
args = parse_args_simple()
run_test(name='eval', suffix='proto', **vars(args))
if __name__ == '__main__':
main()

4
benchmark/bin/pyproto_cpp-carsales Executable file
View File

@@ -0,0 +1,4 @@
#!/usr/bin/env bash
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
pyproto-carsales $@

4
benchmark/bin/pyproto_cpp-catrank Executable file
View File

@@ -0,0 +1,4 @@
#!/usr/bin/env bash
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
pyproto-catrank $@

4
benchmark/bin/pyproto_cpp-eval Executable file
View File

@@ -0,0 +1,4 @@
#!/usr/bin/env bash
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
pyproto-eval $@

75
benchmark/bin/run_all.py Executable file
View File

@@ -0,0 +1,75 @@
#!/usr/bin/env python
from subprocess import Popen, PIPE
import sys
import os
import json
import argparse
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument('-l', "--langs", help="Add languages to test with the form: pycapnp:pycapnp", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp'])
parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true',)
parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true')
parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0)
return parser.parse_args()
def run_cpp(prefix, name, mode, iters, faster, compression):
res_type = prefix
reuse = 'no-reuse'
if faster:
reuse = 'reuse'
res_type += '_reuse'
if compression != 'none':
res_type += '_' + compression
p = Popen(["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
res = p.communicate()[1]
data = {}
res = res.strip()
for line in res.split('\n'):
vals = line.split()
data[vals[0]] = float(vals[1])
data['type'] = res_type
data['mode'] = mode
data['name'] = name
data['iters'] = iters
return data
def run_each(name, langs, reuse, compression, iters):
ret = []
for lang_name in langs:
ret.append(run_cpp(lang_name, name, 'object', iters, False, 'none'))
ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'none'))
if reuse:
ret.append(run_cpp(lang_name, name, 'object', iters, True, 'none'))
ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'none'))
if compression:
ret.append(run_cpp(lang_name, name, 'bytes', iters, True, 'packed'))
if compression:
ret.append(run_cpp(lang_name, name, 'bytes', iters, False, 'packed'))
return ret
def main():
args = parse_args()
del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
os.environ['PATH'] += ':.'
data = []
data += run_each('carsales', args.langs, args.reuse, args.compression, int(2000 * args.scale_iters))
data += run_each('catrank', args.langs, args.reuse, args.compression, int(100 * args.scale_iters))
data += run_each('eval', args.langs, args.reuse, args.compression, int(10000 * args.scale_iters))
json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': '))
if __name__ == '__main__':
main()

42
benchmark/runner.py → benchmark/bin/runner.py
View File

@@ -1,13 +1,26 @@
#!/usr/bin/env python
import argparse
from importlib import import_module
from common import do_benchmark
from timeit import default_timer
import os
import sys
import os
from importlib import import_module
from timeit import default_timer
import random
_this_dir = os.path.dirname(__file__)
sys.path.append(os.path.join(_this_dir, '..'))
from common import do_benchmark
def parse_args_simple():
parser = argparse.ArgumentParser()
parser.add_argument("mode", help="Mode to use for serialization, ie. object or bytes")
parser.add_argument("reuse", help="Currently ignored")
parser.add_argument("compression", help="Valid values are none or packed")
parser.add_argument("iters", help="Number of iterations to run for", type=int)
parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
return parser.parse_args()
def parse_args():
parser = argparse.ArgumentParser()
parser.add_argument("name", help="Name of the benchmark to run, eg. carsales", nargs='?', default='carsales')
@@ -17,31 +30,24 @@ def parse_args():
parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int)
parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true')
parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include')
parser.add_argument("-S", "--seed", help="Seed to use for random", default=0, type=int)
return parser.parse_args()
def run_test(args):
def run_test(name, mode, reuse, compression, iters, suffix, includes):
tic = default_timer()
name = args.name
module = import_module(name + '_' + args.suffix)
benchmark = module.Benchmark(compression=args.compression)
name = name
sys.path.append(includes)
module = import_module(name + '_' + suffix)
benchmark = module.Benchmark(compression=compression)
if args.iters is None:
iters = ITERATIONS[name]
else:
iters = args.iters
do_benchmark(mode=args.mode, benchmark=benchmark, iters=iters, reuse=args.reuse)
do_benchmark(mode=mode, benchmark=benchmark, iters=iters, reuse=reuse)
toc = default_timer()
return toc - tic
def main():
args = parse_args()
sys.path.append(args.includes)
random.seed(args.seed)
run_test(args)
run_test(**vars(args))
if __name__ == '__main__':
main()

13
benchmark/common.py
View File

@@ -1,17 +1,12 @@
from random import random
import pyximport
importers = pyximport.install()
from common_fast import rand_int, rand_double, rand_bool
pyximport.uninstall(*importers)
WORDS = ["foo ", "bar ", "baz ", "qux ", "quux ", "corge ", "grault ", "garply ", "waldo ", "fred ",
"plugh ", "xyzzy ", "thud "]
def rand_int(val):
return int(random() * val)
def rand_double(val):
return random() * val
def rand_bool():
return random() < .5
def from_bytes_helper(klass):
def helper(text):
obj = klass()

95
benchmark/run_all.py
View File

@@ -1,95 +0,0 @@
#!/usr/bin/env python
from subprocess import Popen, PIPE
import sys
import os
import json
import random
seed = int(random.random()*10**6)
def run_one(type, name, mode, iters, faster):
args = []
res_type = type
if faster:
if type == 'proto':
os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
res_type = 'proto_cpp'
if type == 'pycapnp':
args += ['-c', 'packed']
res_type = 'pycapnp_packed'
p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters), '--seed', str(seed)] + args, stdout=PIPE, stderr=PIPE)
res = p.communicate()[1]
data = {}
res = res.strip()
for line in res.split('\n'):
vals = line.split()
data[vals[0]] = float(vals[1])
data['type'] = res_type
data['mode'] = mode
data['name'] = name
data['iters'] = iters
if 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION' in os.environ:
del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION']
return data
def run_cpp(name, mode, iters, faster, compression):
res_type = 'capnp'
reuse = 'no-reuse'
if faster:
reuse = 'reuse'
res_type += '_reuse'
if compression != 'none':
res_type += '_' + compression
p = Popen(["time", "-p", "capnproto-"+name, mode, reuse, compression, str(iters)], stdout=PIPE, stderr=PIPE)
res = p.communicate()[1]
data = {}
res = res.strip()
for line in res.split('\n'):
vals = line.split()
data[vals[0]] = float(vals[1])
data['type'] = res_type
data['mode'] = mode
data['name'] = name
data['iters'] = iters
return data
def run_each(name, iters):
ret = []
ret.append(run_cpp(name, 'object', iters, False, 'none'))
ret.append(run_cpp(name, 'object', iters, True, 'none'))
ret.append(run_cpp(name, 'bytes', iters, False, 'none'))
ret.append(run_cpp(name, 'bytes', iters, False, 'packed'))
ret.append(run_cpp(name, 'bytes', iters, True, 'none'))
ret.append(run_cpp(name, 'bytes', iters, True, 'packed'))
ret.append(run_one('pycapnp', name, 'object', iters, False))
ret.append(run_one('proto', name, 'object', iters, False))
ret.append(run_one('proto', name, 'object', iters, True))
ret.append(run_one('pycapnp', name, 'bytes', iters, False))
ret.append(run_one('pycapnp', name, 'bytes', iters, True))
ret.append(run_one('proto', name, 'bytes', iters, False))
ret.append(run_one('proto', name, 'bytes', iters, True))
return ret
def main():
data = []
data += run_each('carsales', 2000)
data += run_each('catrank', 100)
data += run_each('eval', 10000)
json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': '))
if __name__ == '__main__':
main()