From 52dde9ab22328d0131935f93f0bf542ee59761c4 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 17 Nov 2013 23:06:25 -0800 Subject: [PATCH] Add benchmarks --- benchmark/carsales.capnp | 82 +++++++ benchmark/carsales.proto | 79 +++++++ benchmark/carsales_pb2.py | 422 ++++++++++++++++++++++++++++++++++ benchmark/carsales_proto.py | 97 ++++++++ benchmark/carsales_pycapnp.py | 101 ++++++++ benchmark/catrank.capnp | 37 +++ benchmark/catrank.proto | 34 +++ benchmark/catrank_pb2.py | 107 +++++++++ benchmark/catrank_proto.py | 67 ++++++ benchmark/catrank_pycapnp.py | 70 ++++++ benchmark/common.py | 77 +++++++ benchmark/eval.capnp | 53 +++++ benchmark/eval.proto | 46 ++++ benchmark/eval_pb2.py | 163 +++++++++++++ benchmark/eval_proto.py | 112 +++++++++ benchmark/eval_pycapnp.py | 120 ++++++++++ benchmark/run_all.py | 60 +++++ benchmark/runner.py | 44 ++++ 18 files changed, 1771 insertions(+) create mode 100644 benchmark/carsales.capnp create mode 100644 benchmark/carsales.proto create mode 100644 benchmark/carsales_pb2.py create mode 100755 benchmark/carsales_proto.py create mode 100755 benchmark/carsales_pycapnp.py create mode 100644 benchmark/catrank.capnp create mode 100644 benchmark/catrank.proto create mode 100644 benchmark/catrank_pb2.py create mode 100755 benchmark/catrank_proto.py create mode 100755 benchmark/catrank_pycapnp.py create mode 100644 benchmark/common.py create mode 100644 benchmark/eval.capnp create mode 100644 benchmark/eval.proto create mode 100644 benchmark/eval_pb2.py create mode 100755 benchmark/eval_proto.py create mode 100755 benchmark/eval_pycapnp.py create mode 100755 benchmark/run_all.py create mode 100755 benchmark/runner.py diff --git a/benchmark/carsales.capnp b/benchmark/carsales.capnp new file mode 100644 index 0000000..e8e7fbb --- /dev/null +++ b/benchmark/carsales.capnp @@ -0,0 +1,82 @@ +# Copyright (c) 2013, Kenton Varda +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using Cxx = import "/capnp/c++.capnp"; + +@0xff75ddc6a36723c9; +$Cxx.namespace("capnp::benchmark::capnp"); + +struct ParkingLot { + cars@0: List(Car); +} + +struct TotalValue { + amount@0: UInt64; +} + +struct Car { + make@0: Text; + model@1: Text; + color@2: Color; + seats@3: UInt8; + doors@4: UInt8; + wheels@5: List(Wheel); + length@6: UInt16; + width@7: UInt16; + height@8: UInt16; + weight@9: UInt32; + engine@10: Engine; + fuelCapacity@11: Float32; + fuelLevel@12: Float32; + hasPowerWindows@13: Bool; + hasPowerSteering@14: Bool; + hasCruiseControl@15: Bool; + cupHolders@16: UInt8; + hasNavSystem@17: Bool; +} + +enum Color { + black @0; + white @1; + red @2; + green @3; + blue @4; + cyan @5; + magenta @6; + yellow @7; + silver @8; +} + +struct Wheel { + diameter@0: UInt16; + airPressure@1: Float32; + snowTires@2: Bool; +} + +struct Engine { + horsepower@0: UInt16; + cylinders@1: UInt8; + cc@2: UInt32; + usesGas@3: Bool; + usesElectric@4: Bool; +} diff --git a/benchmark/carsales.proto b/benchmark/carsales.proto new file mode 100644 index 0000000..5ec587e --- /dev/null +++ b/benchmark/carsales.proto @@ -0,0 +1,79 @@ +// Copyright (c) 2013, Kenton Varda +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package capnp.benchmark.protobuf; + +message ParkingLot { + repeated Car car = 1; +} + +message TotalValue { + required uint64 amount = 1; +} + +message Car { + optional string make = 1; + optional string model = 2; + optional Color color = 3; + optional uint32 seats = 4; + optional uint32 doors = 5; + repeated Wheel wheel = 6; + optional uint32 length = 7; + optional uint32 width = 8; + optional uint32 height = 9; + optional uint32 weight = 10; + optional Engine engine = 11; + optional float fuel_capacity = 12; + optional float fuel_level = 13; + optional bool has_power_windows = 14; + optional bool has_power_steering = 15; + optional bool has_cruise_control = 16; + optional uint32 cup_holders = 17; + optional bool has_nav_system = 18; +} + +enum Color { + BLACK = 0; + WHITE = 1; + RED = 2; + GREEN = 3; + BLUE = 4; + CYAN = 5; + MAGENTA = 6; + YELLOW = 7; + SILVER = 8; +} + +message Wheel { + optional uint32 diameter = 1; + optional float air_pressure = 2; + optional bool snow_tires = 3; +} + +message Engine { + optional uint32 horsepower = 1; + optional uint32 cylinders = 2; + optional uint32 cc = 3; + optional bool uses_gas = 4; + optional bool uses_electric = 5; +} diff --git a/benchmark/carsales_pb2.py b/benchmark/carsales_pb2.py new file mode 100644 index 0000000..2cd533a --- /dev/null +++ b/benchmark/carsales_pb2.py @@ -0,0 +1,422 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: carsales.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='carsales.proto', + package='capnp.benchmark.protobuf', + serialized_pb='\n\x0e\x63\x61rsales.proto\x12\x18\x63\x61pnp.benchmark.protobuf\"8\n\nParkingLot\x12*\n\x03\x63\x61r\x18\x01 \x03(\x0b\x32\x1d.capnp.benchmark.protobuf.Car\"\x1c\n\nTotalValue\x12\x0e\n\x06\x61mount\x18\x01 \x02(\x04\"\xbc\x03\n\x03\x43\x61r\x12\x0c\n\x04make\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\x12.\n\x05\x63olor\x18\x03 \x01(\x0e\x32\x1f.capnp.benchmark.protobuf.Color\x12\r\n\x05seats\x18\x04 \x01(\r\x12\r\n\x05\x64oors\x18\x05 \x01(\r\x12.\n\x05wheel\x18\x06 \x03(\x0b\x32\x1f.capnp.benchmark.protobuf.Wheel\x12\x0e\n\x06length\x18\x07 \x01(\r\x12\r\n\x05width\x18\x08 \x01(\r\x12\x0e\n\x06height\x18\t \x01(\r\x12\x0e\n\x06weight\x18\n \x01(\r\x12\x30\n\x06\x65ngine\x18\x0b \x01(\x0b\x32 .capnp.benchmark.protobuf.Engine\x12\x15\n\rfuel_capacity\x18\x0c \x01(\x02\x12\x12\n\nfuel_level\x18\r \x01(\x02\x12\x19\n\x11has_power_windows\x18\x0e \x01(\x08\x12\x1a\n\x12has_power_steering\x18\x0f \x01(\x08\x12\x1a\n\x12has_cruise_control\x18\x10 \x01(\x08\x12\x13\n\x0b\x63up_holders\x18\x11 \x01(\r\x12\x16\n\x0ehas_nav_system\x18\x12 \x01(\x08\"C\n\x05Wheel\x12\x10\n\x08\x64iameter\x18\x01 \x01(\r\x12\x14\n\x0c\x61ir_pressure\x18\x02 \x01(\x02\x12\x12\n\nsnow_tires\x18\x03 \x01(\x08\"d\n\x06\x45ngine\x12\x12\n\nhorsepower\x18\x01 \x01(\r\x12\x11\n\tcylinders\x18\x02 \x01(\r\x12\n\n\x02\x63\x63\x18\x03 \x01(\r\x12\x10\n\x08uses_gas\x18\x04 \x01(\x08\x12\x15\n\ruses_electric\x18\x05 \x01(\x08*j\n\x05\x43olor\x12\t\n\x05\x42LACK\x10\x00\x12\t\n\x05WHITE\x10\x01\x12\x07\n\x03RED\x10\x02\x12\t\n\x05GREEN\x10\x03\x12\x08\n\x04\x42LUE\x10\x04\x12\x08\n\x04\x43YAN\x10\x05\x12\x0b\n\x07MAGENTA\x10\x06\x12\n\n\x06YELLOW\x10\x07\x12\n\n\x06SILVER\x10\x08') + +_COLOR = _descriptor.EnumDescriptor( + name='Color', + full_name='capnp.benchmark.protobuf.Color', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='BLACK', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='WHITE', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='RED', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='GREEN', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BLUE', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='CYAN', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MAGENTA', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='YELLOW', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SILVER', index=8, number=8, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=750, + serialized_end=856, +) + +Color = enum_type_wrapper.EnumTypeWrapper(_COLOR) +BLACK = 0 +WHITE = 1 +RED = 2 +GREEN = 3 +BLUE = 4 +CYAN = 5 +MAGENTA = 6 +YELLOW = 7 +SILVER = 8 + + + +_PARKINGLOT = _descriptor.Descriptor( + name='ParkingLot', + full_name='capnp.benchmark.protobuf.ParkingLot', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='car', full_name='capnp.benchmark.protobuf.ParkingLot.car', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=44, + serialized_end=100, +) + + +_TOTALVALUE = _descriptor.Descriptor( + name='TotalValue', + full_name='capnp.benchmark.protobuf.TotalValue', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='amount', full_name='capnp.benchmark.protobuf.TotalValue.amount', index=0, + number=1, type=4, cpp_type=4, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=102, + serialized_end=130, +) + + +_CAR = _descriptor.Descriptor( + name='Car', + full_name='capnp.benchmark.protobuf.Car', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='make', full_name='capnp.benchmark.protobuf.Car.make', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='model', full_name='capnp.benchmark.protobuf.Car.model', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='color', full_name='capnp.benchmark.protobuf.Car.color', index=2, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='seats', full_name='capnp.benchmark.protobuf.Car.seats', index=3, + number=4, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='doors', full_name='capnp.benchmark.protobuf.Car.doors', index=4, + number=5, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='wheel', full_name='capnp.benchmark.protobuf.Car.wheel', index=5, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='length', full_name='capnp.benchmark.protobuf.Car.length', index=6, + number=7, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='width', full_name='capnp.benchmark.protobuf.Car.width', index=7, + number=8, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='height', full_name='capnp.benchmark.protobuf.Car.height', index=8, + number=9, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='weight', full_name='capnp.benchmark.protobuf.Car.weight', index=9, + number=10, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='engine', full_name='capnp.benchmark.protobuf.Car.engine', index=10, + number=11, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fuel_capacity', full_name='capnp.benchmark.protobuf.Car.fuel_capacity', index=11, + number=12, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='fuel_level', full_name='capnp.benchmark.protobuf.Car.fuel_level', index=12, + number=13, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_power_windows', full_name='capnp.benchmark.protobuf.Car.has_power_windows', index=13, + number=14, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_power_steering', full_name='capnp.benchmark.protobuf.Car.has_power_steering', index=14, + number=15, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_cruise_control', full_name='capnp.benchmark.protobuf.Car.has_cruise_control', index=15, + number=16, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cup_holders', full_name='capnp.benchmark.protobuf.Car.cup_holders', index=16, + number=17, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='has_nav_system', full_name='capnp.benchmark.protobuf.Car.has_nav_system', index=17, + number=18, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=133, + serialized_end=577, +) + + +_WHEEL = _descriptor.Descriptor( + name='Wheel', + full_name='capnp.benchmark.protobuf.Wheel', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='diameter', full_name='capnp.benchmark.protobuf.Wheel.diameter', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='air_pressure', full_name='capnp.benchmark.protobuf.Wheel.air_pressure', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='snow_tires', full_name='capnp.benchmark.protobuf.Wheel.snow_tires', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=579, + serialized_end=646, +) + + +_ENGINE = _descriptor.Descriptor( + name='Engine', + full_name='capnp.benchmark.protobuf.Engine', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='horsepower', full_name='capnp.benchmark.protobuf.Engine.horsepower', index=0, + number=1, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cylinders', full_name='capnp.benchmark.protobuf.Engine.cylinders', index=1, + number=2, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='cc', full_name='capnp.benchmark.protobuf.Engine.cc', index=2, + number=3, type=13, cpp_type=3, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='uses_gas', full_name='capnp.benchmark.protobuf.Engine.uses_gas', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='uses_electric', full_name='capnp.benchmark.protobuf.Engine.uses_electric', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=648, + serialized_end=748, +) + +_PARKINGLOT.fields_by_name['car'].message_type = _CAR +_CAR.fields_by_name['color'].enum_type = _COLOR +_CAR.fields_by_name['wheel'].message_type = _WHEEL +_CAR.fields_by_name['engine'].message_type = _ENGINE +DESCRIPTOR.message_types_by_name['ParkingLot'] = _PARKINGLOT +DESCRIPTOR.message_types_by_name['TotalValue'] = _TOTALVALUE +DESCRIPTOR.message_types_by_name['Car'] = _CAR +DESCRIPTOR.message_types_by_name['Wheel'] = _WHEEL +DESCRIPTOR.message_types_by_name['Engine'] = _ENGINE + +class ParkingLot(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _PARKINGLOT + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.ParkingLot) + +class TotalValue(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _TOTALVALUE + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.TotalValue) + +class Car(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _CAR + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Car) + +class Wheel(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _WHEEL + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Wheel) + +class Engine(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _ENGINE + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Engine) + + +# @@protoc_insertion_point(module_scope) diff --git a/benchmark/carsales_proto.py b/benchmark/carsales_proto.py new file mode 100755 index 0000000..5bdcd17 --- /dev/null +++ b/benchmark/carsales_proto.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +import carsales_pb2 +from common import rand_int, rand_double, rand_bool, from_bytes_helper +from random import choice + +MAKES = ["Toyota", "GM", "Ford", "Honda", "Tesla"] +MODELS = ["Camry", "Prius", "Volt", "Accord", "Leaf", "Model S"] +COLORS = ["black", "white", "red", "green", "blue", "cyan", "magenta", "yellow", "silver"] + +def random_car(car): + car.make = choice(MAKES) + car.model = choice(MODELS) + car.color = rand_int(len(COLORS)) + + car.seats = 2 + rand_int(6) + car.doors = 2 + rand_int(3) + + for _ in range(4): + wheel = car.wheel.add() + wheel.diameter = 25 + rand_int(15) + wheel.air_pressure = 30 + rand_double(20) + wheel.snow_tires = rand_int(16) == 0 + + car.length = 170 + rand_int(150) + car.width = 48 + rand_int(36) + car.height = 54 + rand_int(48) + car.weight = car.length * car.width * car.height / 200 + + engine = car.engine + engine.horsepower = 100 * rand_int(400) + engine.cylinders = 4 + 2 * rand_int(3) + engine.cc = 800 + rand_int(10000) + engine.uses_gas = True + engine.uses_electric = rand_bool() + + car.fuel_capacity = 10.0 + rand_double(30.0) + car.fuel_level = rand_double(car.fuel_capacity) + car.has_power_windows = rand_bool() + car.has_power_steering = rand_bool() + car.has_cruise_control = rand_bool() + car.cup_holders = rand_int(12) + car.has_nav_system = rand_bool() + +def calc_value(car): + result = 0 + + result += car.seats * 200 + result += car.doors * 350 + for wheel in car.wheel: + result += wheel.diameter * wheel.diameter + result += 100 if wheel.snow_tires else 0 + + result += car.length * car.width * car.height / 50 + + engine = car.engine + result += engine.horsepower * 40 + if engine.uses_electric: + if engine.uses_gas: + result += 5000 + else: + result += 3000 + + result += 100 if car.has_power_windows else 0 + result += 200 if car.has_power_steering else 0 + result += 400 if car.has_cruise_control else 0 + result += 2000 if car.has_nav_system else 0 + + result += car.cup_holders * 25 + + return result + +class Benchmark: + def __init__(self, compression): + self.Request = carsales_pb2.ParkingLot + self.Response = carsales_pb2.TotalValue + self.from_bytes_request = from_bytes_helper(carsales_pb2.ParkingLot) + self.from_bytes_response = from_bytes_helper(carsales_pb2.TotalValue) + self.to_bytes = lambda x: x.SerializeToString() + + def setup(self, request): + result = 0 + for _ in range(rand_int(200)): + car = request.car.add() + random_car(car) + result += calc_value(car) + return result + + def handle(self, request, response): + result = 0 + for car in request.car: + result += calc_value(car) + + response.amount = result + + def check(self, response, expected): + return response.amount == expected \ No newline at end of file diff --git a/benchmark/carsales_pycapnp.py b/benchmark/carsales_pycapnp.py new file mode 100755 index 0000000..c545f8a --- /dev/null +++ b/benchmark/carsales_pycapnp.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python + +import capnp +import carsales_capnp +from common import rand_int, rand_double, rand_bool +from random import choice + +MAKES = ["Toyota", "GM", "Ford", "Honda", "Tesla"] +MODELS = ["Camry", "Prius", "Volt", "Accord", "Leaf", "Model S"] +COLORS = ["black", "white", "red", "green", "blue", "cyan", "magenta", "yellow", "silver"] + +def random_car(car): + car.make = choice(MAKES) + car.model = choice(MODELS) + car.color = choice(COLORS) + + car.seats = 2 + rand_int(6) + car.doors = 2 + rand_int(3) + + for wheel in car.init('wheels', 4): + wheel.diameter = 25 + rand_int(15) + wheel.airPressure = 30 + rand_double(20) + wheel.snowTires = rand_int(16) == 0 + + car.length = 170 + rand_int(150) + car.width = 48 + rand_int(36) + car.height = 54 + rand_int(48) + car.weight = car.length * car.width * car.height / 200 + + engine = car.init('engine') + engine.horsepower = 100 * rand_int(400) + engine.cylinders = 4 + 2 * rand_int(3) + engine.cc = 800 + rand_int(10000) + engine.usesGas = True + engine.usesElectric = rand_bool() + + car.fuelCapacity = 10.0 + rand_double(30.0) + car.fuelLevel = rand_double(car.fuelCapacity) + car.hasPowerWindows = rand_bool() + car.hasPowerSteering = rand_bool() + car.hasCruiseControl = rand_bool() + car.cupHolders = rand_int(12) + car.hasNavSystem = rand_bool() + +def calc_value(car): + result = 0 + + result += car.seats * 200 + result += car.doors * 350 + for wheel in car.wheels: + result += wheel.diameter * wheel.diameter + result += 100 if wheel.snowTires else 0 + + result += car.length * car.width * car.height / 50 + + engine = car.engine + result += engine.horsepower * 40 + if engine.usesElectric: + if engine.usesGas: + result += 5000 + else: + result += 3000 + + result += 100 if car.hasPowerWindows else 0 + result += 200 if car.hasPowerSteering else 0 + result += 400 if car.hasCruiseControl else 0 + result += 2000 if car.hasNavSystem else 0 + + result += car.cupHolders * 25 + + return result + +class Benchmark: + def __init__(self, compression): + self.Request = carsales_capnp.ParkingLot.new_message + self.Response = carsales_capnp.TotalValue.new_message + if compression == 'packed': + self.from_bytes_request = carsales_capnp.ParkingLot.from_bytes_packed + self.from_bytes_response = carsales_capnp.TotalValue.from_bytes_packed + self.to_bytes = lambda x: x.to_bytes_packed() + else: + self.from_bytes_request = carsales_capnp.ParkingLot.from_bytes + self.from_bytes_response = carsales_capnp.TotalValue.from_bytes + self.to_bytes = lambda x: x.to_bytes() + + def setup(self, request): + result = 0 + for car in request.init('cars', rand_int(200)): + random_car(car) + result += calc_value(car) + return result + + def handle(self, request, response): + result = 0 + for car in request.cars: + result += calc_value(car) + + response.amount = result + + def check(self, response, expected): + return response.amount == expected \ No newline at end of file diff --git a/benchmark/catrank.capnp b/benchmark/catrank.capnp new file mode 100644 index 0000000..15fda47 --- /dev/null +++ b/benchmark/catrank.capnp @@ -0,0 +1,37 @@ +# Copyright (c) 2013, Kenton Varda +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using Cxx = import "/capnp/c++.capnp"; + +@0x82beb8e37ff79aba; +$Cxx.namespace("capnp::benchmark::capnp"); + +struct SearchResultList { + results@0: List(SearchResult); +} + +struct SearchResult { + url@0: Text; + score@1: Float64; + snippet@2: Text; +} diff --git a/benchmark/catrank.proto b/benchmark/catrank.proto new file mode 100644 index 0000000..87f2cf8 --- /dev/null +++ b/benchmark/catrank.proto @@ -0,0 +1,34 @@ +// Copyright (c) 2013, Kenton Varda +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package capnp.benchmark.protobuf; + +message SearchResultList { + repeated SearchResult result = 1; +} + +message SearchResult { + optional string url = 1; + optional double score = 2; + optional string snippet = 3; +} diff --git a/benchmark/catrank_pb2.py b/benchmark/catrank_pb2.py new file mode 100644 index 0000000..1baf3b0 --- /dev/null +++ b/benchmark/catrank_pb2.py @@ -0,0 +1,107 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: catrank.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='catrank.proto', + package='capnp.benchmark.protobuf', + serialized_pb='\n\rcatrank.proto\x12\x18\x63\x61pnp.benchmark.protobuf\"J\n\x10SearchResultList\x12\x36\n\x06result\x18\x01 \x03(\x0b\x32&.capnp.benchmark.protobuf.SearchResult\";\n\x0cSearchResult\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x01\x12\x0f\n\x07snippet\x18\x03 \x01(\t') + + + + +_SEARCHRESULTLIST = _descriptor.Descriptor( + name='SearchResultList', + full_name='capnp.benchmark.protobuf.SearchResultList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='result', full_name='capnp.benchmark.protobuf.SearchResultList.result', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=43, + serialized_end=117, +) + + +_SEARCHRESULT = _descriptor.Descriptor( + name='SearchResult', + full_name='capnp.benchmark.protobuf.SearchResult', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='url', full_name='capnp.benchmark.protobuf.SearchResult.url', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='score', full_name='capnp.benchmark.protobuf.SearchResult.score', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='snippet', full_name='capnp.benchmark.protobuf.SearchResult.snippet', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=119, + serialized_end=178, +) + +_SEARCHRESULTLIST.fields_by_name['result'].message_type = _SEARCHRESULT +DESCRIPTOR.message_types_by_name['SearchResultList'] = _SEARCHRESULTLIST +DESCRIPTOR.message_types_by_name['SearchResult'] = _SEARCHRESULT + +class SearchResultList(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _SEARCHRESULTLIST + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.SearchResultList) + +class SearchResult(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _SEARCHRESULT + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.SearchResult) + + +# @@protoc_insertion_point(module_scope) diff --git a/benchmark/catrank_proto.py b/benchmark/catrank_proto.py new file mode 100755 index 0000000..d91822e --- /dev/null +++ b/benchmark/catrank_proto.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from common import rand_int, rand_double, rand_bool, WORDS, from_bytes_helper +from random import choice +from string import letters +from itertools import izip +import catrank_pb2 + +class Benchmark: + def __init__(self, compression): + self.Request = catrank_pb2.SearchResultList + self.Response = catrank_pb2.SearchResultList + self.from_bytes_request = from_bytes_helper(catrank_pb2.SearchResultList) + self.from_bytes_response = from_bytes_helper(catrank_pb2.SearchResultList) + self.to_bytes = lambda x: x.SerializeToString() + + def setup(self, request): + goodCount = 0 + count = rand_int(1000) + + for i in range(count): + result = request.result.add() + result.score = 1000 - i + url_size = rand_int(100) + result.url = "http://example.com/" + ''.join([choice(letters) for _ in range(url_size)]) + + isCat = rand_bool() + isDog = rand_bool() + if isCat and not isDog: + goodCount += 1 + + snippet = [choice(WORDS) for i in range(rand_int(20))] + + if isCat: + snippet.append(" cat ") + if isDog: + snippet.append(" dog ") + + snippet += [choice(WORDS) for i in range(rand_int(20))] + + result.snippet = ''.join(snippet) + + return goodCount + + def handle(self, request, response): + for req in request.result: + resp = response.result.add() + score = req.score + + if " cat " in req.snippet: + score *= 10000 + if " dog " in req.snippet: + score /= 10000 + + resp.score = score + resp.url = req.url + resp.snippet = req.snippet + + + def check(self, response, expected): + goodCount = 0 + + for result in response.result: + if result.score > 1001: + goodCount += 1 + + return goodCount == expected diff --git a/benchmark/catrank_pycapnp.py b/benchmark/catrank_pycapnp.py new file mode 100755 index 0000000..4eab1db --- /dev/null +++ b/benchmark/catrank_pycapnp.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import capnp +import catrank_capnp +from common import rand_int, rand_double, rand_bool, WORDS +from random import choice +from string import letters +from itertools import izip + +class Benchmark: + def __init__(self, compression): + self.Request = catrank_capnp.SearchResultList.new_message + self.Response = catrank_capnp.SearchResultList.new_message + self.from_bytes_request = catrank_capnp.SearchResultList.from_bytes + self.from_bytes_response = catrank_capnp.SearchResultList.from_bytes + self.to_bytes = lambda x: x.to_bytes() + + def setup(self, request): + goodCount = 0 + count = rand_int(1000) + + results = request.init('results', count) + + for i, result in enumerate(results): + result.score = 1000 - i + url_size = rand_int(100) + result.url = "http://example.com/" + ''.join([choice(letters) for _ in range(url_size)]) + + isCat = rand_bool() + isDog = rand_bool() + if isCat and not isDog: + goodCount += 1 + + snippet = [choice(WORDS) for i in range(rand_int(20))] + + if isCat: + snippet.append(" cat ") + if isDog: + snippet.append(" dog ") + + snippet += [choice(WORDS) for i in range(rand_int(20))] + + result.snippet = ''.join(snippet) + + return goodCount + + def handle(self, request, response): + results = response.init('results', len(request.results)) + + for req, resp in izip(request.results, results): + score = req.score + + if " cat " in req.snippet: + score *= 10000 + if " dog " in req.snippet: + score /= 10000 + + resp.score = score + resp.url = req.url + resp.snippet = req.snippet + + + def check(self, response, expected): + goodCount = 0 + + for result in response.results: + if result.score > 1001: + goodCount += 1 + + return goodCount == expected diff --git a/benchmark/common.py b/benchmark/common.py new file mode 100644 index 0000000..f71af7a --- /dev/null +++ b/benchmark/common.py @@ -0,0 +1,77 @@ +from random import random + +WORDS = ["foo ", "bar ", "baz ", "qux ", "quux ", "corge ", "grault ", "garply ", "waldo ", "fred ", + "plugh ", "xyzzy ", "thud "] + +def rand_int(val): + return int(random() * val) + +def rand_double(val): + return random() * val + +def rand_bool(): + return random() < .5 + +def from_bytes_helper(klass): + def helper(text): + obj = klass() + obj.ParseFromString(text) + return obj + return helper + +def pass_by_object(reuse, iters, benchmark): + for _ in range(iters): + request = benchmark.Request() + expected = benchmark.setup(request) + + response = benchmark.Response() + benchmark.handle(request, response) + + if not benchmark.check(response, expected): + raise ValueError('Expected {}'.format(expected)) + +def pass_by_bytes(reuse, iters, benchmark): + for _ in range(iters): + request = benchmark.Request() + expected = benchmark.setup(request) + req_bytes = benchmark.to_bytes(request) + + request2 = benchmark.from_bytes_request(req_bytes) + response = benchmark.Response() + benchmark.handle(request2, response) + resp_bytes = benchmark.to_bytes(response) + + response2 = benchmark.from_bytes_response(resp_bytes) + if not benchmark.check(response2, expected): + raise ValueError('Expected {}'.format(expected)) + +def do_benchmark(mode, *args, **kwargs): + if mode == "client": + pass + elif mode == "object": + return pass_by_object(*args, **kwargs) + elif mode == "passByBytes": + return pass_by_bytes(*args, **kwargs) + else: + raise ValueError("Unknown mode: " + str(mode)) +# typedef typename BenchmarkTypes::template BenchmarkMethods +# BenchmarkMethods; +# if (mode == "client") { +# return BenchmarkMethods::syncClient(STDIN_FILENO, STDOUT_FILENO, iters); +# } else if (mode == "server") { +# return BenchmarkMethods::server(STDIN_FILENO, STDOUT_FILENO, iters); +# } else if (mode == "object") { +# return BenchmarkMethods::passByObject(iters, false); +# } else if (mode == "object-size") { +# return BenchmarkMethods::passByObject(iters, true); +# } else if (mode == "bytes") { +# return BenchmarkMethods::passByBytes(iters); +# } else if (mode == "pipe") { +# return passByPipe(BenchmarkMethods::syncClient, iters); +# } else if (mode == "pipe-async") { +# return passByPipe(BenchmarkMethods::asyncClient, iters); +# } else { +# fprintf(stderr, "Unknown mode: %s\n", mode.c_str()); +# exit(1); +# } +# } \ No newline at end of file diff --git a/benchmark/eval.capnp b/benchmark/eval.capnp new file mode 100644 index 0000000..8e008ec --- /dev/null +++ b/benchmark/eval.capnp @@ -0,0 +1,53 @@ +# Copyright (c) 2013, Kenton Varda +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using Cxx = import "/capnp/c++.capnp"; + +@0xe12dc4c3e70e9eda; +$Cxx.namespace("capnp::benchmark::capnp"); + +enum Operation { + add @0; + subtract @1; + multiply @2; + divide @3; + modulus @4; +} + +struct Expression { + op@0: Operation; + + left :union { + value@1: Int32; + expression@2: Expression; + } + + right :union { + value@3: Int32; + expression@4: Expression; + } +} + +struct EvaluationResult { + value@0: Int32; +} diff --git a/benchmark/eval.proto b/benchmark/eval.proto new file mode 100644 index 0000000..4c6f72c --- /dev/null +++ b/benchmark/eval.proto @@ -0,0 +1,46 @@ +// Copyright (c) 2013, Kenton Varda +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package capnp.benchmark.protobuf; + +enum Operation { + ADD = 0; + SUBTRACT = 1; + MULTIPLY = 2; + DIVIDE = 3; + MODULUS = 4; +} + +message Expression { + required Operation op = 1; + + optional int32 left_value = 2; + optional Expression left_expression = 3; + + optional int32 right_value = 4; + optional Expression right_expression = 5; +} + +message EvaluationResult { + required sint32 value = 1; +} diff --git a/benchmark/eval_pb2.py b/benchmark/eval_pb2.py new file mode 100644 index 0000000..e2848f0 --- /dev/null +++ b/benchmark/eval_pb2.py @@ -0,0 +1,163 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: eval.proto + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='eval.proto', + package='capnp.benchmark.protobuf', + serialized_pb='\n\neval.proto\x12\x18\x63\x61pnp.benchmark.protobuf\"\xe5\x01\n\nExpression\x12/\n\x02op\x18\x01 \x02(\x0e\x32#.capnp.benchmark.protobuf.Operation\x12\x12\n\nleft_value\x18\x02 \x01(\x05\x12=\n\x0fleft_expression\x18\x03 \x01(\x0b\x32$.capnp.benchmark.protobuf.Expression\x12\x13\n\x0bright_value\x18\x04 \x01(\x05\x12>\n\x10right_expression\x18\x05 \x01(\x0b\x32$.capnp.benchmark.protobuf.Expression\"!\n\x10\x45valuationResult\x12\r\n\x05value\x18\x01 \x02(\x11*I\n\tOperation\x12\x07\n\x03\x41\x44\x44\x10\x00\x12\x0c\n\x08SUBTRACT\x10\x01\x12\x0c\n\x08MULTIPLY\x10\x02\x12\n\n\x06\x44IVIDE\x10\x03\x12\x0b\n\x07MODULUS\x10\x04') + +_OPERATION = _descriptor.EnumDescriptor( + name='Operation', + full_name='capnp.benchmark.protobuf.Operation', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='ADD', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SUBTRACT', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MULTIPLY', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DIVIDE', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MODULUS', index=4, number=4, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=307, + serialized_end=380, +) + +Operation = enum_type_wrapper.EnumTypeWrapper(_OPERATION) +ADD = 0 +SUBTRACT = 1 +MULTIPLY = 2 +DIVIDE = 3 +MODULUS = 4 + + + +_EXPRESSION = _descriptor.Descriptor( + name='Expression', + full_name='capnp.benchmark.protobuf.Expression', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='op', full_name='capnp.benchmark.protobuf.Expression.op', index=0, + number=1, type=14, cpp_type=8, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='left_value', full_name='capnp.benchmark.protobuf.Expression.left_value', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='left_expression', full_name='capnp.benchmark.protobuf.Expression.left_expression', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='right_value', full_name='capnp.benchmark.protobuf.Expression.right_value', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='right_expression', full_name='capnp.benchmark.protobuf.Expression.right_expression', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=41, + serialized_end=270, +) + + +_EVALUATIONRESULT = _descriptor.Descriptor( + name='EvaluationResult', + full_name='capnp.benchmark.protobuf.EvaluationResult', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='capnp.benchmark.protobuf.EvaluationResult.value', index=0, + number=1, type=17, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=272, + serialized_end=305, +) + +_EXPRESSION.fields_by_name['op'].enum_type = _OPERATION +_EXPRESSION.fields_by_name['left_expression'].message_type = _EXPRESSION +_EXPRESSION.fields_by_name['right_expression'].message_type = _EXPRESSION +DESCRIPTOR.message_types_by_name['Expression'] = _EXPRESSION +DESCRIPTOR.message_types_by_name['EvaluationResult'] = _EVALUATIONRESULT + +class Expression(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _EXPRESSION + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Expression) + +class EvaluationResult(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _EVALUATIONRESULT + + # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.EvaluationResult) + + +# @@protoc_insertion_point(module_scope) diff --git a/benchmark/eval_proto.py b/benchmark/eval_proto.py new file mode 100755 index 0000000..b7f79ec --- /dev/null +++ b/benchmark/eval_proto.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python + +from common import rand_int, rand_double, rand_bool, from_bytes_helper +from random import choice +import eval_pb2 + +MAX_INT = 2**31 - 1 +MIN_INT = -(2**31) + +OPERATIONS = [ + "add", + "subtract", + "multiply", + "divide", + "modulus"] + +def clamp(res): + if res > MAX_INT: + return MAX_INT + elif res < MIN_INT: + return MIN_INT + else: + return res + +def div(a, b): + if b == 0: + return MAX_INT + if a == MIN_INT and b == -1: + return MAX_INT + + return a / b + +def mod(a, b): + if b == 0: + return MAX_INT + if a == MIN_INT and b == -1: + return MAX_INT + + return a % b + +def make_expression(exp, depth): + exp.op = rand_int(len(OPERATIONS)) + + if rand_int(8) < depth: + left = rand_int(128) + 1 + exp.left_value = left + else: + left = make_expression(exp.left_expression, depth+1) + + if rand_int(8) < depth: + right = rand_int(128) + 1 + exp.right_value = right + else: + right = make_expression(exp.right_expression, depth+1) + + op = exp.op + if op == 0: + return clamp(left + right) + elif op == 1: + return clamp(left - right) + elif op == 2: + return clamp(left * right) + elif op == 3: + return div(left, right) + elif op == 4: + return mod(left, right) + raise RuntimeError("op wasn't a valid value: " + str(op)) + +def evaluate_expression(exp): + left = 0 + right = 0 + + if exp.HasField('left_value'): + left = exp.left_value + else: + left = evaluate_expression(exp.left_expression) + + if exp.HasField('right_value'): + right = exp.right_value + else: + right = evaluate_expression(exp.right_expression) + + + op = exp.op + if op == 0: + return clamp(left + right) + elif op == 1: + return clamp(left - right) + elif op == 2: + return clamp(left * right) + elif op == 3: + return div(left, right) + elif op == 4: + return mod(left, right) + raise RuntimeError("op wasn't a valid value: " + str(op)) + +class Benchmark: + def __init__(self, compression): + self.Request = eval_pb2.Expression + self.Response = eval_pb2.EvaluationResult + self.from_bytes_request = from_bytes_helper(eval_pb2.Expression) + self.from_bytes_response = from_bytes_helper(eval_pb2.EvaluationResult) + self.to_bytes = lambda x: x.SerializeToString() + + def setup(self, request): + return make_expression(request, 0) + + def handle(self, request, response): + response.value = evaluate_expression(request) + + def check(self, response, expected): + return response.value == expected \ No newline at end of file diff --git a/benchmark/eval_pycapnp.py b/benchmark/eval_pycapnp.py new file mode 100755 index 0000000..913cb54 --- /dev/null +++ b/benchmark/eval_pycapnp.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +import capnp +import eval_capnp +from common import rand_int, rand_double, rand_bool +from random import choice + +MAX_INT = 2**31 - 1 +MIN_INT = -(2**31) + +OPERATIONS = [ + "add", + "subtract", + "multiply", + "divide", + "modulus"] + +def clamp(res): + if res > MAX_INT: + return MAX_INT + elif res < MIN_INT: + return MIN_INT + else: + return res + +def div(a, b): + if b == 0: + return MAX_INT + if a == MIN_INT and b == -1: + return MAX_INT + + return a / b + +def mod(a, b): + if b == 0: + return MAX_INT + if a == MIN_INT and b == -1: + return MAX_INT + + return a % b + +def make_expression(exp, depth): + exp.op = choice(OPERATIONS) + + if rand_int(8) < depth: + left = rand_int(128) + 1 + exp.left.value = left + else: + left = make_expression(exp.left.init('expression'), depth+1) + + if rand_int(8) < depth: + right = rand_int(128) + 1 + exp.right.value = right + else: + right = make_expression(exp.right.init('expression'), depth+1) + + op = exp.op + if op == 'add': + return clamp(left + right) + elif op == 'subtract': + return clamp(left - right) + elif op == 'multiply': + return clamp(left * right) + elif op == 'divide': + return div(left, right) + elif op == 'modulus': + return mod(left, right) + raise RuntimeError("op wasn't a valid value: " + str(op)) + +def evaluate_expression(exp): + left = 0 + right = 0 + + which = exp.left.which() + if which == 'value': + left = exp.left.value + elif which == 'expression': + left = evaluate_expression(exp.left.expression) + + which = exp.right.which() + if which == 'value': + right = exp.right.value + elif which == 'expression': + right = evaluate_expression(exp.right.expression) + + + op = exp.op + if op == 'add': + return clamp(left + right) + elif op == 'subtract': + return clamp(left - right) + elif op == 'multiply': + return clamp(left * right) + elif op == 'divide': + return div(left, right) + elif op == 'modulus': + return mod(left, right) + raise RuntimeError("op wasn't a valid value: " + str(op)) + +class Benchmark: + def __init__(self, compression): + self.Request = eval_capnp.Expression.new_message + self.Response = eval_capnp.EvaluationResult.new_message + if compression == 'packed': + self.from_bytes_request = eval_capnp.Expression.from_bytes_packed + self.from_bytes_response = eval_capnp.EvaluationResult.from_bytes_packed + self.to_bytes = lambda x: x.to_bytes_packed() + else: + self.from_bytes_request = eval_capnp.Expression.from_bytes + self.from_bytes_response = eval_capnp.EvaluationResult.from_bytes + self.to_bytes = lambda x: x.to_bytes() + + def setup(self, request): + return make_expression(request, 0) + + def handle(self, request, response): + response.value = evaluate_expression(request) + + def check(self, response, expected): + return response.value == expected \ No newline at end of file diff --git a/benchmark/run_all.py b/benchmark/run_all.py new file mode 100755 index 0000000..657ccbd --- /dev/null +++ b/benchmark/run_all.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +from subprocess import Popen, PIPE +import sys +import os +import json + +def run_one(type, name, mode, iters, faster): + args = [] + res_type = type + if faster: + if type == 'proto': + os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp' + res_type = 'proto_cpp' + if type == 'pycapnp': + args += ['-c', 'packed'] + res_type = 'pycapnp_packed' + + p = Popen(["time", "-p", "python", "runner.py", name, '-s', type, '-i', str(iters)] + args, stdout=PIPE, stderr=PIPE) + res = p.communicate()[1] + + data = {} + res = res.strip() + + for line in res.split('\n'): + vals = line.split() + data[vals[0]] = float(vals[1]) + + data['type'] = res_type + data['mode'] = mode + data['name'] = name + data['iters'] = iters + + if 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION' in os.environ: + del os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] + return data + +def run_each(name, iters): + ret = [] + + ret.append(run_one('pycapnp', name, 'object', iters, False)) + ret.append(run_one('proto', name, 'object', iters, False)) + ret.append(run_one('proto', name, 'object', iters, True)) + + ret.append(run_one('pycapnp', name, 'passByBytes', iters, False)) + ret.append(run_one('pycapnp', name, 'passByBytes', iters, True)) + ret.append(run_one('proto', name, 'passByBytes', iters, False)) + ret.append(run_one('proto', name, 'passByBytes', iters, True)) + + return ret + +def main(): + data = [] + data += run_each('carsales', 2000) + data += run_each('catrank', 100) + data += run_each('eval', 10000) + json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(',', ': ')) + +if __name__ == '__main__': + main() diff --git a/benchmark/runner.py b/benchmark/runner.py new file mode 100755 index 0000000..0bfe84c --- /dev/null +++ b/benchmark/runner.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +import argparse +from importlib import import_module +from common import do_benchmark +from timeit import default_timer +import os +import sys + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("name", help="Name of the benchmark to run, eg. carsales", nargs='?', default='carsales') + parser.add_argument("-c", "--compression", help="Specify the compression type", default=None) + parser.add_argument("-s", "--suffix", help="Choose the protocol type.", default='pycapnp') + parser.add_argument("-m", "--mode", help="Specify the mode", default='object') + parser.add_argument("-i", "--iters", help="Specify the number of iterations manually. By default, it will be looked up in preset table", default=10, type=int) + parser.add_argument("-r", "--reuse", help="If this flag is passed, objects will be re-used", action='store_true') + parser.add_argument("-I", "--includes", help="Directories to add to PYTHONPATH", default='/usr/local/include') + + return parser.parse_args() + +def run_test(args): + tic = default_timer() + + name = args.name + module = import_module(name + '_' + args.suffix) + benchmark = module.Benchmark(compression=args.compression) + + if args.iters is None: + iters = ITERATIONS[name] + else: + iters = args.iters + + do_benchmark(mode=args.mode, benchmark=benchmark, iters=iters, reuse=args.reuse) + toc = default_timer() + return toc - tic + +def main(): + args = parse_args() + sys.path.append(args.includes) + run_test(args) + +if __name__ == '__main__': + main() \ No newline at end of file