From 2db935a00bb3109913cd1ac112f0b7f4183ee3c3 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 4 Sep 2014 18:04:52 -0700 Subject: [PATCH] Speed up code generated modules even more. Switch to using generated c++ capnp code --- capnp/_gen.py | 18 +++-- capnp/lib/capnp.pxd | 27 +++++++ capnp/lib/capnp.pyx | 50 +++++++++++-- capnp/templates/module.pyx | 146 +++++++++++++++++++++++++++++++++---- capnp/templates/setup.py | 17 +++++ 5 files changed, 229 insertions(+), 29 deletions(-) diff --git a/capnp/_gen.py b/capnp/_gen.py index bd626b3..e86e86c 100644 --- a/capnp/_gen.py +++ b/capnp/_gen.py @@ -4,6 +4,7 @@ import capnp import schema_capnp import sys from jinja2 import Environment, PackageLoader +import os def main(): env = Environment(loader=PackageLoader('capnp', 'templates')) @@ -11,24 +12,31 @@ def main(): code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) code=code.to_dict() - code['nodes'] = [node for node in code['nodes'] if 'struct' in node] + code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] for node in code['nodes']: displayName = node['displayName'] parent, path = displayName.split(':') node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) node['module_name'] = path.replace('.', '_') + node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) node['schema'] = '_{}_Schema'.format(node['module_name']) is_union = False for field in node['struct']['fields']: if field['discriminantValue'] != 65535: is_union = True + field['c_name'] = field['name'][0].upper() + field['name'][1:] node['is_union'] = is_union + include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) module = env.get_template('module.pyx') - filename = code['requestedFiles'][0]['filename'].replace('.', '_') + '_cython.pyx' - # TODO: handle multiple files - with open(filename, 'w') as out: - out.write(module.render(code=code)) + + for f in code['requestedFiles']: + filename = f['filename'].replace('.', '_') + '_cython.pyx' + + file_code = dict(code) + file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] + with open(filename, 'w') as out: + out.write(module.render(code=file_code, file=f, include_dir=include_dir)) setup = env.get_template('setup.py') with open('setup_capnp.py', 'w') as out: diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index de44bdb..20c1ab4 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -83,7 +83,34 @@ cdef class _Schema: cpdef get_dependency(self, id) cpdef get_proto(self) +cdef class _InterfaceSchema: + cdef C_InterfaceSchema thisptr + cdef object __method_names + cdef _init(self, C_InterfaceSchema other) + cpdef get_dependency(self, id) + +cdef class _DynamicEnum: + cdef capnp.DynamicEnum thisptr + cdef public object _parent + + cdef _init(self, capnp.DynamicEnum other, object parent) + cpdef _as_str(self) + +cdef class _DynamicListBuilder: + cdef C_DynamicList.Builder thisptr + cdef public object _parent + cdef _init(self, C_DynamicList.Builder other, object parent) + + cdef _get(self, index) + cdef _set(self, index, value) + + cpdef adopt(self, index, _DynamicOrphan orphan) + cpdef disown(self, index) + cdef to_python_reader(C_DynamicValue.Reader self, object parent) cdef to_python_builder(C_DynamicValue.Builder self, object parent) cdef _to_dict(msg, bint verbose) +cdef _from_dict(_DynamicStructBuilder msg, dict d) +cdef _from_list(_DynamicListBuilder msg, list d) cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent) +cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 714730e..d2579ba 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -495,8 +495,6 @@ cdef class _DynamicListBuilder: for phone in phones: print phone.number """ - cdef C_DynamicList.Builder thisptr - cdef public object _parent cdef _init(self, C_DynamicList.Builder other, object parent): self.thisptr = other self._parent = parent @@ -778,6 +776,48 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField else: raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) +cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent): + cdef C_DynamicValue.Reader temp + value_type = type(value) + + if value_type is int or value_type is long: + if value < 0: + temp = C_DynamicValue.Reader(value) + else: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is float: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is bool: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is bytes: + _setBytes(thisptr, field, value) + elif isinstance(value, basestring): + _setBaseString(thisptr, field, value) + elif value_type is list: + builder = to_python_builder(thisptr.init(field, len(value)), parent) + _from_list(builder, value) + elif value_type is dict: + builder = to_python_builder(thisptr.get(field), parent) + _from_dict(builder, value) + elif value is None: + temp = C_DynamicValue.Reader(VOID) + thisptr.set(field, temp) + elif value_type is _DynamicStructBuilder: + thisptr.set(field, _extract_dynamic_struct_builder(value)) + elif value_type is _DynamicStructReader: + thisptr.set(field, _extract_dynamic_struct_reader(value)) + elif value_type is _DynamicCapabilityClient: + thisptr.set(field, _extract_dynamic_client(value)) + elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer): + thisptr.set(field, _extract_dynamic_server(value)) + elif value_type is _DynamicEnum: + thisptr.set(field, _extract_dynamic_enum(value)) + else: + raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) + cdef _to_dict(msg, bint verbose): msg_type = type(msg) if msg_type is _DynamicListBuilder or msg_type is _DynamicListReader or msg_type is _DynamicResizableListBuilder: @@ -824,9 +864,6 @@ cdef _from_list(_DynamicListBuilder msg, list d): cdef class _DynamicEnum: - cdef capnp.DynamicEnum thisptr - cdef public object _parent - cdef _init(self, capnp.DynamicEnum other, object parent): self.thisptr = other self._parent = parent @@ -2341,9 +2378,6 @@ cdef class _StructSchemaField: return '' % self.proto.name cdef class _InterfaceSchema: - cdef C_InterfaceSchema thisptr - cdef object __method_names - cdef _init(self, C_InterfaceSchema other): self.thisptr = other return self diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index 2b50fbd..cad4193 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -1,30 +1,122 @@ # addressbook_fast.pyx # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 -# distutils: include_dirs = /usr/local/lib/python2.7/site-packages +# distutils: include_dirs = {{include_dir}} +# distutils: libraries = capnpc capnp capnp-rpc +# distutils: sources = {{file.filename}}.cpp # cython: c_string_type = str # cython: c_string_encoding = default # cython: embedsignature = True import capnp -{%- for file in code.requestedFiles %} import {{file.filename | replace('.', '_')}} -{% endfor %} -from capnp.includes.capnp_cpp cimport DynamicValue -from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldWithField -{%- for node in code.nodes %} -{{node.schema}} = {{node.module_path}}.schema - {%- for field in node.struct.fields %} -cdef _StructSchemaField {{node.module_name}}_{{field.name}} = {{node.schema}}.fields['{{field.name}}'] +from libcpp cimport bool as cbool +from capnp cimport helpers +from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr +from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema + +from capnp.helpers.non_circular cimport reraise_kj_exception + +cdef DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value): + return DynamicValue.Reader(value.thisptr.asReader()) + +cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader value): + return DynamicValue.Reader(value.thisptr) + +cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): + return DynamicValue.Reader(value.thisptr) + +cdef _from_dict(_DynamicStructBuilder msg, dict d): + for key, val in d.iteritems(): + if key != 'which': + try: + msg._set(key, val) + except Exception as e: + if 'expected isSetInUnion(field)' in str(e): + msg.init(key) + msg._set(key, val) + +cdef _from_list(_DynamicListBuilder msg, list d): + cdef size_t count = 0 + for val in d: + msg._set(count, val) + count += 1 + +cdef DynamicValue.Reader to_dynamic_value(value): + cdef DynamicValue.Reader temp + cdef StringPtr temp_string + value_type = type(value) + + if value_type is int or value_type is long: + if value < 0: + temp = DynamicValue.Reader(value) + else: + temp = DynamicValue.Reader(value) + elif value_type is float: + temp = DynamicValue.Reader(value) + elif value_type is bool: + temp = DynamicValue.Reader(value) + elif value_type is bytes: + temp_string = StringPtr(value, len(value)) + temp = DynamicValue.Reader(temp_string) + elif isinstance(value, basestring): + encoded_value = value.encode() + temp_string = StringPtr(encoded_value, len(encoded_value)) + temp = DynamicValue.Reader(temp_string) + elif value is None: + temp = DynamicValue.Reader(VOID) + elif value_type is _DynamicStructBuilder: + temp = _extract_dynamic_struct_builder(value) + elif value_type is _DynamicStructReader: + temp = _extract_dynamic_struct_reader(value) + elif value_type is _DynamicEnum: + temp = _extract_dynamic_enum(value) + else: + raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value)))) + + return temp + + +cdef extern from "{{file.filename}}.h": + {%- for node in code.nodes %} + Schema get{{node.module_name}}Schema"capnp::Schema::from<{{node.c_module_path}}>"() + + cdef cppclass {{node.module_name}}"{{node.c_module_path}}": + cppclass Reader: + {%- for field in node.struct.fields %} + DynamicValue.Reader get{{field.c_name}}() + {%- endfor %} + cppclass Builder: + {%- for field in node.struct.fields %} + DynamicValue.Builder get{{field.c_name}}() + set{{field.c_name}}(DynamicValue.Reader) + {%- endfor %} {%- endfor %} + cdef cppclass C_DynamicStruct_Reader" ::capnp::DynamicStruct::Reader": + {%- for node in code.nodes %} + {{node.module_name}}.Reader as{{node.module_name}}"as<{{node.c_module_path}}>"() + {%- endfor %} + + cdef cppclass C_DynamicStruct_Builder" ::capnp::DynamicStruct::Builder": + {%- for node in code.nodes %} + {{node.module_name}}.Builder as{{node.module_name}}"as<{{node.c_module_path}}>"() + {%- endfor %} + +{%- for node in code.nodes %} + +{{node.schema}} = _Schema()._init(get{{node.module_name}}Schema()).as_struct() +{{node.module_path}}.schema = {{node.schema}} + cdef class {{node.module_name}}_Reader(_DynamicStructReader): + cdef {{node.module_name}}.Reader thisptr_child def __init__(self, _DynamicStructReader struct): self._init(struct.thisptr, struct._parent, struct.is_root, False) + self.thisptr_child = (struct.thisptr).as{{node.module_name}}() {% for field in node.struct.fields %} - cpdef _get_{{field.name}}(self): - cdef DynamicValue.Reader temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr) + cpdef _get_{{field.name}}(self) except +reraise_kj_exception: + cdef DynamicValue.Reader temp = self.thisptr_child.get{{field.c_name}}() return to_python_reader(temp, self._parent) property {{field.name}}: def __get__(self): @@ -32,21 +124,43 @@ cdef class {{node.module_name}}_Reader(_DynamicStructReader): {%- endfor %} def to_dict(self, verbose=False): - return { + ret = { {% for field in node.struct.fields %} + {% if field.discriminantValue == 65535 %} '{{field.name}}': _to_dict(self.{{field.name}}, verbose), + {% endif %} {%- endfor %} } + {% if node.is_union %} + which = self.which() + ret[which] = getattr(self, which) + {% endif %} + + return ret + cdef class {{node.module_name}}_Builder(_DynamicStructBuilder): + cdef {{node.module_name}}.Builder thisptr_child def __init__(self, _DynamicStructBuilder struct): self._init(struct.thisptr, struct._parent, struct.is_root, False) + self.thisptr_child = (struct.thisptr).as{{node.module_name}}() {% for field in node.struct.fields %} - cpdef _get_{{field.name}}(self): - cdef DynamicValue.Builder temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr) + cpdef _get_{{field.name}}(self) except +reraise_kj_exception: + cdef DynamicValue.Builder temp = self.thisptr_child.get{{field.c_name}}() return to_python_builder(temp, self._parent) - cpdef _set_{{field.name}}(self, value): - _setDynamicFieldWithField(self.thisptr, {{node.module_name}}_{{field.name}}, value, self._parent) + cpdef _set_{{field.name}}(self, value) except +reraise_kj_exception: + _setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent) + # cdef DynamicValue.Builder temp + # value_type = type(value) + # if value_type is list: + # builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent) + # _from_list(builder, value) + # elif value_type is dict: + # builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent) + # _from_dict(builder, value) + # else: + # self.thisptr_child.set{{field.c_name}}(to_dynamic_value(value)) + property {{field.name}}: def __get__(self): return self._get_{{field.name}}() diff --git a/capnp/templates/setup.py b/capnp/templates/setup.py index 46883bb..29c0b2a 100644 --- a/capnp/templates/setup.py +++ b/capnp/templates/setup.py @@ -2,6 +2,23 @@ from distutils.core import setup from Cython.Build import cythonize import os +import re + + +files = [{% for file in code.requestedFiles %}"{{file.filename}}",{% endfor %}] + +for f in files: + cpp_file = f + '.cpp' + if not os.path.exists(cpp_file): + if not os.path.exists(f + '.c++'): + raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.") + os.rename(f + '.c++', cpp_file) + + with open(f + '.h', "r") as file: + lines = file.readlines() + with open(f + '.h', "w") as file: + for line in lines: + file.write(re.sub(r'Builder\(\)\s*=\s*delete;', 'Builder() = default;', line)) setup( name="{{code.requestedFiles[0] | replace('.', '_')}}",