IQ.Lvbs/pycapnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed up code generated modules even more.

Browse Source 
Switch to using generated c++ capnp code
This commit is contained in:
Jason Paryani
2014-09-04 18:04:52 -07:00
parent 93352a31d2
commit 2db935a00b
5 changed files with 229 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
capnp/_gen.py
View File

@@ -4,6 +4,7 @@ import capnp
import schema_capnp
import sys
from jinja2 import Environment, PackageLoader
import os
def main():
env = Environment(loader=PackageLoader('capnp', 'templates'))
@@ -11,24 +12,31 @@ def main():
code = schema_capnp.CodeGeneratorRequest.read(sys.stdin)
code=code.to_dict()
code['nodes'] = [node for node in code['nodes'] if 'struct' in node]
code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0]
for node in code['nodes']:
displayName = node['displayName']
parent, path = displayName.split(':')
node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')])
node['module_name'] = path.replace('.', '_')
node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')])
node['schema'] = '_{}_Schema'.format(node['module_name'])
is_union = False
for field in node['struct']['fields']:
if field['discriminantValue'] != 65535:
is_union = True
field['c_name'] = field['name'][0].upper() + field['name'][1:]
node['is_union'] = is_union
include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..'))
module = env.get_template('module.pyx')
filename = code['requestedFiles'][0]['filename'].replace('.', '_') + '_cython.pyx'
# TODO: handle multiple files
for f in code['requestedFiles']:
filename = f['filename'].replace('.', '_') + '_cython.pyx'
file_code = dict(code)
file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])]
with open(filename, 'w') as out:
out.write(module.render(code=code))
out.write(module.render(code=file_code, file=f, include_dir=include_dir))
setup = env.get_template('setup.py')
with open('setup_capnp.py', 'w') as out:

27
capnp/lib/capnp.pxd
View File

@@ -83,7 +83,34 @@ cdef class _Schema:
cpdef get_dependency(self, id)
cpdef get_proto(self)
cdef class _InterfaceSchema:
cdef C_InterfaceSchema thisptr
cdef object __method_names
cdef _init(self, C_InterfaceSchema other)
cpdef get_dependency(self, id)
cdef class _DynamicEnum:
cdef capnp.DynamicEnum thisptr
cdef public object _parent
cdef _init(self, capnp.DynamicEnum other, object parent)
cpdef _as_str(self)
cdef class _DynamicListBuilder:
cdef C_DynamicList.Builder thisptr
cdef public object _parent
cdef _init(self, C_DynamicList.Builder other, object parent)
cdef _get(self, index)
cdef _set(self, index, value)
cpdef adopt(self, index, _DynamicOrphan orphan)
cpdef disown(self, index)
cdef to_python_reader(C_DynamicValue.Reader self, object parent)
cdef to_python_builder(C_DynamicValue.Builder self, object parent)
cdef _to_dict(msg, bint verbose)
cdef _from_dict(_DynamicStructBuilder msg, dict d)
cdef _from_list(_DynamicListBuilder msg, list d)
cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent)
cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)

50
capnp/lib/capnp.pyx
View File

@@ -495,8 +495,6 @@ cdef class _DynamicListBuilder:
for phone in phones:
print phone.number
"""
cdef C_DynamicList.Builder thisptr
cdef public object _parent
cdef _init(self, C_DynamicList.Builder other, object parent):
self.thisptr = other
self._parent = parent
@@ -778,6 +776,48 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField
else:
raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent):
cdef C_DynamicValue.Reader temp
value_type = type(value)
if value_type is int or value_type is long:
if value < 0:
temp = C_DynamicValue.Reader(<long long>value)
else:
temp = C_DynamicValue.Reader(<unsigned long long>value)
thisptr.set(field, temp)
elif value_type is float:
temp = C_DynamicValue.Reader(<double>value)
thisptr.set(field, temp)
elif value_type is bool:
temp = C_DynamicValue.Reader(<cbool>value)
thisptr.set(field, temp)
elif value_type is bytes:
_setBytes(thisptr, field, value)
elif isinstance(value, basestring):
_setBaseString(thisptr, field, value)
elif value_type is list:
builder = to_python_builder(thisptr.init(field, len(value)), parent)
_from_list(builder, value)
elif value_type is dict:
builder = to_python_builder(thisptr.get(field), parent)
_from_dict(builder, value)
elif value is None:
temp = C_DynamicValue.Reader(VOID)
thisptr.set(field, temp)
elif value_type is _DynamicStructBuilder:
thisptr.set(field, _extract_dynamic_struct_builder(value))
elif value_type is _DynamicStructReader:
thisptr.set(field, _extract_dynamic_struct_reader(value))
elif value_type is _DynamicCapabilityClient:
thisptr.set(field, _extract_dynamic_client(value))
elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer):
thisptr.set(field, _extract_dynamic_server(value))
elif value_type is _DynamicEnum:
thisptr.set(field, _extract_dynamic_enum(value))
else:
raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
cdef _to_dict(msg, bint verbose):
msg_type = type(msg)
if msg_type is _DynamicListBuilder or msg_type is _DynamicListReader or msg_type is _DynamicResizableListBuilder:
@@ -824,9 +864,6 @@ cdef _from_list(_DynamicListBuilder msg, list d):
cdef class _DynamicEnum:
cdef capnp.DynamicEnum thisptr
cdef public object _parent
cdef _init(self, capnp.DynamicEnum other, object parent):
self.thisptr = other
self._parent = parent
@@ -2341,9 +2378,6 @@ cdef class _StructSchemaField:
return '<field schema for %s>' % self.proto.name
cdef class _InterfaceSchema:
cdef C_InterfaceSchema thisptr
cdef object __method_names
cdef _init(self, C_InterfaceSchema other):
self.thisptr = other
return self

144
capnp/templates/module.pyx
View File

@@ -1,30 +1,122 @@
# addressbook_fast.pyx
# distutils: language = c++
# distutils: extra_compile_args = --std=c++11
# distutils: include_dirs = /usr/local/lib/python2.7/site-packages
# distutils: include_dirs = {{include_dir}}
# distutils: libraries = capnpc capnp capnp-rpc
# distutils: sources = {{file.filename}}.cpp
# cython: c_string_type = str
# cython: c_string_encoding = default
# cython: embedsignature = True
import capnp
{%- for file in code.requestedFiles %}
import {{file.filename | replace('.', '_')}}
{% endfor %}
from capnp.includes.capnp_cpp cimport DynamicValue
from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldWithField
{%- for node in code.nodes %}
{{node.schema}} = {{node.module_path}}.schema
from libcpp cimport bool as cbool
from capnp cimport helpers
from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr
from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema
from capnp.helpers.non_circular cimport reraise_kj_exception
cdef DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value):
return DynamicValue.Reader(value.thisptr.asReader())
cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader value):
return DynamicValue.Reader(value.thisptr)
cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value):
return DynamicValue.Reader(value.thisptr)
cdef _from_dict(_DynamicStructBuilder msg, dict d):
for key, val in d.iteritems():
if key != 'which':
try:
msg._set(key, val)
except Exception as e:
if 'expected isSetInUnion(field)' in str(e):
msg.init(key)
msg._set(key, val)
cdef _from_list(_DynamicListBuilder msg, list d):
cdef size_t count = 0
for val in d:
msg._set(count, val)
count += 1
cdef DynamicValue.Reader to_dynamic_value(value):
cdef DynamicValue.Reader temp
cdef StringPtr temp_string
value_type = type(value)
if value_type is int or value_type is long:
if value < 0:
temp = DynamicValue.Reader(<long long>value)
else:
temp = DynamicValue.Reader(<unsigned long long>value)
elif value_type is float:
temp = DynamicValue.Reader(<double>value)
elif value_type is bool:
temp = DynamicValue.Reader(<cbool>value)
elif value_type is bytes:
temp_string = StringPtr(<char*>value, len(value))
temp = DynamicValue.Reader(temp_string)
elif isinstance(value, basestring):
encoded_value = value.encode()
temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
temp = DynamicValue.Reader(temp_string)
elif value is None:
temp = DynamicValue.Reader(VOID)
elif value_type is _DynamicStructBuilder:
temp = _extract_dynamic_struct_builder(value)
elif value_type is _DynamicStructReader:
temp = _extract_dynamic_struct_reader(value)
elif value_type is _DynamicEnum:
temp = _extract_dynamic_enum(value)
else:
raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value))))
return temp
cdef extern from "{{file.filename}}.h":
{%- for node in code.nodes %}
Schema get{{node.module_name}}Schema"capnp::Schema::from<{{node.c_module_path}}>"()
cdef cppclass {{node.module_name}}"{{node.c_module_path}}":
cppclass Reader:
{%- for field in node.struct.fields %}
cdef _StructSchemaField {{node.module_name}}_{{field.name}} = {{node.schema}}.fields['{{field.name}}']
DynamicValue.Reader get{{field.c_name}}()
{%- endfor %}
cppclass Builder:
{%- for field in node.struct.fields %}
DynamicValue.Builder get{{field.c_name}}()
set{{field.c_name}}(DynamicValue.Reader)
{%- endfor %}
{%- endfor %}
cdef cppclass C_DynamicStruct_Reader" ::capnp::DynamicStruct::Reader":
{%- for node in code.nodes %}
{{node.module_name}}.Reader as{{node.module_name}}"as<{{node.c_module_path}}>"()
{%- endfor %}
cdef cppclass C_DynamicStruct_Builder" ::capnp::DynamicStruct::Builder":
{%- for node in code.nodes %}
{{node.module_name}}.Builder as{{node.module_name}}"as<{{node.c_module_path}}>"()
{%- endfor %}
{%- for node in code.nodes %}
{{node.schema}} = _Schema()._init(get{{node.module_name}}Schema()).as_struct()
{{node.module_path}}.schema = {{node.schema}}
cdef class {{node.module_name}}_Reader(_DynamicStructReader):
cdef {{node.module_name}}.Reader thisptr_child
def __init__(self, _DynamicStructReader struct):
self._init(struct.thisptr, struct._parent, struct.is_root, False)
self.thisptr_child = (<C_DynamicStruct_Reader>struct.thisptr).as{{node.module_name}}()
{% for field in node.struct.fields %}
cpdef _get_{{field.name}}(self):
cdef DynamicValue.Reader temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr)
cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
cdef DynamicValue.Reader temp = self.thisptr_child.get{{field.c_name}}()
return to_python_reader(temp, self._parent)
property {{field.name}}:
def __get__(self):
@@ -32,21 +124,43 @@ cdef class {{node.module_name}}_Reader(_DynamicStructReader):
{%- endfor %}
def to_dict(self, verbose=False):
return {
ret = {
{% for field in node.struct.fields %}
{% if field.discriminantValue == 65535 %}
'{{field.name}}': _to_dict(self.{{field.name}}, verbose),
{% endif %}
{%- endfor %}
}
{% if node.is_union %}
which = self.which()
ret[which] = getattr(self, which)
{% endif %}
return ret
cdef class {{node.module_name}}_Builder(_DynamicStructBuilder):
cdef {{node.module_name}}.Builder thisptr_child
def __init__(self, _DynamicStructBuilder struct):
self._init(struct.thisptr, struct._parent, struct.is_root, False)
self.thisptr_child = (<C_DynamicStruct_Builder>struct.thisptr).as{{node.module_name}}()
{% for field in node.struct.fields %}
cpdef _get_{{field.name}}(self):
cdef DynamicValue.Builder temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr)
cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
cdef DynamicValue.Builder temp = self.thisptr_child.get{{field.c_name}}()
return to_python_builder(temp, self._parent)
cpdef _set_{{field.name}}(self, value):
_setDynamicFieldWithField(self.thisptr, {{node.module_name}}_{{field.name}}, value, self._parent)
cpdef _set_{{field.name}}(self, value) except +reraise_kj_exception:
_setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent)
# cdef DynamicValue.Builder temp
# value_type = type(value)
# if value_type is list:
# builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
# _from_list(builder, value)
# elif value_type is dict:
# builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
# _from_dict(builder, value)
# else:
# self.thisptr_child.set{{field.c_name}}(to_dynamic_value(value))
property {{field.name}}:
def __get__(self):
return self._get_{{field.name}}()

17
capnp/templates/setup.py
View File

@@ -2,6 +2,23 @@
from distutils.core import setup
from Cython.Build import cythonize
import os
import re
files = [{% for file in code.requestedFiles %}"{{file.filename}}",{% endfor %}]
for f in files:
cpp_file = f + '.cpp'
if not os.path.exists(cpp_file):
if not os.path.exists(f + '.c++'):
raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.")
os.rename(f + '.c++', cpp_file)
with open(f + '.h', "r") as file:
lines = file.readlines()
with open(f + '.h', "w") as file:
for line in lines:
file.write(re.sub(r'Builder\(\)\s*=\s*delete;', 'Builder() = default;', line))
setup(
name="{{code.requestedFiles[0] | replace('.', '_')}}",