Speed up code generated modules even more.

Switch to using generated c++ capnp code

capnp/_gen.py

@@ -4,6 +4,7 @@ import capnp
 import schema_capnp
 import sys
 from jinja2 import Environment, PackageLoader
+import os
 
 def main():
   env = Environment(loader=PackageLoader('capnp', 'templates'))
@@ -11,24 +12,31 @@ def main():
 
   code = schema_capnp.CodeGeneratorRequest.read(sys.stdin)
   code=code.to_dict()
-  code['nodes'] = [node for node in code['nodes'] if 'struct' in node]
+  code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0]
   for node in code['nodes']:
     displayName = node['displayName']
     parent, path = displayName.split(':')
     node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')])
     node['module_name'] = path.replace('.', '_')
+    node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')])
     node['schema'] = '_{}_Schema'.format(node['module_name'])
     is_union = False
     for field in node['struct']['fields']:
       if field['discriminantValue'] != 65535:
         is_union = True
+      field['c_name'] = field['name'][0].upper() + field['name'][1:]
     node['is_union'] = is_union
 
+  include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..'))
   module = env.get_template('module.pyx')
-  filename = code['requestedFiles'][0]['filename'].replace('.', '_') + '_cython.pyx'
+
-  # TODO: handle multiple files
+  for f in code['requestedFiles']:
-  with open(filename, 'w') as out:
+    filename = f['filename'].replace('.', '_') + '_cython.pyx'
-    out.write(module.render(code=code))
+
+    file_code = dict(code)
+    file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])]
+    with open(filename, 'w') as out:
+      out.write(module.render(code=file_code, file=f, include_dir=include_dir))
 
   setup = env.get_template('setup.py')
   with open('setup_capnp.py', 'w') as out:

capnp/lib/capnp.pxd

@@ -83,7 +83,34 @@ cdef class _Schema:
     cpdef get_dependency(self, id)
     cpdef get_proto(self)
 
+cdef class _InterfaceSchema:
+    cdef C_InterfaceSchema thisptr
+    cdef object __method_names
+    cdef _init(self, C_InterfaceSchema other)
+    cpdef get_dependency(self, id)
+
+cdef class _DynamicEnum:
+    cdef capnp.DynamicEnum thisptr
+    cdef public object _parent
+
+    cdef _init(self, capnp.DynamicEnum other, object parent)
+    cpdef _as_str(self)
+
+cdef class _DynamicListBuilder:
+    cdef C_DynamicList.Builder thisptr
+    cdef public object _parent
+    cdef _init(self, C_DynamicList.Builder other, object parent)
+
+    cdef _get(self, index)
+    cdef _set(self, index, value)
+
+    cpdef adopt(self, index, _DynamicOrphan orphan)
+    cpdef disown(self, index)
+
 cdef to_python_reader(C_DynamicValue.Reader self, object parent)
 cdef to_python_builder(C_DynamicValue.Builder self, object parent)
 cdef _to_dict(msg, bint verbose)
+cdef _from_dict(_DynamicStructBuilder msg, dict d)
+cdef _from_list(_DynamicListBuilder msg, list d)
 cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent)
+cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)

capnp/lib/capnp.pyx

@@ -495,8 +495,6 @@ cdef class _DynamicListBuilder:
         for phone in phones:
             print phone.number
     """
-    cdef C_DynamicList.Builder thisptr
-    cdef public object _parent
     cdef _init(self, C_DynamicList.Builder other, object parent):
         self.thisptr = other
         self._parent = parent
@@ -778,6 +776,48 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField
     else:
         raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
 
+cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent):
+    cdef C_DynamicValue.Reader temp
+    value_type = type(value)
+
+    if value_type is int or value_type is long:
+        if value < 0:
+           temp = C_DynamicValue.Reader(<long long>value)
+        else:
+           temp = C_DynamicValue.Reader(<unsigned long long>value)
+        thisptr.set(field, temp)
+    elif value_type is float:
+        temp = C_DynamicValue.Reader(<double>value)
+        thisptr.set(field, temp)
+    elif value_type is bool:
+        temp = C_DynamicValue.Reader(<cbool>value)
+        thisptr.set(field, temp)
+    elif value_type is bytes:
+        _setBytes(thisptr, field, value)
+    elif isinstance(value, basestring):
+        _setBaseString(thisptr, field, value)
+    elif value_type is list:
+        builder = to_python_builder(thisptr.init(field, len(value)), parent)
+        _from_list(builder, value)
+    elif value_type is dict:
+        builder = to_python_builder(thisptr.get(field), parent)
+        _from_dict(builder, value)
+    elif value is None:
+        temp = C_DynamicValue.Reader(VOID)
+        thisptr.set(field, temp)
+    elif value_type is _DynamicStructBuilder:
+        thisptr.set(field, _extract_dynamic_struct_builder(value))
+    elif value_type is _DynamicStructReader:
+        thisptr.set(field, _extract_dynamic_struct_reader(value))
+    elif value_type is _DynamicCapabilityClient:
+        thisptr.set(field, _extract_dynamic_client(value))
+    elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer):
+        thisptr.set(field, _extract_dynamic_server(value))
+    elif value_type is _DynamicEnum:
+        thisptr.set(field, _extract_dynamic_enum(value))
+    else:
+        raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
+
 cdef _to_dict(msg, bint verbose):
     msg_type = type(msg)
     if msg_type is _DynamicListBuilder or msg_type is _DynamicListReader or msg_type is _DynamicResizableListBuilder:
@@ -824,9 +864,6 @@ cdef _from_list(_DynamicListBuilder msg, list d):
 
 
 cdef class _DynamicEnum:
-    cdef capnp.DynamicEnum thisptr
-    cdef public object _parent
-
     cdef _init(self, capnp.DynamicEnum other, object parent):
         self.thisptr = other
         self._parent = parent
@@ -2341,9 +2378,6 @@ cdef class _StructSchemaField:
         return '<field schema for %s>' % self.proto.name
 
 cdef class _InterfaceSchema:
-    cdef C_InterfaceSchema thisptr
-    cdef object __method_names
-
     cdef _init(self, C_InterfaceSchema other):
         self.thisptr = other
         return self

capnp/templates/module.pyx

@@ -1,30 +1,122 @@
 # addressbook_fast.pyx
 # distutils: language = c++
 # distutils: extra_compile_args = --std=c++11
-# distutils: include_dirs = /usr/local/lib/python2.7/site-packages
+# distutils: include_dirs = {{include_dir}}
+# distutils: libraries = capnpc capnp capnp-rpc
+# distutils: sources = {{file.filename}}.cpp
 # cython: c_string_type = str
 # cython: c_string_encoding = default
 # cython: embedsignature = True
 
 import capnp
-{%- for file in code.requestedFiles %}
 import {{file.filename | replace('.', '_')}}
-{% endfor %}
-from capnp.includes.capnp_cpp cimport DynamicValue
-from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldWithField
 
-{%- for node in code.nodes %}
+from libcpp cimport bool as cbool
-{{node.schema}} = {{node.module_path}}.schema
+from capnp cimport helpers
-    {%- for field in node.struct.fields %}
+from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr
-cdef _StructSchemaField {{node.module_name}}_{{field.name}} = {{node.schema}}.fields['{{field.name}}']
+from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema
+
+from capnp.helpers.non_circular cimport reraise_kj_exception
+
+cdef DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value):
+    return DynamicValue.Reader(value.thisptr.asReader())
+
+cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader value):
+    return DynamicValue.Reader(value.thisptr)
+
+cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value):
+    return DynamicValue.Reader(value.thisptr)
+
+cdef _from_dict(_DynamicStructBuilder msg, dict d):
+    for key, val in d.iteritems():
+        if key != 'which':
+            try:
+                msg._set(key, val)
+            except Exception as e:
+                if 'expected isSetInUnion(field)' in str(e):
+                    msg.init(key)
+                    msg._set(key, val)
+
+cdef _from_list(_DynamicListBuilder msg, list d):
+    cdef size_t count = 0
+    for val in d:
+        msg._set(count, val)
+        count += 1
+
+cdef DynamicValue.Reader to_dynamic_value(value):
+    cdef DynamicValue.Reader temp
+    cdef StringPtr temp_string
+    value_type = type(value)
+
+    if value_type is int or value_type is long:
+        if value < 0:
+           temp = DynamicValue.Reader(<long long>value)
+        else:
+           temp = DynamicValue.Reader(<unsigned long long>value)
+    elif value_type is float:
+        temp = DynamicValue.Reader(<double>value)
+    elif value_type is bool:
+        temp = DynamicValue.Reader(<cbool>value)
+    elif value_type is bytes:
+        temp_string = StringPtr(<char*>value, len(value))
+        temp = DynamicValue.Reader(temp_string)
+    elif isinstance(value, basestring):
+        encoded_value = value.encode()
+        temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
+        temp = DynamicValue.Reader(temp_string)
+    elif value is None:
+        temp = DynamicValue.Reader(VOID)
+    elif value_type is _DynamicStructBuilder:
+        temp = _extract_dynamic_struct_builder(value)
+    elif value_type is _DynamicStructReader:
+        temp = _extract_dynamic_struct_reader(value)
+    elif value_type is _DynamicEnum:
+        temp = _extract_dynamic_enum(value)
+    else:
+        raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value))))
+
+    return temp
+
+
+cdef extern from "{{file.filename}}.h":
+    {%- for node in code.nodes %}
+    Schema get{{node.module_name}}Schema"capnp::Schema::from<{{node.c_module_path}}>"()
+
+    cdef cppclass {{node.module_name}}"{{node.c_module_path}}":
+        cppclass Reader:
+        {%- for field in node.struct.fields %}
+            DynamicValue.Reader get{{field.c_name}}()
+        {%- endfor %}
+        cppclass Builder:
+        {%- for field in node.struct.fields %}
+            DynamicValue.Builder get{{field.c_name}}()
+            set{{field.c_name}}(DynamicValue.Reader)
+        {%- endfor %}
     {%- endfor %}
 
+    cdef cppclass C_DynamicStruct_Reader" ::capnp::DynamicStruct::Reader":
+    {%- for node in code.nodes %}
+        {{node.module_name}}.Reader as{{node.module_name}}"as<{{node.c_module_path}}>"()
+    {%- endfor %}
+
+    cdef cppclass C_DynamicStruct_Builder" ::capnp::DynamicStruct::Builder":
+    {%- for node in code.nodes %}
+        {{node.module_name}}.Builder as{{node.module_name}}"as<{{node.c_module_path}}>"()
+    {%- endfor %}
+
+{%- for node in code.nodes %}
+
+{{node.schema}} = _Schema()._init(get{{node.module_name}}Schema()).as_struct()
+{{node.module_path}}.schema = {{node.schema}}
+
 cdef class {{node.module_name}}_Reader(_DynamicStructReader):
+    cdef {{node.module_name}}.Reader thisptr_child
     def __init__(self, _DynamicStructReader struct):
         self._init(struct.thisptr, struct._parent, struct.is_root, False)
+        self.thisptr_child = (<C_DynamicStruct_Reader>struct.thisptr).as{{node.module_name}}()
     {% for field in node.struct.fields %}
-    cpdef _get_{{field.name}}(self):
+    cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
-        cdef DynamicValue.Reader temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr)
+        cdef DynamicValue.Reader temp = self.thisptr_child.get{{field.c_name}}()
         return to_python_reader(temp, self._parent)
     property {{field.name}}:
         def __get__(self):
@@ -32,21 +124,43 @@ cdef class {{node.module_name}}_Reader(_DynamicStructReader):
     {%- endfor %}
 
     def to_dict(self, verbose=False):
-        return {
+        ret = {
         {% for field in node.struct.fields %}
+        {% if field.discriminantValue == 65535 %}
         '{{field.name}}': _to_dict(self.{{field.name}}, verbose),
+        {% endif %}
         {%- endfor %}
         }
 
+        {% if node.is_union %}
+        which = self.which()
+        ret[which] = getattr(self, which)
+        {% endif %}
+
+        return ret
+
 cdef class {{node.module_name}}_Builder(_DynamicStructBuilder):
+    cdef {{node.module_name}}.Builder thisptr_child
     def __init__(self, _DynamicStructBuilder struct):
         self._init(struct.thisptr, struct._parent, struct.is_root, False)
+        self.thisptr_child = (<C_DynamicStruct_Builder>struct.thisptr).as{{node.module_name}}()
     {% for field in node.struct.fields %}
-    cpdef _get_{{field.name}}(self):
+    cpdef _get_{{field.name}}(self) except +reraise_kj_exception:
-        cdef DynamicValue.Builder temp = self.thisptr.getByField({{node.module_name}}_{{field.name}}.thisptr)
+        cdef DynamicValue.Builder temp = self.thisptr_child.get{{field.c_name}}()
         return to_python_builder(temp, self._parent)
-    cpdef _set_{{field.name}}(self, value):
+    cpdef _set_{{field.name}}(self, value) except +reraise_kj_exception:
-        _setDynamicFieldWithField(self.thisptr, {{node.module_name}}_{{field.name}}, value, self._parent)
+        _setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent)
+        # cdef DynamicValue.Builder temp
+        # value_type = type(value)
+        # if value_type is list:
+        #     builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
+        #     _from_list(builder, value)
+        # elif value_type is dict:
+        #     builder = to_python_builder(self.thisptr_child.get{{field.c_name}}(), self._parent)
+        #     _from_dict(builder, value)
+        # else:
+        #     self.thisptr_child.set{{field.c_name}}(to_dynamic_value(value))
+
     property {{field.name}}:
         def __get__(self):
             return self._get_{{field.name}}()

capnp/templates/setup.py

@@ -2,6 +2,23 @@
 from distutils.core import setup
 from Cython.Build import cythonize
 import os
+import re
+
+
+files = [{% for file in code.requestedFiles %}"{{file.filename}}",{% endfor %}]
+
+for f in files:
+  cpp_file = f + '.cpp'
+  if not os.path.exists(cpp_file):
+    if not os.path.exists(f + '.c++'):
+      raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.")
+    os.rename(f + '.c++', cpp_file)
+
+    with open(f + '.h', "r") as file:
+        lines = file.readlines()
+    with open(f + '.h', "w") as file:
+        for line in lines:
+            file.write(re.sub(r'Builder\(\)\s*=\s*delete;', 'Builder() = default;', line))
 
 setup(
     name="{{code.requestedFiles[0] | replace('.', '_')}}",