Use SchemaParser API to remove dependency on subprocess call to compiler

.travis.yml

@@ -10,6 +10,7 @@ python:
 compiler: gcc
 
 before_install:
+  - sudo apt-get install autoconf
   - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
   - sudo apt-get -qq update
   - sudo apt-get -qq install g++-4.8 libstdc++-4.8-dev
@@ -17,7 +18,7 @@ before_install:
                                      --slave   /usr/bin/g++  g++  /usr/bin/g++-4.8
                                      --slave   /usr/bin/gcov gcov /usr/bin/gcov-4.8
   - sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8
-  - "curl -L https://gist.github.com/jparyani/6216515/raw/a50cb02574033a14f3856dac949f83b4c0557c82/install_capnproto.sh | /bin/bash"
+  - "curl -L https://gist.github.com/jparyani/6216515/raw/6ca2194faf253f2770de515764a5a0057aa86f48/install_capnproto.sh | /bin/bash"
   - pip install -U setuptools
   - pip install cython
 

capnp/capnp.pyx

@@ -1,14 +1,14 @@
 # capnp.pyx
 # distutils: language = c++
 # distutils: extra_compile_args = --std=c++11 -fpermissive
-# distutils: libraries = capnp
+# distutils: libraries = capnpc
 # cython: c_string_type = str
 # cython: c_string_encoding = default
 
 cimport cython
 cimport capnp_cpp as capnp
 cimport schema_cpp
-from capnp_cpp cimport SchemaLoader as C_SchemaLoader, Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, DynamicUnion as C_DynamicUnion, fixMaybe, VOID
+from capnp_cpp cimport SchemaLoader as C_SchemaLoader, Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, DynamicUnion as C_DynamicUnion, fixMaybe, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr
 
 from schema_cpp cimport CodeGeneratorRequest as C_CodeGeneratorRequest, Node as C_Node, EnumNode as C_EnumNode
 from cython.operator cimport dereference as deref
@@ -83,6 +83,22 @@ cdef class _NodeReader:
     property id:
         def __get__(self):
             return self.thisptr.getId()
+    property nestedNodes:
+        def __get__(self):
+            return _List_NestedNode_Reader()._init(self.thisptr.getNestedNodes())
+
+cdef class _NestedNodeReader:
+    cdef C_Node.NestedNode.Reader thisptr
+    cdef init(self, C_Node.NestedNode.Reader other):
+        self.thisptr = other
+        return self
+
+    property name:
+        def __get__(self):
+            return self.thisptr.getName().cStr()
+    property id:
+        def __get__(self):
+            return self.thisptr.getId()
 
 cdef class _DynamicListReader:
     cdef C_DynamicList.Reader thisptr
@@ -169,6 +185,21 @@ cdef class _List_Node_Reader:
     def __len__(self):
         return self.thisptr.size()
 
+cdef class _List_NestedNode_Reader:
+    cdef List[C_Node.NestedNode].Reader thisptr
+    cdef _init(self, List[C_Node.NestedNode].Reader other):
+        self.thisptr = other
+        return self
+    def __getitem__(self, index):
+        size = self.thisptr.size()
+        if index >= size:
+            raise IndexError('Out of bounds')
+        index = index % size
+        return _NestedNodeReader().init(<C_Node.NestedNode.Reader>self.thisptr[index])
+
+    def __len__(self):
+        return self.thisptr.size()
+
 cdef class _DynamicValueReader:
     cdef C_DynamicValue.Reader thisptr
     cdef public object _parent
@@ -453,6 +484,24 @@ cdef class StructSchema:
         self.thisptr = other
         return self
 
+cdef class ParsedSchema:
+    cdef C_ParsedSchema thisptr
+    cdef _init(self, C_ParsedSchema other):
+        self.thisptr = other
+        return self
+
+    cpdef asStruct(self):
+        return StructSchema()._init(self.thisptr.asStruct())
+
+    cpdef getDependency(self, id):
+        return Schema()._init(self.thisptr.getDependency(id))
+
+    cpdef getProto(self):
+        return _NodeReader().init(self.thisptr.getProto())
+
+    cpdef getNested(self, name):
+        return ParsedSchema()._init(self.thisptr.getNested(name))
+
 cdef class SchemaLoader:
     cdef C_SchemaLoader * thisptr
     def __cinit__(self):
@@ -467,6 +516,23 @@ cdef class SchemaLoader:
     cpdef get(self, id):
         return Schema()._init(self.thisptr.get(id))
 
+cdef class SchemaParser:
+    cdef C_SchemaParser * thisptr
+    def __cinit__(self):
+        self.thisptr = new C_SchemaParser()
+
+    def __dealloc__(self):
+        del self.thisptr
+
+    def parseDiskFile(self, displayName, diskPath):
+        cdef StringPtr * importArray = []
+        cdef ArrayPtr[StringPtr] imports = ArrayPtr[StringPtr](importArray, <size_t>0)
+
+        ret = ParsedSchema()
+        ret._init(self.thisptr.parseDiskFile(displayName, diskPath, imports))
+
+        return ret
+
 cdef class MessageBuilder:
     cdef schema_cpp.MessageBuilder * thisptr
     def __dealloc__(self):
@@ -544,48 +610,35 @@ def upper_and_under(s):
   return ''.join(ret).upper()
 
 from types import ModuleType
-import re
+import os
-import subprocess
 
-def _load(module, node, loader, name):
+def _load(nodeSchema, module):
-    if name is None or len(name) == 0: # This only is true for the root fileNode
+    module._nodeSchema = nodeSchema
-        return
+    nodeProto = nodeSchema.getProto()
-    if name[0] == ':':
+    module._nodeProto = nodeProto
-        name = name[1:]
-    local_module = module
-    
-    for sub_name in re.split(u'[:.]', name):
-        new_m = local_module.__dict__.get(sub_name, ModuleType(sub_name))
-        new_m._parent_module = local_module
-        local_module.__dict__[sub_name] = new_m
-        local_module = new_m
-    local_module._root_module = module
 
-    return local_module
+    for node in nodeProto.nestedNodes:
+        local_module = ModuleType(node.name)
+        module.__dict__[node.name] = local_module
 
-def load(file_name, cat_path='/bin/cat'):
+        schema = nodeSchema.getNested(node.name)
-    p = subprocess.Popen(['capnp', 'compile', '-o'+cat_path, file_name], stdout=subprocess.PIPE)
-    retcode = p.wait()
-    if retcode != 0:
-        raise RuntimeError("capnpc failed for some reason. Make sure `capnpc` is in your path, and that the path to cat (%s) is correct" % cat_path)
-
-    reader = StreamFdMessageReader(p.stdout.fileno())
-    request = reader.getRootCodeGeneratorRequest()
-    module = ModuleType(file_name)
-    loader = SchemaLoader()
-
-    module._loader = loader
-    
-    for node in request.nodes:
-        loader.load(node)
-    
-    for node in request.nodes:
-        s = loader.load(node)
-        local_module = _load(module, node, loader, name = node.displayName.replace(file_name, '', 1))
         try:
-            s = s.asStruct()
+            s = schema.asStruct()
             local_module.Schema = s
-        except: pass # We only need to store away StructSchemas
+        except: pass
+        _load(schema, local_module)
+
+
+def load(file_name, display_name=None):
+    if display_name is None:
+        display_name = os.path.basename(file_name)
+    module = ModuleType(display_name)
+    parser = SchemaParser()
+
+    module._parser = parser
+
+    fileSchema = parser.parseDiskFile(display_name, file_name)
+    _load(fileSchema, module)
 
     return module
 

capnp/capnp_cpp.pxd

@@ -1,7 +1,7 @@
 # schema.capnp.cpp.pyx
 # distutils: language = c++
 # distutils: extra_compile_args = --std=c++11
-# distutils: libraries = capnp
+# distutils: libraries = capnpc
 from schema_cpp cimport Node, Data, StructNode, EnumNode
 
 from libc.stdint cimport *
@@ -11,9 +11,18 @@ cdef extern from "capnp/common.h" namespace " ::capnp":
     enum Void:
         VOID " ::capnp::Void::VOID"
 
-cdef extern from "kj/common.h" namespace "::kj":
+cdef extern from "kj/string.h" namespace " ::kj":
+    cdef cppclass StringPtr:
+        StringPtr(char *)
+
+cdef extern from "kj/common.h" namespace " ::kj":
     cdef cppclass Maybe[T]:
         pass
+    cdef cppclass ArrayPtr[T]:
+        ArrayPtr()
+        ArrayPtr(T *, size_t size)
+        size_t size()
+        T& operator[](size_t index)
 
 cdef extern from "capnp/schema.h" namespace " ::capnp":
     cdef cppclass Schema:
@@ -175,4 +184,11 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             DynamicStruct.Builder asStruct"as< ::capnp::DynamicStruct>"()
             DynamicUnion.Builder asUnion"as< ::capnp::DynamicUnion>"()
             DynamicEnum asEnum"as< ::capnp::DynamicEnum>"()
-            Data.Builder asData"as< ::capnp::Data>"()
+            Data.Builder asData"as< ::capnp::Data>"()
+
+cdef extern from "capnp/schema-parser.h" namespace " ::capnp":
+    cdef cppclass ParsedSchema(Schema):
+        ParsedSchema getNested(char * name) except +
+    cdef cppclass SchemaParser:
+        SchemaParser()
+        ParsedSchema parseDiskFile(char * displayName, char * diskPath, ArrayPtr[StringPtr] importPath) except +