From f411fc216acbf2e675e1a6d4a7c5d7f8ed645970 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 13 Aug 2013 22:21:38 -0700 Subject: [PATCH] Use SchemaParser API to remove dependency on subprocess call to compiler --- .travis.yml | 3 +- capnp/capnp.pyx | 131 +++++++++++++++++++++++++++++++------------- capnp/capnp_cpp.pxd | 22 +++++++- 3 files changed, 113 insertions(+), 43 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80da412..94c0382 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ python: compiler: gcc before_install: + - sudo apt-get install autoconf - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - sudo apt-get -qq update - sudo apt-get -qq install g++-4.8 libstdc++-4.8-dev @@ -17,7 +18,7 @@ before_install: --slave /usr/bin/g++ g++ /usr/bin/g++-4.8 --slave /usr/bin/gcov gcov /usr/bin/gcov-4.8 - sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 - - "curl -L https://gist.github.com/jparyani/6216515/raw/a50cb02574033a14f3856dac949f83b4c0557c82/install_capnproto.sh | /bin/bash" + - "curl -L https://gist.github.com/jparyani/6216515/raw/6ca2194faf253f2770de515764a5a0057aa86f48/install_capnproto.sh | /bin/bash" - pip install -U setuptools - pip install cython diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index 5e518e0..4d44c45 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -1,14 +1,14 @@ # capnp.pyx # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 -fpermissive -# distutils: libraries = capnp +# distutils: libraries = capnpc # cython: c_string_type = str # cython: c_string_encoding = default cimport cython cimport capnp_cpp as capnp cimport schema_cpp -from capnp_cpp cimport SchemaLoader as C_SchemaLoader, Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, DynamicUnion as C_DynamicUnion, fixMaybe, VOID +from capnp_cpp cimport SchemaLoader as C_SchemaLoader, Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, DynamicUnion as C_DynamicUnion, fixMaybe, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr from schema_cpp cimport CodeGeneratorRequest as C_CodeGeneratorRequest, Node as C_Node, EnumNode as C_EnumNode from cython.operator cimport dereference as deref @@ -83,6 +83,22 @@ cdef class _NodeReader: property id: def __get__(self): return self.thisptr.getId() + property nestedNodes: + def __get__(self): + return _List_NestedNode_Reader()._init(self.thisptr.getNestedNodes()) + +cdef class _NestedNodeReader: + cdef C_Node.NestedNode.Reader thisptr + cdef init(self, C_Node.NestedNode.Reader other): + self.thisptr = other + return self + + property name: + def __get__(self): + return self.thisptr.getName().cStr() + property id: + def __get__(self): + return self.thisptr.getId() cdef class _DynamicListReader: cdef C_DynamicList.Reader thisptr @@ -169,6 +185,21 @@ cdef class _List_Node_Reader: def __len__(self): return self.thisptr.size() +cdef class _List_NestedNode_Reader: + cdef List[C_Node.NestedNode].Reader thisptr + cdef _init(self, List[C_Node.NestedNode].Reader other): + self.thisptr = other + return self + def __getitem__(self, index): + size = self.thisptr.size() + if index >= size: + raise IndexError('Out of bounds') + index = index % size + return _NestedNodeReader().init(self.thisptr[index]) + + def __len__(self): + return self.thisptr.size() + cdef class _DynamicValueReader: cdef C_DynamicValue.Reader thisptr cdef public object _parent @@ -453,6 +484,24 @@ cdef class StructSchema: self.thisptr = other return self +cdef class ParsedSchema: + cdef C_ParsedSchema thisptr + cdef _init(self, C_ParsedSchema other): + self.thisptr = other + return self + + cpdef asStruct(self): + return StructSchema()._init(self.thisptr.asStruct()) + + cpdef getDependency(self, id): + return Schema()._init(self.thisptr.getDependency(id)) + + cpdef getProto(self): + return _NodeReader().init(self.thisptr.getProto()) + + cpdef getNested(self, name): + return ParsedSchema()._init(self.thisptr.getNested(name)) + cdef class SchemaLoader: cdef C_SchemaLoader * thisptr def __cinit__(self): @@ -467,6 +516,23 @@ cdef class SchemaLoader: cpdef get(self, id): return Schema()._init(self.thisptr.get(id)) +cdef class SchemaParser: + cdef C_SchemaParser * thisptr + def __cinit__(self): + self.thisptr = new C_SchemaParser() + + def __dealloc__(self): + del self.thisptr + + def parseDiskFile(self, displayName, diskPath): + cdef StringPtr * importArray = [] + cdef ArrayPtr[StringPtr] imports = ArrayPtr[StringPtr](importArray, 0) + + ret = ParsedSchema() + ret._init(self.thisptr.parseDiskFile(displayName, diskPath, imports)) + + return ret + cdef class MessageBuilder: cdef schema_cpp.MessageBuilder * thisptr def __dealloc__(self): @@ -544,48 +610,35 @@ def upper_and_under(s): return ''.join(ret).upper() from types import ModuleType -import re -import subprocess +import os -def _load(module, node, loader, name): - if name is None or len(name) == 0: # This only is true for the root fileNode - return - if name[0] == ':': - name = name[1:] - local_module = module - - for sub_name in re.split(u'[:.]', name): - new_m = local_module.__dict__.get(sub_name, ModuleType(sub_name)) - new_m._parent_module = local_module - local_module.__dict__[sub_name] = new_m - local_module = new_m - local_module._root_module = module +def _load(nodeSchema, module): + module._nodeSchema = nodeSchema + nodeProto = nodeSchema.getProto() + module._nodeProto = nodeProto - return local_module + for node in nodeProto.nestedNodes: + local_module = ModuleType(node.name) + module.__dict__[node.name] = local_module -def load(file_name, cat_path='/bin/cat'): - p = subprocess.Popen(['capnp', 'compile', '-o'+cat_path, file_name], stdout=subprocess.PIPE) - retcode = p.wait() - if retcode != 0: - raise RuntimeError("capnpc failed for some reason. Make sure `capnpc` is in your path, and that the path to cat (%s) is correct" % cat_path) - - reader = StreamFdMessageReader(p.stdout.fileno()) - request = reader.getRootCodeGeneratorRequest() - module = ModuleType(file_name) - loader = SchemaLoader() - - module._loader = loader - - for node in request.nodes: - loader.load(node) - - for node in request.nodes: - s = loader.load(node) - local_module = _load(module, node, loader, name = node.displayName.replace(file_name, '', 1)) + schema = nodeSchema.getNested(node.name) try: - s = s.asStruct() + s = schema.asStruct() local_module.Schema = s - except: pass # We only need to store away StructSchemas + except: pass + _load(schema, local_module) + + +def load(file_name, display_name=None): + if display_name is None: + display_name = os.path.basename(file_name) + module = ModuleType(display_name) + parser = SchemaParser() + + module._parser = parser + + fileSchema = parser.parseDiskFile(display_name, file_name) + _load(fileSchema, module) return module diff --git a/capnp/capnp_cpp.pxd b/capnp/capnp_cpp.pxd index 0c0a4a5..7a152b5 100644 --- a/capnp/capnp_cpp.pxd +++ b/capnp/capnp_cpp.pxd @@ -1,7 +1,7 @@ # schema.capnp.cpp.pyx # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 -# distutils: libraries = capnp +# distutils: libraries = capnpc from schema_cpp cimport Node, Data, StructNode, EnumNode from libc.stdint cimport * @@ -11,9 +11,18 @@ cdef extern from "capnp/common.h" namespace " ::capnp": enum Void: VOID " ::capnp::Void::VOID" -cdef extern from "kj/common.h" namespace "::kj": +cdef extern from "kj/string.h" namespace " ::kj": + cdef cppclass StringPtr: + StringPtr(char *) + +cdef extern from "kj/common.h" namespace " ::kj": cdef cppclass Maybe[T]: pass + cdef cppclass ArrayPtr[T]: + ArrayPtr() + ArrayPtr(T *, size_t size) + size_t size() + T& operator[](size_t index) cdef extern from "capnp/schema.h" namespace " ::capnp": cdef cppclass Schema: @@ -175,4 +184,11 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicStruct.Builder asStruct"as< ::capnp::DynamicStruct>"() DynamicUnion.Builder asUnion"as< ::capnp::DynamicUnion>"() DynamicEnum asEnum"as< ::capnp::DynamicEnum>"() - Data.Builder asData"as< ::capnp::Data>"() \ No newline at end of file + Data.Builder asData"as< ::capnp::Data>"() + +cdef extern from "capnp/schema-parser.h" namespace " ::capnp": + cdef cppclass ParsedSchema(Schema): + ParsedSchema getNested(char * name) except + + cdef cppclass SchemaParser: + SchemaParser() + ParsedSchema parseDiskFile(char * displayName, char * diskPath, ArrayPtr[StringPtr] importPath) except +