From 7b8de8bbfe232d8eec9b91c567daca3319a55218 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sat, 6 Jul 2013 16:53:00 -0700 Subject: [PATCH] Added a load function to the capnp module It takes .capnp file and will load it through the C++ capnproto library. Only major thing left is initializing lists and structs in MessageBuilder --- bootstrap/capnp.tmpl.pyx | 4 +- capnp.pyx | 143 ++++++++++++++++++++++++++++++++------- capnp_cpp.pxd | 44 +++++++++++- schema.pxd | 9 +++ schema.pyx | 6 +- setup.py | 4 +- 6 files changed, 179 insertions(+), 31 deletions(-) create mode 100644 schema.pxd diff --git a/bootstrap/capnp.tmpl.pyx b/bootstrap/capnp.tmpl.pyx index 126809c..a4b4a10 100644 --- a/bootstrap/capnp.tmpl.pyx +++ b/bootstrap/capnp.tmpl.pyx @@ -208,10 +208,12 @@ from types import ModuleType {%- for node_name, node_dict in nodes.items() recursive %} temp = {{ node_dict['full_name'] }} = ModuleType('{{ node_dict['full_name'] }}') -{%- if node_dict['body'] != 'enum' %} +{%- if node_dict['body'] == 'struct' %} temp.Reader = {{ node_dict['full_name_cython'] }}Reader temp.Builder = {{ node_dict['full_name_cython'] }}Builder {%- elif node_dict['body'] == 'union' %} +temp.Reader = {{ node_dict['full_name_cython'] }}Reader +temp.Builder = {{ node_dict['full_name_cython'] }}Builder temp.Which = {{ node_dict['full_name_cython'] }}_Which {%- else %} {{ node_dict['full_name'] }} = {{ node_dict['full_name_cython'] }} diff --git a/capnp.pyx b/capnp.pyx index 205f10d..e862983 100644 --- a/capnp.pyx +++ b/capnp.pyx @@ -7,9 +7,11 @@ cimport cython cimport capnp_cpp as capnp cimport schema_cpp from capnp_cpp cimport SchemaLoader as C_SchemaLoader, Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, DynamicUnion as C_DynamicUnion, fixMaybe -from schema_cpp cimport CodeGeneratorRequest as C_CodeGeneratorRequest, Node as C_Node + +from schema_cpp cimport CodeGeneratorRequest as C_CodeGeneratorRequest, Node as C_Node, EnumNode as C_EnumNode from cython.operator cimport dereference as deref +from schema cimport _NodeReader from libc.stdint cimport * ctypedef unsigned int uint ctypedef uint8_t UInt8 @@ -32,6 +34,7 @@ ctypedef fused valid_values: float double bint + cython.p_char def _make_enum(enum_name, *sequential, **named): enums = dict(zip(sequential, range(len(sequential))), **named) @@ -242,12 +245,34 @@ cdef class _DynamicStructBuilder: def __getattr__(self, field): return toPython(self.thisptr.get(field)) - def _setattr(self, field, valid_values value): - cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + cdef _setattrInt(self, field, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(field, temp) + + cdef _setattrDouble(self, field, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(field, temp) + + cdef _setattrBool(self, field, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) + self.thisptr.set(field, temp) + + cdef _setattrString(self, field, value): + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(value) self.thisptr.set(field, temp) def __setattr__(self, field, value): - self._setattr(field, value) + value_type = type(value) + if value_type is int: + self._setattrInt(field, value) + elif value_type is float: + self._setattrDouble(field, value) + elif value_type is bool: + self._setattrBool(field, value) + elif value_type is bytes: + self._setattrString(field, value) + else: + raise ValueError("Non primitive type") def _has(self, field): return self.thisptr.has(field) @@ -299,21 +324,6 @@ cdef class _CodeGeneratorRequestReader: def __get__(self): return _List_UInt64_Reader().init(self.thisptr.getRequestedFiles()) -cdef class _NodeReader: - cdef C_Node.Reader thisptr - cdef init(self, C_Node.Reader other): - self.thisptr = other - return self - property displayName: - def __get__(self): - return self.thisptr.getDisplayName().cStr() - property scopeId: - def __get__(self): - return self.thisptr.getScopeId() - property id: - def __get__(self): - return self.thisptr.getId() - cdef class Schema: cdef C_Schema thisptr cdef init(self, C_Schema other): @@ -323,6 +333,12 @@ cdef class Schema: cpdef asStruct(self): return StructSchema().init(self.thisptr.asStruct()) + cpdef getDependency(self, id): + return Schema().init(self.thisptr.getDependency(id)) + + cpdef getProto(self): + return _NodeReader().init(self.thisptr.getProto()) + cdef class StructSchema: cdef C_StructSchema thisptr cdef init(self, C_StructSchema other): @@ -340,18 +356,21 @@ cdef class SchemaLoader: cpdef load(self, _NodeReader node): return Schema().init(self.thisptr.load(node.thisptr)) + cpdef get(self, id): + return Schema().init(self.thisptr.get(id)) + cdef class MessageBuilder: cdef schema_cpp.MessageBuilder * thisptr def __dealloc__(self): del self.thisptr + cpdef initRoot(self, StructSchema schema): + return _DynamicStructBuilder().init(self.thisptr.initRootDynamicStruct(schema.thisptr)) + cpdef getRoot(self, StructSchema schema): + return _DynamicStructBuilder().init(self.thisptr.getRootDynamicStruct(schema.thisptr)) cdef class MallocMessageBuilder(MessageBuilder): def __cinit__(self): self.thisptr = new schema_cpp.MallocMessageBuilder() - cpdef initRootDynamicStruct(self, StructSchema schema): - return _DynamicStructBuilder().init(self.thisptr.initRootDynamicStruct(schema.thisptr)) - cpdef getRootDynamicStruct(self, StructSchema schema): - return _DynamicStructBuilder().init(self.thisptr.getRootDynamicStruct(schema.thisptr)) cdef class MessageReader: cdef schema_cpp.MessageReader * thisptr @@ -363,6 +382,8 @@ cdef class MessageReader: return _CodeGeneratorRequestReader().init(self.thisptr.getRootCodeGeneratorRequest()) cpdef getRootDynamicStruct(self, StructSchema schema): return _DynamicStructReader().init(self.thisptr.getRootDynamicStruct(schema.thisptr)) + cpdef getRoot(self, StructSchema schema): + return _DynamicStructReader().init(self.thisptr.getRootDynamicStruct(schema.thisptr)) cdef class StreamFdMessageReader(MessageReader): def __cinit__(self, int fd): @@ -375,4 +396,78 @@ cdef class PackedFdMessageReader(MessageReader): def writeMessageToFd(int fd, MessageBuilder m): schema_cpp.writeMessageToFd(fd, deref(m.thisptr)) def writePackedMessageToFd(int fd, MessageBuilder m): - schema_cpp.writePackedMessageToFd(fd, deref(m.thisptr)) \ No newline at end of file + schema_cpp.writePackedMessageToFd(fd, deref(m.thisptr)) + +def capitalize(s): + if len(s) < 2: + return s + return s[0].upper() + s[1:] +def upper_and_under(s): + if len(s) < 2: + return s + ret = [s[0]] + for letter in s[1:]: + if letter.isupper(): + ret.append('_') + ret.append(letter) + return ''.join(ret).upper() + +def make_enum(enum_name, *sequential, **named): + enums = dict(zip(sequential, range(len(sequential))), **named) + reverse = dict((value, key) for key, value in enums.iteritems()) + enums['reverse_mapping'] = reverse + return type(enum_name, (), enums) + +from types import ModuleType +import re +import schema +import subprocess + +def _load(m, n, loader, name = None, isUnion = False): + if name is None: + name = n.displayName + local_m = m + for sub_name in re.split('[:.]', name): + new_m = local_m.__dict__.get(sub_name, ModuleType(sub_name)) + local_m.__dict__[sub_name] = new_m + local_m = new_m + for nestedNode in n.nestedNodes: + s = loader.get(nestedNode.id) + _load(m, s.getProto(), loader, name + ':' + nestedNode.name) + body = n.body + which = body.which() + if which == schema.Node.Body.Which.enumNode: + enum = body.enumNode + local_m.Which = make_enum(name+':Which', **{upper_and_under(e.name) : e.codeOrder for e in enum.enumerants}) + elif which == schema.Node.Body.Which.structNode: + struct = body.structNode + for member in struct.members: + if member.body.which() == schema.StructNode.Member.Body.Which.unionMember: + sub_name = capitalize(member.name) + new_m = local_m.__dict__.get(sub_name, ModuleType(sub_name)) + local_m.__dict__[sub_name] = new_m + new_m.Which = make_enum(sub_name+':Which', **{upper_and_under(e.name) : e.ordinal for e in struct.members}) + return local_m + +def load(file_name, cat_path='/bin/cat'): + p = subprocess.Popen(['capnpc', '-o'+cat_path, file_name], stdout=subprocess.PIPE) + ret = p.wait() + if ret != 0: + raise RuntimeError("capnpc failed for some reason") + r = schema.StreamFdMessageReader(p.stdout.fileno()) + c = r.getRootCodeGeneratorRequest() + m = ModuleType(file_name) + l = SchemaLoader() + m._loader = l + for node in c.nodes: + s = l.load(node) + for node in c.nodes: + s = l.load(node) + local_m = _load(m, node, l) + try: + s = s.asStruct() + local_m.Schema = s + except: pass + + return m + diff --git a/capnp_cpp.pxd b/capnp_cpp.pxd index d61cab4..f16a8e5 100644 --- a/capnp_cpp.pxd +++ b/capnp_cpp.pxd @@ -2,7 +2,7 @@ # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 # distutils: libraries = capnp -from schema_cpp cimport Node, Data, StructNode +from schema_cpp cimport Node, Data, StructNode, EnumNode from libc.stdint cimport * ctypedef unsigned int uint @@ -14,16 +14,54 @@ cdef extern from "kj/common.h" namespace "::kj": cdef extern from "capnp/schema.h" namespace "::capnp": cdef cppclass Schema: Node.Reader getProto() - StructSchema asStruct() + StructSchema asStruct() except + + EnumSchema asEnum() except + + Schema getDependency(uint64_t id) except + + #InterfaceSchema asInterface() const; + + cdef cppclass MemberForward"::capnp::StructSchema::Member": + pass + cdef cppclass StructSchema(Schema): + cppclass MemberList: + uint size() + MemberForward operator[](uint index) + + cppclass Union: + StructNode.Union.Reader getProto() + MemberList getMembers() + MemberForward getMemberByName(char * name) + cppclass Member: StructNode.Member.Reader getProto() - StructNode.Reader getProto() + StructSchema getContainingStruct() + uint getIndex() + MemberList getMembers() + Union asUnion() except + + + Node.Reader getProto() + MemberList getMembers() + Member getMemberByName(char * name) + + cdef cppclass EnumSchema: + cppclass Enumerant: + EnumNode.Enumerant.Reader getProto() + EnumSchema getContainingEnum() + uint16_t getOrdinal() + + cppclass EnumerantList: + uint size() + Enumerant operator[](uint index) + + EnumerantList getEnumerants() + Enumerant getEnumerantByName(char * name) + Node.Reader getProto() cdef extern from "capnp/schema-loader.h" namespace "::capnp": cdef cppclass SchemaLoader: SchemaLoader() Schema load(Node.Reader &) except + + Schema get(uint64_t id) except + cdef extern from "capnp/dynamic.h" namespace "::capnp": cdef cppclass DynamicValueForward"::capnp::DynamicValue": diff --git a/schema.pxd b/schema.pxd new file mode 100644 index 0000000..47e7323 --- /dev/null +++ b/schema.pxd @@ -0,0 +1,9 @@ +# schema.capnp.cpp.pyx +# distutils: language = c++ +# distutils: extra_compile_args = --std=c++11 +# distutils: libraries = capnp +from schema_cpp cimport Node as C_Node +cdef class _NodeReader: + cdef C_Node.Reader thisptr + cdef init(self, C_Node.Reader other) + \ No newline at end of file diff --git a/schema.pyx b/schema.pyx index 6edcdab..d87136f 100644 --- a/schema.pyx +++ b/schema.pyx @@ -1156,7 +1156,6 @@ cdef class _Node_NestedNodeBuilder: def __set__(self, val): self.thisptr.setId(val) cdef class _NodeReader: - cdef C_Node.Reader thisptr cdef init(self, C_Node.Reader other): self.thisptr = other return self @@ -1733,6 +1732,7 @@ temp.Builder = _ValueBuilder temp = Value.Body = ModuleType('Value.Body') temp.Reader = _Value_BodyReader temp.Builder = _Value_BodyBuilder +temp.Which = _Value_Body_Which temp = ConstNode = ModuleType('ConstNode') temp.Reader = _ConstNodeReader @@ -1745,6 +1745,7 @@ temp.Builder = _TypeBuilder temp = Type.Body = ModuleType('Type.Body') temp.Reader = _Type_BodyReader temp.Builder = _Type_BodyBuilder +temp.Which = _Type_Body_Which temp = FileNode = ModuleType('FileNode') temp.Reader = _FileNodeReader @@ -1761,6 +1762,7 @@ temp.Builder = _NodeBuilder temp = Node.Body = ModuleType('Node.Body') temp.Reader = _Node_BodyReader temp.Builder = _Node_BodyBuilder +temp.Which = _Node_Body_Which temp = Node.NestedNode = ModuleType('Node.NestedNode') temp.Reader = _Node_NestedNodeReader @@ -1793,6 +1795,7 @@ temp.Builder = _StructNode_MemberBuilder temp = StructNode.Member.Body = ModuleType('StructNode.Member.Body') temp.Reader = _StructNode_Member_BodyReader temp.Builder = _StructNode_Member_BodyBuilder +temp.Which = _StructNode_Member_Body_Which temp = StructNode.Field = ModuleType('StructNode.Field') temp.Reader = _StructNode_FieldReader @@ -1802,3 +1805,4 @@ temp = Annotation = ModuleType('Annotation') temp.Reader = _AnnotationReader temp.Builder = _AnnotationBuilder + diff --git a/setup.py b/setup.py index c297202..98c75af 100644 --- a/setup.py +++ b/setup.py @@ -3,5 +3,5 @@ from distutils.core import setup from Cython.Build import cythonize setup( name = "capnp", - ext_modules = cythonize('capnp.pyx'), -) \ No newline at end of file + ext_modules = cythonize('*.pyx'), +)