From 448ea93891f2faa59454452745eba1268269b2a5 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 26 Aug 2013 22:00:12 -0700 Subject: [PATCH] Add full orphan functionality. Also, allow special orphan lists that can grow over time. --- benchmark/addressbook.capnp.orphan.py | 56 +++++++++ capnp/__init__.py | 2 +- capnp/capnp.pyx | 170 +++++++++++++++++++++++++- capnp/capnp_cpp.pxd | 6 + capnp/schema_cpp.pxd | 3 + docs/capnp.rst | 7 ++ examples/example_orphan.py | 64 ++++++++++ 7 files changed, 304 insertions(+), 4 deletions(-) create mode 100644 benchmark/addressbook.capnp.orphan.py create mode 100644 examples/example_orphan.py diff --git a/benchmark/addressbook.capnp.orphan.py b/benchmark/addressbook.capnp.orphan.py new file mode 100644 index 0000000..ff98574 --- /dev/null +++ b/benchmark/addressbook.capnp.orphan.py @@ -0,0 +1,56 @@ +from __future__ import print_function +import os +import capnp + +this_dir = os.path.dirname(__file__) +addressbook = capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + +print = lambda *x: x + + +def writeAddressBook(fd): + message = capnp.MallocMessageBuilder() + addressBook = message.initRoot(addressbook.AddressBook) + people = addressBook.init('people', 0) + + alice = people.add() + alice.id = 123 + alice.name = 'Alice' + alice.email = 'alice@example.com' + alicePhones = alice.init('phones', 1) + alicePhones[0].number = "555-1212" + alicePhones[0].type = 'mobile' + + bob = people.add() + bob.id = 456 + bob.name = 'Bob' + bob.email = 'bob@example.com' + bobPhones = bob.init('phones', 2) + bobPhones[0].number = "555-4567" + bobPhones[0].type = 'home' + bobPhones[1].number = "555-7654" + bobPhones[1].type = 'work' + + capnp.writePackedMessageToFd(fd, message) + + +def printAddressBook(fd): + message = capnp.PackedFdMessageReader(f.fileno()) + addressBook = message.getRoot(addressbook.AddressBook) + + for person in addressBook.people: + print(person.name, ':', person.email) + for phone in person.phones: + print(phone.type, ':', phone.number) + print() + + +if __name__ == '__main__': + for i in range(10000): + f = open('example', 'w') + writeAddressBook(f.fileno()) + + f = open('example', 'r') + printAddressBook(f.fileno()) + +os.remove('example') diff --git a/capnp/__init__.py b/capnp/__init__.py index 530b695..10be71a 100644 --- a/capnp/__init__.py +++ b/capnp/__init__.py @@ -35,5 +35,5 @@ Example Usage:: """ from .version import version as __version__ from .capnp import * -from .capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan +from .capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicOrphanListBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan del capnp diff --git a/capnp/capnp.pyx b/capnp/capnp.pyx index d2ea45a..5be3e18 100644 --- a/capnp/capnp.pyx +++ b/capnp/capnp.pyx @@ -71,7 +71,7 @@ cdef extern from "capnp/list.h" namespace " ::capnp": uint size() cdef extern from "" namespace "std": - C_DynamicOrphan moveOrphan"std::move"(C_DynamicOrphan &) + C_DynamicOrphan moveOrphan"std::move"(C_DynamicOrphan) cdef class _NodeReader: cdef C_Node.Reader thisptr @@ -144,6 +144,42 @@ cdef class _DynamicListReader: def __len__(self): return self.thisptr.size() +cdef class _DynamicOrphanListBuilder: + cdef public object _parent, _message, _field, _schema + cdef public list _list + def __init__(self, parent, field, schema): + self._parent = parent + self._message = parent._parent + self._field = field + self._schema = schema + + self._list = list() + + cpdef add(self): + if len(self) == 0: + self._message._addSerializeEvent(self._serialize) + + orphan = self._message.newOrphan(self._schema) + orphan_val = orphan.get() + self._list.append((orphan, orphan_val)) + return orphan_val + + def __getitem__(self, index): + return self._list[index][1] + + # def __setitem__(self, index, val): + # self._list[index] = val + + def __len__(self): + return len(self._list) + + def _serialize(self): + cdef int i = 0 + new_list = self._parent.init(self._field, len(self)) + for orphan, _ in self._list: + new_list.adopt(i, orphan) + i += 1 + cdef class _DynamicListBuilder: """Class for building Cap'n Proto Lists @@ -193,6 +229,47 @@ cdef class _DynamicListBuilder: def __len__(self): return self.thisptr.size() + cpdef adopt(self, index, _DynamicOrphan orphan): + """A method for adopting Cap'n Proto orphans + + Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie:: + + message = capnp.MallocMessageBuilder() + + alice = m.newOrphan(addressbook.Person) + alice.get().name = 'alice' + + bob = m.newOrphan(addressbook.Person) + bob.get().name = 'bob' + + addressBook = message.initRoot(addressbook.AddressBook) + people = addressBook.init('people', 2) + + people.adopt(0, alice) + people.adopt(1, bob) + + :type index: int + :param index: The index of the element in the list to replace with the newly adopted object + + :type orphan: :class:`_DynamicOrphan` + :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation. + + :rtype: void + """ + self.thisptr.adopt(index, orphan.move()) + + cpdef disown(self, index): + """A method for disowning Cap'n Proto orphans + + Don't use this method unless you know what you're doing. + + :type index: int + :param index: The index of the element in the list to disown + + :rtype: :class:`_DynamicOrphan` + """ + return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) + cdef class _List_NestedNode_Reader: cdef List[C_Node.NestedNode].Reader thisptr cdef _init(self, List[C_Node.NestedNode].Reader other): @@ -253,7 +330,7 @@ cdef toPython(C_DynamicValue.Builder self, object parent): temp = self.asData() return (temp.begin())[:temp.size()] elif type == capnp.TYPE_LIST: - return list(_DynamicListBuilder()._init(self.asList(), parent)) + return _DynamicListBuilder()._init(self.asList(), parent) elif type == capnp.TYPE_STRUCT: return _DynamicStructBuilder()._init(self.asStruct(), parent) elif type == capnp.TYPE_ENUM: @@ -389,6 +466,8 @@ cdef class _DynamicStructBuilder: """ if size is None: return toPython(self.thisptr.init(field), self._parent) + elif size == 0: + return _DynamicOrphanListBuilder(self, field, _StructSchema()._init((self.thisptr.get(field)).asList().getStructElementType())) else: return toPython(self.thisptr.init(field, size), self._parent) @@ -413,6 +492,47 @@ cdef class _DynamicStructBuilder: """ return fixMaybe(self.thisptr.which()).getProto().getName().cStr() + cpdef adopt(self, field, _DynamicOrphan orphan): + """A method for adopting Cap'n Proto orphans + + Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie:: + + message = capnp.MallocMessageBuilder() + + alice = m.newOrphan(addressbook.Person) + alice.get().name = 'alice' + + bob = m.newOrphan(addressbook.Person) + bob.get().name = 'bob' + + addressBook = message.initRoot(addressbook.AddressBook) + people = addressBook.init('people', 2) + + people.adopt(0, alice) + people.adopt(1, bob) + + :type field: str + :param field: The field name in the struct + + :type orphan: :class:`_DynamicOrphan` + :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation. + + :rtype: void + """ + self.thisptr.adopt(field, orphan.move()) + + cpdef disown(self, field): + """A method for disowning Cap'n Proto orphans + + Don't use this method unless you know what you're doing. + + :type field: str + :param field: The field name in the struct + + :rtype: :class:`_DynamicOrphan` + """ + return _DynamicOrphan()._init(self.thisptr.disown(field), self._parent) + cdef class _DynamicOrphan: cdef C_DynamicOrphan thisptr cdef public object _parent @@ -421,6 +541,16 @@ cdef class _DynamicOrphan: self._parent = parent return self + cdef C_DynamicOrphan move(self): + return moveOrphan(self.thisptr) + + cpdef get(self): + """Returns a python object corresponding to the DynamicValue owned by this orphan + + Use this DynamicValue to set fields inside the orphan + """ + return toPython(self.thisptr.get(), self._parent) + cdef class _Schema: cdef C_Schema thisptr cdef _init(self, C_Schema other): @@ -495,6 +625,7 @@ cdef class MessageBuilder: .. warning:: Don't ever instantiate this class directly. It is only used for inheritance. """ cdef schema_cpp.MessageBuilder * thisptr + cdef public list _serializeEvents def __dealloc__(self): del self.thisptr @@ -525,7 +656,7 @@ cdef class MessageBuilder: return _DynamicStructBuilder()._init(self.thisptr.initRootDynamicStruct(s.thisptr), self) cpdef getRoot(self, schema): - """A method for instantiating Cap'n Proto structs, from an already pre-written buffers + """A method for instantiating Cap'n Proto structs, from an already pre-written buffer Don't use this method unless you know what you're doing. You probably want to use initRoot instead:: @@ -549,6 +680,36 @@ cdef class MessageBuilder: s = schema return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(s.thisptr), self) + cpdef newOrphan(self, schema): + """A method for instantiating Cap'n Proto orphans + + Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie:: + + addressbook = capnp.load('addressbook.capnp') + + alice = m.newOrphan(addressbook.Person) + + :type schema: Schema + :param schema: A Cap'n proto schema specifying which struct to instantiate + + :rtype: :class:`_DynamicOrphan` + :return: An orphan representing a :class:`_DynamicStructBuilder` + """ + cdef _StructSchema s + if hasattr(schema, 'Schema'): + s = schema.Schema + else: + s = schema + + return _DynamicOrphan()._init(self.thisptr.newOrphan(s.thisptr), self) + + def _addSerializeEvent(self, event): + self._serializeEvents.append(event) + + def _serialize(self): + for event in self._serializeEvents: + event() + cdef class MallocMessageBuilder(MessageBuilder): """The main class for building Cap'n Proto messages @@ -566,6 +727,7 @@ cdef class MallocMessageBuilder(MessageBuilder): """ def __cinit__(self): self.thisptr = new schema_cpp.MallocMessageBuilder() + self._serializeEvents = list() def __init__(self): pass @@ -661,6 +823,7 @@ def writeMessageToFd(int fd, MessageBuilder message): :rtype: void """ + message._serialize() schema_cpp.writeMessageToFd(fd, deref(message.thisptr)) def writePackedMessageToFd(int fd, MessageBuilder message): @@ -687,6 +850,7 @@ def writePackedMessageToFd(int fd, MessageBuilder message): :rtype: void """ + message._serialize() schema_cpp.writePackedMessageToFd(fd, deref(message.thisptr)) from types import ModuleType as _ModuleType diff --git a/capnp/capnp_cpp.pxd b/capnp/capnp_cpp.pxd index 585b666..ca411fd 100644 --- a/capnp/capnp_cpp.pxd +++ b/capnp/capnp_cpp.pxd @@ -104,6 +104,8 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicValueForward.Builder init(char *) StructSchema getSchema() Maybe[StructSchema.Field] which() + void adopt(char *, DynamicOrphan) + DynamicOrphan disown(char *) cdef extern from "fixMaybe.h": StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except+ @@ -123,6 +125,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": uint size() void set(uint index, DynamicValueForward.Reader& value) except +ValueError DynamicValueForward.Builder init(uint index, uint size) except +ValueError + void adopt(uint, DynamicOrphan) + DynamicOrphan disown(uint) + StructSchema getStructElementType'getSchema().getStructElementType'() cdef cppclass DynamicValue: cppclass Reader: @@ -155,6 +160,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicStruct.Reader asStruct"as< ::capnp::DynamicStruct>"() DynamicEnum asEnum"as< ::capnp::DynamicEnum>"() Data.Reader asData"as< ::capnp::Data>"() + cppclass Builder: Type getType() int64_t asInt"as"() diff --git a/capnp/schema_cpp.pxd b/capnp/schema_cpp.pxd index c779541..f9a56b8 100644 --- a/capnp/schema_cpp.pxd +++ b/capnp/schema_cpp.pxd @@ -4,6 +4,7 @@ # distutils: libraries = capnp from libc.stdint cimport * +from capnp_cpp cimport DynamicOrphan ctypedef unsigned int uint ctypedef uint8_t UInt8 ctypedef uint16_t UInt16 @@ -664,6 +665,8 @@ cdef extern from "capnp/message.h" namespace " ::capnp": DynamicStruct.Builder getRootDynamicStruct'getRoot< ::capnp::DynamicStruct>'(StructSchema) DynamicStruct.Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema) + DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema) + cdef cppclass MessageReader: CodeGeneratorRequest.Reader getRootCodeGeneratorRequest'getRoot< ::capnp::schema::CodeGeneratorRequest>'() InterfaceNode.Reader getRootInterfaceNode'getRoot< ::capnp::schema::InterfaceNode>'() diff --git a/docs/capnp.rst b/docs/capnp.rst index f4da7e2..a4c8a7d 100644 --- a/docs/capnp.rst +++ b/docs/capnp.rst @@ -67,3 +67,10 @@ Builders :members: :undoc-members: :inherited-members: + +Miscellaneous +~~~~~~~~~~~~~ +.. autoclass:: _DynamicOrphan + :members: + :undoc-members: + :inherited-members: diff --git a/examples/example_orphan.py b/examples/example_orphan.py new file mode 100644 index 0000000..fdb819f --- /dev/null +++ b/examples/example_orphan.py @@ -0,0 +1,64 @@ +from __future__ import print_function +import os +import capnp + +this_dir = os.path.dirname(__file__) +addressbook = capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + +def writeAddressBook(fd): + message = capnp.MallocMessageBuilder() + addressBook = message.initRoot(addressbook.AddressBook) + people = addressBook.init('people', 0) + + alice = people.add() + alice.id = 123 + alice.name = 'Alice' + alice.email = 'alice@example.com' + alicePhones = alice.init('phones', 1) + alicePhones[0].number = "555-1212" + alicePhones[0].type = 'mobile' + alice.employment.school = "MIT" + + bob = people.add() + bob.id = 456 + bob.name = 'Bob' + bob.email = 'bob@example.com' + bobPhones = bob.init('phones', 2) + bobPhones[0].number = "555-4567" + bobPhones[0].type = 'home' + bobPhones[1].number = "555-7654" + bobPhones[1].type = 'work' + bob.employment.unemployed = None + + capnp.writePackedMessageToFd(fd, message) + + +def printAddressBook(fd): + message = capnp.PackedFdMessageReader(f.fileno()) + addressBook = message.getRoot(addressbook.AddressBook) + + for person in addressBook.people: + print(person.name, ':', person.email) + for phone in person.phones: + print(phone.type, ':', phone.number) + + which = person.employment.which() + print(which) + + if which == 'unemployed': + print('unemployed') + elif which == 'employer': + print('employer:', person.employment.employer) + elif which == 'school': + print('student at:', person.employment.school) + elif which == 'selfEmployed': + print('self employed') + print() + + +if __name__ == '__main__': + f = open('example', 'w') + writeAddressBook(f.fileno()) + + f = open('example', 'r') + printAddressBook(f.fileno())