Add full orphan functionality. Also, allow special orphan lists

that can grow over time.

benchmark/addressbook.capnp.orphan.py

@@ -0,0 +1,56 @@
+from __future__ import print_function
+import os
+import capnp
+
+this_dir = os.path.dirname(__file__)
+addressbook = capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
+
+print = lambda *x: x
+
+
+def writeAddressBook(fd):
+    message = capnp.MallocMessageBuilder()
+    addressBook = message.initRoot(addressbook.AddressBook)
+    people = addressBook.init('people', 0)
+
+    alice = people.add()
+    alice.id = 123
+    alice.name = 'Alice'
+    alice.email = 'alice@example.com'
+    alicePhones = alice.init('phones', 1)
+    alicePhones[0].number = "555-1212"
+    alicePhones[0].type = 'mobile'
+
+    bob = people.add()
+    bob.id = 456
+    bob.name = 'Bob'
+    bob.email = 'bob@example.com'
+    bobPhones = bob.init('phones', 2)
+    bobPhones[0].number = "555-4567"
+    bobPhones[0].type = 'home'
+    bobPhones[1].number = "555-7654"
+    bobPhones[1].type = 'work'
+
+    capnp.writePackedMessageToFd(fd, message)
+
+
+def printAddressBook(fd):
+    message = capnp.PackedFdMessageReader(f.fileno())
+    addressBook = message.getRoot(addressbook.AddressBook)
+
+    for person in addressBook.people:
+        print(person.name, ':', person.email)
+        for phone in person.phones:
+            print(phone.type, ':', phone.number)
+        print()
+
+
+if __name__ == '__main__':
+    for i in range(10000):
+        f = open('example', 'w')
+        writeAddressBook(f.fileno())
+
+        f = open('example', 'r')
+        printAddressBook(f.fileno())
+
+os.remove('example')

capnp/__init__.py

@@ -35,5 +35,5 @@ Example Usage::
 """
 from .version import version as __version__
 from .capnp import *
-from .capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan
+from .capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicOrphanListBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan
 del capnp

capnp/capnp.pyx

@@ -71,7 +71,7 @@ cdef extern from "capnp/list.h" namespace " ::capnp":
             uint size()
 
 cdef extern from "<utility>" namespace "std":
-    C_DynamicOrphan moveOrphan"std::move"(C_DynamicOrphan &)
+    C_DynamicOrphan moveOrphan"std::move"(C_DynamicOrphan)
 
 cdef class _NodeReader:
     cdef C_Node.Reader thisptr
@@ -144,6 +144,42 @@ cdef class _DynamicListReader:
     def __len__(self):
         return self.thisptr.size()
 
+cdef class _DynamicOrphanListBuilder:
+    cdef public object _parent, _message, _field, _schema
+    cdef public list _list
+    def __init__(self, parent, field, schema):
+        self._parent = parent
+        self._message = parent._parent
+        self._field = field
+        self._schema = schema
+
+        self._list = list()
+
+    cpdef add(self):
+        if len(self) == 0:
+            self._message._addSerializeEvent(self._serialize)
+
+        orphan = self._message.newOrphan(self._schema)
+        orphan_val = orphan.get()
+        self._list.append((orphan, orphan_val))
+        return orphan_val
+        
+    def __getitem__(self, index):
+        return self._list[index][1]
+
+    # def __setitem__(self, index, val):
+    #     self._list[index] = val
+
+    def __len__(self):
+        return len(self._list)
+
+    def _serialize(self):
+        cdef int i = 0
+        new_list = self._parent.init(self._field, len(self))
+        for orphan, _ in self._list:
+            new_list.adopt(i, orphan)
+            i += 1
+
 cdef class _DynamicListBuilder:
     """Class for building Cap'n Proto Lists
 
@@ -193,6 +229,47 @@ cdef class _DynamicListBuilder:
     def __len__(self):
         return self.thisptr.size()
 
+    cpdef adopt(self, index, _DynamicOrphan orphan):
+        """A method for adopting Cap'n Proto orphans
+
+        Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie::
+
+            message = capnp.MallocMessageBuilder()
+
+            alice = m.newOrphan(addressbook.Person)
+            alice.get().name = 'alice'
+
+            bob = m.newOrphan(addressbook.Person)
+            bob.get().name = 'bob'
+
+            addressBook = message.initRoot(addressbook.AddressBook)
+            people = addressBook.init('people', 2)
+
+            people.adopt(0, alice)
+            people.adopt(1, bob)
+
+        :type index: int
+        :param index: The index of the element in the list to replace with the newly adopted object
+
+        :type orphan: :class:`_DynamicOrphan`
+        :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation.
+
+        :rtype: void
+        """
+        self.thisptr.adopt(index, orphan.move())
+
+    cpdef disown(self, index):
+        """A method for disowning Cap'n Proto orphans
+
+        Don't use this method unless you know what you're doing.
+
+        :type index: int
+        :param index: The index of the element in the list to disown
+
+        :rtype: :class:`_DynamicOrphan`
+        """
+        return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent)
+
 cdef class _List_NestedNode_Reader:
     cdef List[C_Node.NestedNode].Reader thisptr
     cdef _init(self, List[C_Node.NestedNode].Reader other):
@@ -253,7 +330,7 @@ cdef toPython(C_DynamicValue.Builder self, object parent):
         temp = self.asData()
         return (<char*>temp.begin())[:temp.size()]
     elif type == capnp.TYPE_LIST:
-        return list(_DynamicListBuilder()._init(self.asList(), parent))
+        return _DynamicListBuilder()._init(self.asList(), parent)
     elif type == capnp.TYPE_STRUCT:
         return _DynamicStructBuilder()._init(self.asStruct(), parent)
     elif type == capnp.TYPE_ENUM:
@@ -389,6 +466,8 @@ cdef class _DynamicStructBuilder:
         """
         if size is None:
             return toPython(self.thisptr.init(field), self._parent)
+        elif size == 0:
+            return _DynamicOrphanListBuilder(self, field, _StructSchema()._init((<C_DynamicValue.Builder>self.thisptr.get(field)).asList().getStructElementType()))
         else:
             return toPython(self.thisptr.init(field, size), self._parent)
 
@@ -413,6 +492,47 @@ cdef class _DynamicStructBuilder:
         """
         return fixMaybe(self.thisptr.which()).getProto().getName().cStr()
 
+    cpdef adopt(self, field, _DynamicOrphan orphan):
+        """A method for adopting Cap'n Proto orphans
+
+        Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie::
+
+            message = capnp.MallocMessageBuilder()
+
+            alice = m.newOrphan(addressbook.Person)
+            alice.get().name = 'alice'
+
+            bob = m.newOrphan(addressbook.Person)
+            bob.get().name = 'bob'
+
+            addressBook = message.initRoot(addressbook.AddressBook)
+            people = addressBook.init('people', 2)
+
+            people.adopt(0, alice)
+            people.adopt(1, bob)
+
+        :type field: str
+        :param field: The field name in the struct
+
+        :type orphan: :class:`_DynamicOrphan`
+        :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation.
+
+        :rtype: void
+        """
+        self.thisptr.adopt(field, orphan.move())
+
+    cpdef disown(self, field):
+        """A method for disowning Cap'n Proto orphans
+
+        Don't use this method unless you know what you're doing.
+
+        :type field: str
+        :param field: The field name in the struct
+
+        :rtype: :class:`_DynamicOrphan`
+        """
+        return _DynamicOrphan()._init(self.thisptr.disown(field), self._parent)
+
 cdef class _DynamicOrphan:
     cdef C_DynamicOrphan thisptr
     cdef public object _parent
@@ -421,6 +541,16 @@ cdef class _DynamicOrphan:
         self._parent = parent
         return self
 
+    cdef C_DynamicOrphan move(self):
+        return moveOrphan(self.thisptr)
+
+    cpdef get(self):
+        """Returns a python object corresponding to the DynamicValue owned by this orphan
+
+        Use this DynamicValue to set fields inside the orphan
+        """
+        return toPython(self.thisptr.get(), self._parent)
+
 cdef class _Schema:
     cdef C_Schema thisptr
     cdef _init(self, C_Schema other):
@@ -495,6 +625,7 @@ cdef class MessageBuilder:
     .. warning:: Don't ever instantiate this class directly. It is only used for inheritance.
     """
     cdef schema_cpp.MessageBuilder * thisptr
+    cdef public list _serializeEvents
     def __dealloc__(self):
         del self.thisptr
 
@@ -525,7 +656,7 @@ cdef class MessageBuilder:
         return _DynamicStructBuilder()._init(self.thisptr.initRootDynamicStruct(s.thisptr), self)
 
     cpdef getRoot(self, schema):
-        """A method for instantiating Cap'n Proto structs, from an already pre-written buffers
+        """A method for instantiating Cap'n Proto structs, from an already pre-written buffer
 
         Don't use this method unless you know what you're doing. You probably
         want to use initRoot instead::
@@ -549,6 +680,36 @@ cdef class MessageBuilder:
             s = schema
         return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(s.thisptr), self)
 
+    cpdef newOrphan(self, schema):
+        """A method for instantiating Cap'n Proto orphans
+
+        Don't use this method unless you know what you're doing. Orphans are useful for dynamically allocating objects for an unkown sized list, ie::
+
+            addressbook = capnp.load('addressbook.capnp')
+
+            alice = m.newOrphan(addressbook.Person)
+
+        :type schema: Schema
+        :param schema: A Cap'n proto schema specifying which struct to instantiate
+
+        :rtype: :class:`_DynamicOrphan`
+        :return: An orphan representing a :class:`_DynamicStructBuilder`
+        """
+        cdef _StructSchema s
+        if hasattr(schema, 'Schema'):
+            s = schema.Schema
+        else:
+            s = schema
+
+        return _DynamicOrphan()._init(self.thisptr.newOrphan(s.thisptr), self)
+
+    def _addSerializeEvent(self, event):
+        self._serializeEvents.append(event)
+
+    def _serialize(self):
+        for event in self._serializeEvents:
+            event()
+
 cdef class MallocMessageBuilder(MessageBuilder):
     """The main class for building Cap'n Proto messages
 
@@ -566,6 +727,7 @@ cdef class MallocMessageBuilder(MessageBuilder):
     """
     def __cinit__(self):
         self.thisptr = new schema_cpp.MallocMessageBuilder()
+        self._serializeEvents = list()
 
     def __init__(self):
         pass
@@ -661,6 +823,7 @@ def writeMessageToFd(int fd, MessageBuilder message):
 
     :rtype: void
     """
+    message._serialize()
     schema_cpp.writeMessageToFd(fd, deref(message.thisptr))
 
 def writePackedMessageToFd(int fd, MessageBuilder message):
@@ -687,6 +850,7 @@ def writePackedMessageToFd(int fd, MessageBuilder message):
 
     :rtype: void
     """
+    message._serialize()
     schema_cpp.writePackedMessageToFd(fd, deref(message.thisptr))
 
 from types import ModuleType as _ModuleType

capnp/capnp_cpp.pxd

@@ -104,6 +104,8 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             DynamicValueForward.Builder init(char *)
             StructSchema getSchema()
             Maybe[StructSchema.Field] which()
+            void adopt(char *, DynamicOrphan)
+            DynamicOrphan disown(char *)
 
 cdef extern from "fixMaybe.h":
     StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except+
@@ -123,6 +125,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             uint size()
             void set(uint index, DynamicValueForward.Reader& value) except +ValueError
             DynamicValueForward.Builder init(uint index, uint size) except +ValueError
+            void adopt(uint, DynamicOrphan)
+            DynamicOrphan disown(uint)
+            StructSchema getStructElementType'getSchema().getStructElementType'()
 
     cdef cppclass DynamicValue:
         cppclass Reader:
@@ -155,6 +160,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             DynamicStruct.Reader asStruct"as< ::capnp::DynamicStruct>"()
             DynamicEnum asEnum"as< ::capnp::DynamicEnum>"()
             Data.Reader asData"as< ::capnp::Data>"()
+
         cppclass Builder:
             Type getType()
             int64_t asInt"as<int64_t>"()

capnp/schema_cpp.pxd

@@ -4,6 +4,7 @@
 # distutils: libraries = capnp
 
 from libc.stdint cimport *
+from capnp_cpp cimport DynamicOrphan
 ctypedef unsigned int uint
 ctypedef uint8_t UInt8
 ctypedef uint16_t UInt16
@@ -664,6 +665,8 @@ cdef extern from "capnp/message.h" namespace " ::capnp":
         DynamicStruct.Builder getRootDynamicStruct'getRoot< ::capnp::DynamicStruct>'(StructSchema)
         DynamicStruct.Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema)
 
+        DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema)
+
     cdef cppclass MessageReader:
         CodeGeneratorRequest.Reader getRootCodeGeneratorRequest'getRoot< ::capnp::schema::CodeGeneratorRequest>'()
         InterfaceNode.Reader getRootInterfaceNode'getRoot< ::capnp::schema::InterfaceNode>'()

docs/capnp.rst

@@ -67,3 +67,10 @@ Builders
   :members:
   :undoc-members:
   :inherited-members:
+
+Miscellaneous
+~~~~~~~~~~~~~
+.. autoclass:: _DynamicOrphan
+  :members:
+  :undoc-members:
+  :inherited-members:

examples/example_orphan.py

@@ -0,0 +1,64 @@
+from __future__ import print_function
+import os
+import capnp
+
+this_dir = os.path.dirname(__file__)
+addressbook = capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
+
+def writeAddressBook(fd):
+    message = capnp.MallocMessageBuilder()
+    addressBook = message.initRoot(addressbook.AddressBook)
+    people = addressBook.init('people', 0)
+
+    alice = people.add()
+    alice.id = 123
+    alice.name = 'Alice'
+    alice.email = 'alice@example.com'
+    alicePhones = alice.init('phones', 1)
+    alicePhones[0].number = "555-1212"
+    alicePhones[0].type = 'mobile'
+    alice.employment.school = "MIT"
+
+    bob = people.add()
+    bob.id = 456
+    bob.name = 'Bob'
+    bob.email = 'bob@example.com'
+    bobPhones = bob.init('phones', 2)
+    bobPhones[0].number = "555-4567"
+    bobPhones[0].type = 'home'
+    bobPhones[1].number = "555-7654"
+    bobPhones[1].type = 'work'
+    bob.employment.unemployed = None
+
+    capnp.writePackedMessageToFd(fd, message)
+
+
+def printAddressBook(fd):
+    message = capnp.PackedFdMessageReader(f.fileno())
+    addressBook = message.getRoot(addressbook.AddressBook)
+
+    for person in addressBook.people:
+        print(person.name, ':', person.email)
+        for phone in person.phones:
+            print(phone.type, ':', phone.number)
+
+        which = person.employment.which()
+        print(which)
+
+        if which == 'unemployed':
+            print('unemployed')
+        elif which == 'employer':
+            print('employer:', person.employment.employer)
+        elif which == 'school':
+            print('student at:', person.employment.school)
+        elif which == 'selfEmployed':
+            print('self employed')
+        print()
+
+
+if __name__ == '__main__':
+    f = open('example', 'w')
+    writeAddressBook(f.fileno())
+
+    f = open('example', 'r')
+    printAddressBook(f.fileno())