Add _DynamicStructBuilder.to_bytes() and <struct module>.read_from_memory

to_bytes is not very efficient (it makes one necessary copy and one
unnecessary copy), but it will simplify a lot of use cases.

read_from_memory should be maximally efficient for most input types
(including mmap).

capnp/capnp.pyx

@@ -9,7 +9,7 @@
 cimport cython
 cimport capnp_cpp as capnp
 cimport schema_cpp
-from capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, fixMaybe, getEnumString, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan
+from capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, fixMaybe, getEnumString, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, WordArrayPtr
 
 from schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode
 from cython.operator cimport dereference as deref
@@ -40,6 +40,9 @@ ctypedef fused _DynamicSetterClasses:
     C_DynamicList.Builder
     C_DynamicStruct.Builder
 
+cdef extern from "Python.h":
+    cdef int PyObject_AsReadBuffer(object, void** b, Py_ssize_t* c)
+
 def _make_enum(enum_name, *sequential, **named):
     enums = dict(zip(sequential, range(len(sequential))), **named)
     reverse = dict((value, key) for key, value in enums.iteritems())
@@ -587,6 +590,22 @@ cdef class _DynamicStructBuilder:
             raise ValueError("You can only call write() on the message's root struct.")
         _write_packed_message_to_fd(file.fileno(), self._parent)
 
+    def to_bytes(_DynamicStructBuilder self):
+        """Returns the struct's containing message as a Python bytes object in the unpacked binary format.
+
+        This is inefficient; it makes several copies.
+
+        :rtype: bytes
+
+        :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct.
+        """
+        if not self._isRoot:
+            raise ValueError("You can only call write() on the message's root struct.")
+        cdef _MessageBuilder builder = self._parent
+        array = schema_cpp.messageToFlatArray(deref(builder.thisptr))
+        cdef const char* ptr = <const char *>array.begin()
+        return ptr[:8*array.size()]
+
     cdef _get(self, field):
         return to_python_builder(self.thisptr.get(field), self._parent)
 
@@ -927,6 +946,15 @@ cdef class SchemaParser:
                             reader = _PackedFdMessageReader(file.fileno())
                             return reader.get_root(local_module)
                         return read_helper
+                    def make_read_from_memory(local_module):
+                        def read_from_memory(buf):
+                            """Returns a Reader for the unpacked object in buf.
+
+                            :type buf: buffer
+                            :param buf: Any Python object that supports the readable buffer interface.  If buf is mutable, then changes to the object will be reflected in the returned Reader, which may be surprising.  If buf is an ordinary bytes object, then there should be no concern."""
+                            reader = _FlatArrayMessageReader(buf)
+                            return reader.get_root(local_module)
+                        return read_from_memory
                     def new_message(local_module):
                         def helper():
                             builder = _MallocMessageBuilder()
@@ -957,6 +985,7 @@ cdef class SchemaParser:
 
                     local_module.read = read(local_module)
                     local_module.read_packed = read_packed(local_module)
+                    local_module.read_from_memory = make_read_from_memory(local_module)
                     local_module.new_message = new_message(local_module)
                     local_module.from_dict = from_dict(local_module)
                     local_module.Reader = Reader
@@ -1170,6 +1199,18 @@ cdef class _PackedFdMessageReader(_MessageReader):
     def __init__(self, int fd):
         self.thisptr = new schema_cpp.PackedFdMessageReader(fd)
 
+@cython.internal
+cdef class _FlatArrayMessageReader(_MessageReader):
+    cdef object _object_to_pin
+    def __init__(self, buf):
+        cdef const void *ptr
+        cdef Py_ssize_t sz
+        PyObject_AsReadBuffer(buf, &ptr, &sz)
+        if sz % 8 != 0:
+            raise ValueError("input length must be a multiple of eight bytes")
+        self._object_to_pin = buf
+        self.thisptr = new schema_cpp.FlatArrayMessageReader(capnp.WordArrayPtr(<capnp.word*>ptr, sz//8))
+
 def _write_message_to_fd(int fd, _MessageBuilder message):
     """Serialize a Cap'n Proto message to a file descriptor
 

capnp/capnp_cpp.pxd

@@ -11,6 +11,8 @@ from libcpp cimport bool as cbool
 cdef extern from "capnp/common.h" namespace " ::capnp":
     enum Void:
         VOID " ::capnp::VOID"
+    cdef cppclass word:
+        pass
 
 cdef extern from "kj/string.h" namespace " ::kj":
     cdef cppclass StringPtr:
@@ -31,6 +33,23 @@ cdef extern from "kj/common.h" namespace " ::kj":
         size_t size()
         T& operator[](size_t index)
 
+    # Cython can't handle ArrayPtr[word] as a function argument
+    cdef cppclass WordArrayPtr "::kj::ArrayPtr<::capnp::word>":
+        WordArrayPtr()
+        WordArrayPtr(word *, size_t size)
+        size_t size()
+        word& operator[](size_t index)
+
+cdef extern from "kj/array.h" namespace " ::kj":
+    cdef cppclass Array[T]:
+        T* begin()
+        size_t size()
+
+    # Cython can't handle Array[word] as a function argument
+    cdef cppclass WordArray "::kj::Array<::capnp::word>":
+        word* begin()
+        size_t size()
+
 cdef extern from "capnp/schema.h" namespace " ::capnp":
     cdef cppclass Schema:
         Node.Reader getProto() except +

capnp/schema_cpp.pxd

@@ -5,6 +5,7 @@
 
 from libc.stdint cimport *
 from capnp_cpp cimport DynamicOrphan
+cimport capnp_cpp
 ctypedef unsigned int uint
 ctypedef uint8_t UInt8
 ctypedef uint16_t UInt16
@@ -695,11 +696,17 @@ cdef extern from "capnp/serialize.h" namespace " ::capnp":
         StreamFdMessageReader(int)
         StreamFdMessageReader(int, ReaderOptions)
 
+    cdef cppclass FlatArrayMessageReader(MessageReader):
+        FlatArrayMessageReader(capnp_cpp.WordArrayPtr array)
+        FlatArrayMessageReader(capnp_cpp.WordArrayPtr array, ReaderOptions)
+
     void writeMessageToFd(int, MessageBuilder&)
 
+    capnp_cpp.WordArray messageToFlatArray(MessageBuilder &)
+
 cdef extern from "capnp/serialize-packed.h" namespace " ::capnp":
     cdef cppclass PackedFdMessageReader(MessageReader):
         PackedFdMessageReader(int)
         StreamFdMessageReader(int, ReaderOptions)
 
-    void writePackedMessageToFd(int, MessageBuilder&)
+    void writePackedMessageToFd(int, MessageBuilder&)