Merge branch 'release/v0.5.4'

CHANGELOG.md

@@ -1,3 +1,14 @@
+## v0.5.4 (2015-03-02)
+- Update bundled C++ libcapnp to v0.5.1.1 security release
+- Add bootstrap RPC methods
+- Fix possible segfault when importing multiple schemas
+
+
+## v0.5.3 (2015-02-23)
+- Fix possible crash due to bad destructor ordering in MessageReader (by @JohnEmhoff)
+- Default to no longer using cython
+
+
 ## v0.5.2 (2015-02-20)
 - Add read\_multiple\_bytes/read\_multiple\_bytes\_packed methods
 - Added Python 3.4 to the travis build matrix

README.md

@@ -29,7 +29,7 @@ Install with `pip install pycapnp`. You can set the CC environment variable to c
 Or you can clone the repo like so:
 
     git clone https://github.com/jparyani/pycapnp.git
-    pip install ./pycapnp
+    pip install --install-option '--force-cython' ./pycapnp
 
 Note: for OSX, if using clang from Xcode 5, you will need to set `CFLAGS` like so:
 

buildutils/bundle.py

@@ -35,8 +35,8 @@ pjoin = os.path.join
 # Constants
 #-----------------------------------------------------------------------------
 
-bundled_version = (0,5,1)
+bundled_version = (0,5,1,1)
-libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version)
+libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version)
 libcapnp_url = "https://capnproto.org/" + libcapnp
 
 HERE = os.path.dirname(__file__)

capnp/helpers/helpers.pxd

@@ -30,6 +30,7 @@ cdef extern from "capnp/helpers/rpcHelper.h":
     Capability.Client restoreHelper(RpcSystem&, MessageReader&)
     Capability.Client restoreHelper(RpcSystem&, AnyPointer.Reader&)
     Capability.Client restoreHelper(RpcSystem&, AnyPointer.Builder&)
+    Capability.Client bootstrapHelper(RpcSystem&)
     RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&)
     PyPromise connectServer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr)
 

capnp/helpers/rpcHelper.h

@@ -73,6 +73,13 @@ capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::S
     return client.restore(hostId, objectId);
 }
 
+capnp::Capability::Client bootstrapHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
+    capnp::MallocMessageBuilder hostIdMessage(8);
+    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
+    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
+    return client.bootstrap(hostId);
+}
+
 template <typename SturdyRefHostId, typename ProvisionId,
           typename RecipientId, typename ThirdPartyCapId, typename JoinAnswer>
 capnp::RpcSystem<SturdyRefHostId> makeRpcClientWithRestorer(

capnp/includes/capnp_cpp.pxd

@@ -345,6 +345,7 @@ cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp":
         VoidPromise onDisconnect()
         VoidPromise onDrained()
     RpcSystem makeRpcServer(TwoPartyVatNetwork&, PyRestorer&)
+    RpcSystem makeRpcServerBootstrap"makeRpcServer"(TwoPartyVatNetwork&, Capability.Client)
     RpcSystem makeRpcClient(TwoPartyVatNetwork&)
 
 cdef extern from "capnp/dynamic.h" namespace " ::capnp":

capnp/lib/capnp.pxd

@@ -1,6 +1,6 @@
 from capnp.includes cimport capnp_cpp as capnp
 from capnp.includes cimport schema_cpp
-from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder
+from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder
 from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode
 from capnp.includes.types cimport *
 from capnp.helpers.non_circular cimport reraise_kj_exception

capnp/lib/capnp.pyx

@@ -2253,6 +2253,9 @@ cdef class TwoPartyClient:
 
         return self.restore(ref)
 
+    cpdef bootstrap(self) except +reraise_kj_exception:
+        return _CapabilityClient()._init(helpers.bootstrapHelper(deref(self.thisptr)), self)
+
     cpdef on_disconnect(self) except +reraise_kj_exception:
         return _VoidPromise()._init(deref(self._network.thisptr).onDisconnect())
 
@@ -2263,12 +2266,18 @@ cdef class TwoPartyServer:
     cdef public _Restorer _restorer
     cdef public _AsyncIoStream _stream
     cdef object _port
-    cdef public object port_promise
+    cdef public object port_promise, _bootstrap
     cdef capnp.TaskSet * _task_set
     cdef capnp.ErrorHandler _error_handler
 
-    def __init__(self, socket, restorer, server_socket=None):
+    def __init__(self, socket, restorer=None, server_socket=None, bootstrap=None):
-        self._restorer = _convert_restorer(restorer)
+        if not restorer and not bootstrap:
+            raise KjException("You must provide either a bootstrap interface or a restorer (deperecated) to a server constructor.")
+
+        cdef _InterfaceSchema schema
+        self._restorer = None
+        self._bootstrap = None
+
         if isinstance(socket, basestring):
             self._connect(socket)
         else:
@@ -2277,11 +2286,19 @@ cdef class TwoPartyServer:
             self._server_socket = server_socket
             self._port = 0
             self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER)
-            self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr)))
+
+            if bootstrap:
+                self._bootstrap = bootstrap
+                schema = bootstrap.schema
+                self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, <PyObject *>bootstrap)))
+            elif restorer:
+                self._restorer = _convert_restorer(restorer)
+                self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr)))
 
             Py_INCREF(self._orig_stream)
             Py_INCREF(self._stream)
             Py_INCREF(self._restorer)
+            Py_INCREF(self._bootstrap)
             Py_INCREF(self._network)
             self._disconnect_promise = self.on_disconnect().then(self._decref)
 
@@ -2292,6 +2309,7 @@ cdef class TwoPartyServer:
         self.port_promise = Promise()._init(helpers.connectServer(deref(self._task_set), deref(self._restorer.thisptr), loop.thisptr, temp_string))
 
     def _decref(self):
+        Py_DECREF(self._bootstrap)
         Py_DECREF(self._restorer)
         Py_DECREF(self._orig_stream)
         Py_DECREF(self._stream)
@@ -2318,8 +2336,6 @@ cdef class TwoPartyServer:
             else:
                 return self._port
 
-    # TODO: add restore functionality here?
-
 cdef class _AsyncIoStream:
     cdef Own[AsyncIoStream] thisptr
 
@@ -2963,6 +2979,23 @@ class _EnumModule(object):
         for name, val in schema.enumerants.items():
             setattr(self, name, val)
 
+cdef class _StringArrayPtr:
+    cdef StringPtr * thisptr
+    cdef object parent
+    cdef size_t size
+
+    def __cinit__(self, size_t size, parent):
+        self.size = size
+        self.thisptr = <StringPtr *>malloc(sizeof(StringPtr) * size)
+        self.parent = parent
+
+    def __dealloc__(self):
+        free(self.thisptr)
+
+    cdef ArrayPtr[StringPtr] asArrayPtr(self) except +reraise_kj_exception:
+        return ArrayPtr[StringPtr](self.thisptr, self.size)
+
+
 cdef class SchemaParser:
     """A class for loading Cap'n Proto schema files.
 
@@ -2970,26 +3003,34 @@ cdef class SchemaParser:
     """
     cdef C_SchemaParser * thisptr
     cdef public dict modules_by_id
+    cdef list _all_imports
+    cdef _StringArrayPtr _last_import_array
 
     def __cinit__(self):
         self.thisptr = new C_SchemaParser()
         self.modules_by_id = {}
+        self._all_imports = []
 
     def __dealloc__(self):
         del self.thisptr
 
     cpdef _parse_disk_file(self, displayName, diskPath, imports) except +reraise_kj_exception:
-        cdef StringPtr * importArray = <StringPtr *>malloc(sizeof(StringPtr) * len(imports))
+        cdef _StringArrayPtr importArray
 
-        for i in range(len(imports)):
+        if self._last_import_array and self._last_import_array.parent == imports:
-            importArray[i] = StringPtr(imports[i])
+            importArray = self._last_import_array
+        else:
+            importArray = _StringArrayPtr(len(imports), imports)
 
-        cdef ArrayPtr[StringPtr] importsPtr = ArrayPtr[StringPtr](importArray, <size_t>len(imports))
+            for i in range(len(imports)):
+                curr_import = imports[i]
+                importArray.thisptr[i] = StringPtr(curr_import, <size_t>len(curr_import))
+
+            self._all_imports.append(importArray)
+            self._last_import_array = importArray
 
         ret = _ParsedSchema()
-        ret._init_child(self.thisptr.parseDiskFile(displayName, diskPath, importsPtr))
+        ret._init_child(self.thisptr.parseDiskFile(displayName, diskPath, importArray.asArrayPtr()))
-
-        free(importArray)
 
         return ret
 
@@ -3229,8 +3270,7 @@ cdef class _MessageReader:
     """
     cdef public object _parent
     cdef schema_cpp.MessageReader * thisptr
-    def __dealloc__(self):
+
-        del self.thisptr
     def __init__(self):
         raise NotImplementedError("This is an abstract base class")
 
@@ -3290,6 +3330,10 @@ cdef class _StreamFdMessageReader(_MessageReader):
 
         self.thisptr = new schema_cpp.StreamFdMessageReader(fd, opts)
 
+    def __dealloc__(self):
+        del self.thisptr
+
+
 cdef class _PackedMessageReader(_MessageReader):
     """Read a Cap'n Proto message from a file descriptor in a packed manner
 
@@ -3318,6 +3362,10 @@ cdef class _PackedMessageReader(_MessageReader):
         self.thisptr = new schema_cpp.PackedMessageReader(stream, opts)
         return self
 
+    def __dealloc__(self):
+        del self.thisptr
+
+
 cdef class _PackedMessageReaderBytes(_MessageReader):
     cdef schema_cpp.ArrayInputStream * stream
 
@@ -3340,6 +3388,7 @@ cdef class _PackedMessageReaderBytes(_MessageReader):
         self.thisptr = new schema_cpp.PackedMessageReader(deref(self.stream), opts)
 
     def __dealloc__(self):
+        del self.thisptr
         del self.stream
 
 cdef class _InputMessageReader(_MessageReader):
@@ -3370,6 +3419,10 @@ cdef class _InputMessageReader(_MessageReader):
         self.thisptr = new schema_cpp.InputStreamMessageReader(stream, opts)
         return self
 
+    def __dealloc__(self):
+        del self.thisptr
+
+
 cdef class _PackedFdMessageReader(_MessageReader):
     """Read a Cap'n Proto message from a file descriptor in a packed manner
 
@@ -3392,6 +3445,10 @@ cdef class _PackedFdMessageReader(_MessageReader):
 
         self.thisptr = new schema_cpp.PackedFdMessageReader(fd, opts)
 
+    def __dealloc__(self):
+        del self.thisptr
+
+
 cdef class _MultipleMessageReader:
     cdef schema_cpp.FdInputStream * stream
     cdef schema_cpp.BufferedInputStream * buffered_stream
@@ -3574,6 +3631,10 @@ cdef class _FlatArrayMessageReader(_MessageReader):
 
         self.thisptr = new schema_cpp.FlatArrayMessageReader(schema_cpp.WordArrayPtr(<schema_cpp.word*>ptr, sz//8))
 
+    def __dealloc__(self):
+        del self.thisptr
+
+
 @cython.internal
 cdef class _FlatMessageBuilder(_MessageBuilder):
     cdef object _object_to_pin

setup.py

@@ -1,21 +1,7 @@
 #!/usr/bin/env python
 from __future__ import print_function
 
-use_cython = True
+use_cython = False
-try:
-    from Cython.Build import cythonize
-    import Cython
-except ImportError:
-    use_cython = False
-
-if use_cython and Cython.__version__ < '0.19.1':
-    use_cython = False
-
-import pkg_resources
-setuptools_version = pkg_resources.get_distribution("setuptools").version
-if setuptools_version < '0.8':
-    # older versions of setuptools don't work with cython
-    use_cython = False
 
 from distutils.core import setup
 import os
@@ -23,16 +9,12 @@ import sys
 from buildutils import test_build, fetch_libcapnp, build_libcapnp, info
 from distutils.errors import CompileError
 from distutils.extension import Extension
-if use_cython:
-    from Cython.Distutils import build_ext as build_ext_c
-else:
-    from distutils.command.build_ext import build_ext as build_ext_c
 
 _this_dir = os.path.dirname(__file__)
 
 MAJOR = 0
 MINOR = 5
-MICRO = 2
+MICRO = 4
 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
 
 
@@ -78,6 +60,11 @@ class clean(_clean):
             except OSError:
                 pass
 
+# set use_cython if lib/capnp.cpp is not detected
+capnp_compiled_file = os.path.join(os.path.dirname(__file__), 'capnp', 'lib', 'capnp.cpp')
+if not os.path.isfile(capnp_compiled_file):
+    use_cython = True
+
 # hack to parse commandline arguments
 force_bundled_libcapnp = "--force-bundled-libcapnp" in sys.argv
 if force_bundled_libcapnp:
@@ -85,15 +72,15 @@ if force_bundled_libcapnp:
 force_system_libcapnp = "--force-system-libcapnp" in sys.argv
 if force_system_libcapnp:
     sys.argv.remove("--force-system-libcapnp")
-disable_cython = "--disable-cython" in sys.argv
-if disable_cython:
-    sys.argv.remove("--disable-cython")
-    use_cython = False
 force_cython = "--force-cython" in sys.argv
 if force_cython:
     sys.argv.remove("--force-cython")
     use_cython = True
 
+if use_cython:
+    from Cython.Distutils import build_ext as build_ext_c
+else:
+    from distutils.command.build_ext import build_ext as build_ext_c
 
 class build_libcapnp_ext(build_ext_c):
     def build_extension(self, ext):
@@ -127,6 +114,8 @@ class build_libcapnp_ext(build_ext_c):
         return build_ext_c.run(self)
 
 if use_cython:
+    from Cython.Build import cythonize
+    import Cython
     extensions = cythonize('capnp/lib/*.pyx')
 else:
     extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"],

test/test_rpc.py

@@ -86,3 +86,17 @@ def test_ez_rpc():
 
     with pytest.raises(capnp.KjException):
         response = remote.wait()
+
+def test_simple_rpc_bootstrap():
+    read, write = socket.socketpair(socket.AF_UNIX)
+
+    server = capnp.TwoPartyServer(write, bootstrap=Server(100))
+    client = capnp.TwoPartyClient(read)
+
+    cap = client.bootstrap()
+    cap = cap.cast_as(test_capability_capnp.TestInterface)
+
+    remote = cap.foo(i=5)
+    response = remote.wait()
+
+    assert response.x == '125'