Various fixups to the RPC api.

* change how restorer works
* fix join_promises
* add incref's all around to make sure we aren't freeing objects early
* make it so we return PyPromises everywhere and make chains collapsible

capnp/helpers/capabilityHelper.h

@@ -6,12 +6,14 @@
 #include <iostream>
 
 extern "C" {
-   void wrap_remote_call(PyObject * func, capnp::Response<capnp::DynamicStruct> &);
+   PyObject * wrap_remote_call(PyObject * func, capnp::Response<capnp::DynamicStruct> &);
    PyObject * wrap_dynamic_struct_reader(capnp::DynamicStruct::Reader &);
    ::kj::Promise<void> * call_server_method(PyObject * py_server, char * name, capnp::CallContext< capnp::DynamicStruct, capnp::DynamicStruct> & context);
    PyObject * wrap_kj_exception(kj::Exception &);
    PyObject * wrap_kj_exception_for_reraise(kj::Exception &);
    PyObject * get_exception_info(PyObject *, PyObject *, PyObject *);
+   PyObject * convert_array_pyobject(kj::Array<PyObject *>&);
+   ::kj::Promise<PyObject *> * extract_promise(PyObject *);
  }
 
 void reraise_kj_exception() {
@@ -39,6 +41,8 @@ void check_py_error() {
     if(err) {
         PyObject * ptype, *pvalue, *ptraceback;
         PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+        if(ptype == NULL || pvalue == NULL || ptraceback == NULL)
+          throw kj::Exception(kj::Exception::Nature::OTHER, kj::Exception::Durability::PERMANENT, kj::heapString("capabilityHelper.h"), 44, kj::heapString("Unknown error occurred"));
 
         PyObject * info = get_exception_info(ptype, pvalue, ptraceback);
 
@@ -62,26 +66,39 @@ void check_py_error() {
 }
 
 // TODO: need to decref error_func as well on successful run
-PyObject * wrapPyFunc(PyObject * func, PyObject * arg) {
+kj::Promise<PyObject *> wrapPyFunc(PyObject * func, PyObject * arg) {
     PyObject * result = PyObject_CallFunctionObjArgs(func, arg, NULL);
     Py_DECREF(func);
 
     check_py_error();
+
+    auto promise = extract_promise(result);
+    if(promise != NULL)
+      return kj::mv(*promise);
     return result;
 }
 
-PyObject * wrapPyFuncNoArg(PyObject * func) {
+kj::Promise<PyObject *> wrapPyFuncNoArg(PyObject * func) {
     PyObject * result = PyObject_CallFunctionObjArgs(func, NULL);
     Py_DECREF(func);
 
     check_py_error();
+
+    auto promise = extract_promise(result);
+    if(promise != NULL)
+      return kj::mv(*promise);
     return result;
 }
 
-void wrapRemoteCall(PyObject * func, capnp::Response<capnp::DynamicStruct> & arg) {
+kj::Promise<PyObject *> wrapRemoteCall(PyObject * func, capnp::Response<capnp::DynamicStruct> & arg) {
-    wrap_remote_call(func, arg);
+    PyObject * ret = wrap_remote_call(func, arg);
 
     check_py_error();
+
+    auto promise = extract_promise(ret);
+    if(promise != NULL)
+      return kj::mv(*promise);
+    return ret;
 }
 
 ::kj::Promise<PyObject *> then(kj::Promise<PyObject *> & promise, PyObject * func, PyObject * error_func) {
@@ -92,12 +109,12 @@ void wrapRemoteCall(PyObject * func, capnp::Response<capnp::DynamicStruct> & arg
                                      , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
 }
 
-::kj::Promise<void> then(::capnp::RemotePromise< ::capnp::DynamicStruct> & promise, PyObject * func, PyObject * error_func) {
+::kj::Promise<PyObject *> then(::capnp::RemotePromise< ::capnp::DynamicStruct> & promise, PyObject * func, PyObject * error_func) {
   if(error_func == Py_None)
-    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { wrapRemoteCall(func, arg); } );
+    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { return wrapRemoteCall(func, arg); } );
   else
-    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { wrapRemoteCall(func, arg); } 
+    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { return  wrapRemoteCall(func, arg); } 
-                                     , [error_func](kj::Exception arg) { wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
+                                     , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
 }
 
 ::kj::Promise<PyObject *> then(kj::Promise<void> & promise, PyObject * func, PyObject * error_func) {
@@ -108,6 +125,10 @@ void wrapRemoteCall(PyObject * func, capnp::Response<capnp::DynamicStruct> & arg
                                      , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
 }
 
+::kj::Promise<PyObject *> then(kj::Promise<kj::Array<PyObject *> > && promise) {
+  return promise.then([](kj::Array<PyObject *>&& arg) { return convert_array_pyobject(arg); } );
+}
+
 class PythonInterfaceDynamicImpl final: public capnp::DynamicCapability::Server {
 public:
   PyObject * py_server;
@@ -152,3 +173,12 @@ capnp::Capability::Client server_to_client(capnp::InterfaceSchema & schema, PyOb
 ::kj::Promise<PyObject *> convert_to_pypromise(capnp::RemotePromise<capnp::DynamicStruct> & promise) {
     return promise.then([](capnp::Response<capnp::DynamicStruct>&& response) { return wrap_dynamic_struct_reader(response); } );
 }
+
+::kj::Promise<PyObject *> convert_to_pypromise(kj::Promise<void> & promise) {
+    return promise.then([]() { Py_RETURN_NONE;} );
+}
+
+template<class T>
+::kj::Promise<void> convert_to_voidpromise(kj::Promise<T> & promise) {
+    return promise.then([](T) { } );
+}

capnp/helpers/helpers.pxd

@@ -1,4 +1,4 @@
-from .capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, PyPromise, VoidPromise, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer
+from .capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer
 
 from non_circular cimport reraise_kj_exception
 
@@ -14,12 +14,15 @@ cdef extern from "../helpers/capabilityHelper.h":
     # PyPromise evalLater(EventLoop &, PyObject * func)
     # PyPromise there(EventLoop & loop, PyPromise & promise, PyObject * func, PyObject * error_func)
     PyPromise then(PyPromise & promise, PyObject * func, PyObject * error_func)
-    VoidPromise then(RemotePromise & promise, PyObject * func, PyObject * error_func)
+    PyPromise then(RemotePromise & promise, PyObject * func, PyObject * error_func)
     PyPromise then(VoidPromise & promise, PyObject * func, PyObject * error_func)
+    PyPromise then(PyPromiseArray & promise)
     DynamicCapability.Client new_client(InterfaceSchema&, PyObject *)
     DynamicValue.Reader new_server(InterfaceSchema&, PyObject *)
     Capability.Client server_to_client(InterfaceSchema&, PyObject *)
     PyPromise convert_to_pypromise(RemotePromise&)
+    PyPromise convert_to_pypromise(VoidPromise&)
+    VoidPromise convert_to_voidpromise(PyPromise&)
 
 cdef extern from "../helpers/rpcHelper.h":
     Capability.Client restoreHelper(RpcSystem&, MessageBuilder&)

capnp/helpers/rpcHelper.h

@@ -6,12 +6,12 @@
 #include "capabilityHelper.h"
 
 extern "C" {
-   capnp::Capability::Client * call_py_restorer(PyObject *, capnp::DynamicStruct::Reader &);
+   capnp::Capability::Client * call_py_restorer(PyObject *, capnp::AnyPointer::Reader &);
 }
 
 class PyRestorer final: public capnp::SturdyRefRestorer<capnp::AnyPointer> {
 public:
-  PyRestorer(PyObject * _py_restorer, capnp::StructSchema& _schema): py_restorer(_py_restorer), schema(_schema) {
+  PyRestorer(PyObject * _py_restorer): py_restorer(_py_restorer) {
     // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere.
     // Py_INCREF(py_restorer);
   }
@@ -21,8 +21,7 @@ public:
   // }
   
   capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override {
-    auto reader = objectId.getAs<capnp::DynamicStruct>(schema);
+    capnp::Capability::Client * ret = call_py_restorer(py_restorer, objectId);
-    capnp::Capability::Client * ret = call_py_restorer(py_restorer, reader);
     check_py_error();
     capnp::Capability::Client stack_ret(*ret);
     delete ret;
@@ -32,7 +31,6 @@ public:
 
 private:
   PyObject * py_restorer;
-  capnp::StructSchema schema;
 };
 
 capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::MessageBuilder & objectId) {  capnp::MallocMessageBuilder hostIdMessage(8);

capnp/includes/capnp_cpp.pxd

@@ -12,6 +12,7 @@ cdef extern from "kj/async.h" namespace " ::kj":
     cdef cppclass Promise[T]:
         Promise()
         Promise(Promise)
+        Promise(T)
         T wait(WaitScope)
 
 ctypedef Promise[PyObject *] PyPromise
@@ -60,6 +61,19 @@ cdef extern from "kj/array.h" namespace " ::kj":
     cdef cppclass Array[T]:
         T* begin()
         size_t size()
+        T& operator[](size_t index)
+    cdef cppclass ArrayBuilder[T]:
+        T* begin()
+        size_t size()
+        T& operator[](size_t index)
+        T& add(T&)
+        Array[T] finish()
+
+    ArrayBuilder[PyPromise] heapArrayBuilderPyPromise"::kj::heapArrayBuilder< ::kj::Promise<PyObject *> >"(size_t)
+
+    ctypedef Array[PyObject *] PyArray' ::kj::Array<PyObject *>'
+
+ctypedef Promise[PyArray] PyPromiseArray
 
 cdef extern from "kj/async-io.h" namespace " ::kj":
     cdef cppclass AsyncIoStream:
@@ -217,7 +231,7 @@ cdef extern from "capnp/capability.h" namespace " ::capnp":
 
 cdef extern from "../helpers/rpcHelper.h":
     cdef cppclass PyRestorer:
-        PyRestorer(PyObject *, StructSchema&)
+        PyRestorer(PyObject *)
 
 cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp":
     cdef cppclass RpcSystem" ::capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>":
@@ -249,10 +263,13 @@ cdef extern from "capnp/any.h" namespace " ::capnp":
     cdef cppclass AnyPointer:
         cppclass Reader:
             DynamicStruct.Reader getAs"getAs< ::capnp::DynamicStruct>"(StructSchema)
+            String getAsText"getAs< ::capnp::Text>"()
         cppclass Builder:
             Builder(Builder)
             DynamicStruct.Builder getAs"getAs< ::capnp::DynamicStruct>"(StructSchema)
-            void setAsText"setAs< ::capnp::Text>"(char*)
+            String getAsText"getAs< ::capnp::Text>"()
+            void setAsStruct"setAs< ::capnp::DynamicStruct>"(DynamicStruct.Reader&) except +reraise_kj_exception
+            void setAsText"setAs< ::capnp::Text>"(char*) except +reraise_kj_exception
 
 cdef extern from "capnp/dynamic.h" namespace " ::capnp":
     cdef cppclass DynamicEnum:
@@ -381,3 +398,4 @@ cdef extern from "kj/async.h" namespace " ::kj":
         VoidPromise promise
         Own[PromiseFulfiller] fulfiller
     PromiseFulfillerPair newPromiseAndFulfiller" ::kj::newPromiseAndFulfiller<void>"()
+    PyPromiseArray joinPromises(Array[PyPromise])

capnp/lib/capnp.pxd

@@ -1,6 +1,6 @@
 from .capnp.includes cimport capnp_cpp as capnp
 from .capnp.includes cimport schema_cpp
-from .capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller
+from .capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray
 from .capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode
 from .capnp.includes.types cimport *
 from .capnp.helpers.non_circular cimport reraise_kj_exception

capnp/lib/capnp.pyx

@@ -17,6 +17,7 @@ from types import ModuleType as _ModuleType
 import os as _os
 import sys as _sys
 import imp as _imp
+import traceback as _traceback
 from functools import partial as _partial
 import warnings as _warnings
 import inspect as _inspect
@@ -26,12 +27,14 @@ from operator import attrgetter as _attrgetter
 cdef public object wrap_dynamic_struct_reader(C_DynamicStruct.Reader & reader):
     return _DynamicStructReader()._init(reader, None)
 
-cdef public void wrap_remote_call(PyObject * func, Response & r) except *:
+cdef public PyObject * wrap_remote_call(PyObject * func, Response & r) except *:
     response = _Response()._init_childptr(new Response(moveResponse(r)), None)
 
     func_obj = <object>func
     # TODO: decref func?
-    func_obj(response)
+    ret = func_obj(response)
+    Py_INCREF(ret)
+    return <PyObject *>ret
 
 cdef _find_field_order(struct_node):
     return [f.name for f in sorted(struct_node.fields, key=_attrgetter('codeOrder'))]
@@ -44,24 +47,29 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_
     func = getattr(server, method_name+'_context', None)
     if func is not None:
         ret = func(context)
+        Py_INCREF(ret) #TODO: stop leaking this
         if ret is not None:
             if type(ret) is _VoidPromise:
                 return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+            elif type(ret) is Promise:
+                return new VoidPromise(helpers.convert_to_voidpromise(deref((<Promise>ret).thisptr)))
             else:
                 try:
-                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise: return = %s' % (method_name, str(ret))
+                    warning_msg = 'Server function (%s) returned a value that was not a Promise: return = %s' % (method_name, str(ret))
                 except:
-                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise' % (method_name)
+                    warning_msg = 'Server function (%s) returned a value that was not a Promise' % (method_name)
                 _warnings.warn_explicit(warning_msg, UserWarning, _inspect.getsourcefile(func), _inspect.getsourcelines(func)[1])
 
         if ret is not None:
-            if type(ret) is _VoidPromise:
+            if type(ret) is Promise:
-                return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+                return new VoidPromise(helpers.convert_to_voidpromise(deref((<Promise>ret).thisptr)))
+            elif type(ret) is Promise:
+                return new VoidPromise(helpers.convert_to_voidpromise(deref((<Promise>ret).thisptr)))
             else:
                 try:
-                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise: return = %s' % (method_name, str(ret))
+                    warning_msg = 'Server function (%s) returned a value that was not a Promise: return = %s' % (method_name, str(ret))
                 except:
-                    warning_msg = 'Server function (%s) returned a value that was not a VoidPromise' % (method_name)
+                    warning_msg = 'Server function (%s) returned a value that was not a Promise' % (method_name)
                 _warnings.warn_explicit(warning_msg, UserWarning, _inspect.getsourcefile(func), _inspect.getsourcelines(func)[1])
     else:
         func = getattr(server, method_name) # will raise if no function found
@@ -69,10 +77,13 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_
         params_dict = {name : getattr(params, name) for name in params.schema.fieldnames}
         params_dict['_context'] = context
         ret = func(**params_dict)
+        Py_INCREF(ret) #TODO: stop leaking this
 
         if ret is not None:
             if type(ret) is _VoidPromise:
                 return new VoidPromise(moveVoidPromise(deref((<_VoidPromise>ret).thisptr)))
+            elif type(ret) is Promise:
+                return new VoidPromise(helpers.convert_to_voidpromise(deref((<Promise>ret).thisptr)))
             if not isinstance(ret, tuple):
                 ret = (ret,)
             names = _find_field_order(context.results.schema.node.struct)
@@ -85,16 +96,29 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_
 
     return NULL
     
-cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicStruct.Reader & _reader) except *:
+cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except *:
     restorer = <object>_restorer
-    reader = _DynamicStructReader()._init(_reader, None)
+    reader = _DynamicObjectReader()._init(_reader, None)
 
-    ret = restorer.restore(reader)
+    ret = restorer._restore(reader)
     cdef _DynamicCapabilityServer server = ret
     cdef _InterfaceSchema schema = ret.schema
 
     return new C_Capability.Client(helpers.server_to_client(schema.thisptr, <PyObject *>server))
 
+
+cdef public convert_array_pyobject(PyArray & arr):
+    return [<object>arr[i] for i in range(arr.size())]
+
+cdef public PyPromise * extract_promise(object obj):
+    if type(obj) is Promise:
+        promise = <Promise>obj
+        promise.is_consumed = True
+        Py_INCREF(promise) # TODO: fix leak
+        return promise.thisptr
+
+    return NULL
+
 cdef extern from "<kj/string.h>" namespace " ::kj":
     String strStructReader" ::kj::str"(C_DynamicStruct.Reader)
     String strStructBuilder" ::kj::str"(C_DynamicStruct.Builder)
@@ -234,8 +258,8 @@ ctypedef fused _DynamicSetterClasses:
     C_DynamicStruct.Builder
     Request
 
-ctypedef fused _PromiseTypes:
+ctypedef fused PromiseTypes:
-    _Promise
+    Promise
     _RemotePromise
     _VoidPromise
     PromiseFulfillerPair
@@ -637,7 +661,7 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent):
     elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer):
         thisptr.set(field, _extract_dynamic_server(value))
     else:
-        raise ValueError("Non primitive type")
+        raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
 
 cdef _setDynamicFieldPtr(_DynamicSetterClasses * thisptr, field, value, parent):
     cdef C_DynamicValue.Reader temp
@@ -672,7 +696,7 @@ cdef _setDynamicFieldPtr(_DynamicSetterClasses * thisptr, field, value, parent):
     elif value_type is _DynamicCapabilityClient:
         thisptr.set(field, _extract_dynamic_client(value))
     else:
-        raise ValueError("Non primitive type")
+        raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value))))
 
 cdef _to_dict(msg, bint verbose):
     msg_type = type(msg)
@@ -1108,7 +1132,7 @@ cdef class _DynamicObjectReader:
         self._parent = parent
         return self
 
-    cpdef as_struct(self, schema):
+    cpdef as_struct(self, schema) except +reraise_kj_exception:
         cdef _StructSchema s
         if hasattr(schema, 'schema'):
             s = schema.schema
@@ -1117,6 +1141,9 @@ cdef class _DynamicObjectReader:
 
         return _DynamicStructReader()._init(self.thisptr.getAs(s.thisptr), self._parent)
 
+    cpdef as_text(self) except +reraise_kj_exception:
+        return (<char*>self.thisptr.getAsText().cStr())[:]
+
 cdef class _DynamicObjectBuilder:
     cdef C_DynamicObject.Builder * thisptr
     cdef public object _parent
@@ -1129,7 +1156,7 @@ cdef class _DynamicObjectBuilder:
     def __dealloc__(self):
         del self.thisptr
 
-    cpdef as_struct(self, schema):
+    cpdef as_struct(self, schema) except +reraise_kj_exception:
         cdef _StructSchema s
         if hasattr(schema, 'schema'):
             s = schema.schema
@@ -1141,6 +1168,9 @@ cdef class _DynamicObjectBuilder:
     cpdef set_as_text(self, text):
         self.thisptr.setAsText(text)
 
+    cpdef as_text(self) except +reraise_kj_exception:
+        return (<char*>self.thisptr.getAsText().cStr())[:]
+
 cdef class _EventLoop:
     cdef capnp.AsyncIoContext * thisptr
 
@@ -1164,6 +1194,11 @@ cdef class _EventLoop:
 
 cdef _EventLoop C_DEFAULT_EVENT_LOOP = _EventLoop()
 
+cpdef reset_event_loop():
+    global C_DEFAULT_EVENT_LOOP
+    C_DEFAULT_EVENT_LOOP._remove()
+    C_DEFAULT_EVENT_LOOP = _EventLoop()
+
 cdef class _CallContext:
     cdef CallContext * thisptr
 
@@ -1192,18 +1227,28 @@ cdef class _CallContext:
         self.thisptr.allowCancellation()
 
     cpdef tail_call(self, _Request tailRequest):
-        return _VoidPromise()._init(self.thisptr.tailCall(moveRequest(deref(tailRequest.thisptr_child))))
+        promise = _VoidPromise()._init(self.thisptr.tailCall(moveRequest(deref(tailRequest.thisptr_child))))
+        promise.is_consumed = True
+        return promise
 
-cdef class _Promise:
+cdef class Promise:
     cdef PyPromise * thisptr
     cdef public bint is_consumed
+    cdef public object _parent, _obj
 
-    def __init__(self):
+    def __init__(self, obj=None):
-        self.is_consumed = True
+        if obj is None:
+            self.is_consumed = True
+        else:
+            self.is_consumed = False
+            self._obj = obj
+            Py_INCREF(obj) # TODO: fix this
+            self.thisptr = new PyPromise(<PyObject *>obj)
 
-    cdef _init(self, PyPromise other):
+    cdef _init(self, PyPromise other, parent=None):
         self.is_consumed = False
         self.thisptr = new PyPromise(movePromise(other))
+        self._parent = parent
         return self
 
     def __dealloc__(self):
@@ -1225,7 +1270,7 @@ cdef class _Promise:
         Py_INCREF(func)
         Py_INCREF(error_func)
 
-        return _Promise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func))
+        return Promise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func), self)
 
 cdef class _VoidPromise:
     cdef VoidPromise * thisptr
@@ -1257,7 +1302,12 @@ cdef class _VoidPromise:
         Py_INCREF(func)
         Py_INCREF(error_func)
 
-        return _Promise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func))
+        return Promise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func), self)
+
+    cpdef as_pypromise(self) except +reraise_kj_exception:
+        if self.is_consumed:
+            raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object')
+        Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)), self)
 
 cdef class _RemotePromise:
     cdef RemotePromise * thisptr
@@ -1273,8 +1323,8 @@ cdef class _RemotePromise:
         self._parent = parent
         return self
 
-    def __dealloc__(self):
+    # def __dealloc__(self):
-        del self.thisptr
+    #     del self.thisptr
 
     cpdef wait(self) except +reraise_kj_exception:
         if self.is_consumed:
@@ -1286,7 +1336,9 @@ cdef class _RemotePromise:
         return ret
 
     cpdef as_pypromise(self) except +reraise_kj_exception:
-        _Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)))
+        if self.is_consumed:
+            raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object')
+        Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)), self)
 
     cpdef then(self, func, error_func=None) except +reraise_kj_exception:
         if self.is_consumed:
@@ -1295,7 +1347,7 @@ cdef class _RemotePromise:
         Py_INCREF(func)
         Py_INCREF(error_func)
 
-        return _VoidPromise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func))
+        return Promise()._init(helpers.then(deref(self.thisptr), <PyObject *>func, <PyObject *>error_func), self)
 
     cpdef _get(self, field) except +reraise_kj_exception:
         cdef int type = (<C_DynamicValue.Pipeline>self.thisptr.get(field)).getType()
@@ -1322,6 +1374,26 @@ cdef class _RemotePromise:
     def to_dict(self, verbose=False):
         return _to_dict(self, verbose)
 
+cpdef join_promises(promises) except +reraise_kj_exception:
+    heap = capnp.heapArrayBuilderPyPromise(len(promises))
+
+    new_promises = []
+    new_promises_append = new_promises.append
+
+    for promise in promises:
+        promise_type = type(promise)
+        if promise_type is Promise:
+            pyPromise = <Promise>promise
+        elif promise_type is _RemotePromise or promise_type is _VoidPromise:
+            pyPromise = <Promise>promise.as_pypromise()
+            new_promises_append(pyPromise)
+        else:
+            raise ValueError('One of the promises passed to `join_promises` had a non promise value of: ' + str(promise))
+        heap.add(movePromise(deref(pyPromise.thisptr)))
+        pyPromise.is_consumed = True
+
+    return Promise()._init(helpers.then(capnp.joinPromises(heap.finish())))
+
 cdef class _Request(_DynamicStructBuilder):
     cdef Request * thisptr_child
 
@@ -1330,6 +1402,7 @@ cdef class _Request(_DynamicStructBuilder):
         self._init(<C_DynamicStruct.Builder>deref(self.thisptr_child), parent)
         return self
 
+    #TODO: dealloc
     cpdef send(self):
         return _RemotePromise()._init(self.thisptr_child.send(), self._parent)
 
@@ -1341,6 +1414,7 @@ cdef class _Response(_DynamicStructReader):
         self._init(<C_DynamicStruct.Reader>deref(self.thisptr_child), parent)
         return self
 
+    #TODO: dealloc
     cdef _init_childptr(self, Response * other, parent):
         self.thisptr_child = other
         self._init(<C_DynamicStruct.Reader>deref(self.thisptr_child), parent)
@@ -1483,26 +1557,20 @@ cdef class _CapabilityClient:
             s = schema
         return _DynamicCapabilityClient()._init(self.thisptr.castAs(s.thisptr), self._parent)
 
-cdef class Restorer:
+cdef class _Restorer:
     cdef PyRestorer * thisptr
-    cdef C_StructSchema schema
-
     cdef public object restore
 
-    def __init__(self, schema, restore_func):
+    def __init__(self, restore):
-        cdef _StructSchema s
+        self.thisptr = new PyRestorer(<PyObject*>self)
-        if hasattr(schema, 'schema'):
+        self.restore = restore
-            s = schema.schema
-        else:
-            s = schema
-
-        self.schema = s.thisptr
-        self.restore = restore_func
-        self.thisptr = new PyRestorer(<PyObject*>self, self.schema)
 
     def __dealloc__(self):
         del self.thisptr
 
+    def _restore(self, obj):
+        return self.restore(obj)
+
 cdef class _TwoPartyVatNetwork:
     cdef Own[C_TwoPartyVatNetwork] thisptr
 
@@ -1510,21 +1578,34 @@ cdef class _TwoPartyVatNetwork:
         self.thisptr = makeTwoPartyVatNetwork(stream, side)
         return self
 
-cdef class RpcClient:
+cdef _Restorer _convert_restorer(restorer):
+    if isinstance(restorer, _RestorerImpl):
+        return _Restorer(restorer._restore)
+    elif type(restorer) is _Restorer:
+        return restorer
+    elif hasattr(restorer, 'restore'):
+        return _Restorer(restorer.restore)
+    elif callable(restorer):
+        return _Restorer(restorer)
+    else:
+        raise ValueError("Restorer object ({}) isn't able to be used as a restore".format(str(restorer)))
+
+cdef class TwoPartyClient:
     cdef RpcSystem * thisptr
     cdef public _TwoPartyVatNetwork network
-    cdef public object restorer, _stream
+    cdef public object _stream
+    cdef public _Restorer restorer
     cdef public _FdAsyncIoStream stream
 
-    def __init__(self, stream, Restorer restorer=None):
+    def __init__(self, stream, restorer=None):
         self._stream = stream
         self.stream = _FdAsyncIoStream(stream.fileno())
         self.network = _TwoPartyVatNetwork()._init(deref(self.stream.thisptr), capnp.CLIENT)
         if restorer is None:
             self.thisptr = new RpcSystem(makeRpcClient(deref(self.network.thisptr)))
         else:
-            self.restorer = restorer
+            self.restorer = _convert_restorer(restorer)
-            self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self.network.thisptr), deref(restorer.thisptr)))
+            self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self.network.thisptr), deref(self.restorer.thisptr)))
 
     def __dealloc__(self):
         del self.thisptr
@@ -1568,18 +1649,23 @@ cdef class RpcClient:
 
         return self.restore(ref.objectId)
 
-cdef class RpcServer:
+cdef class TwoPartyServer:
     cdef RpcSystem * thisptr
     cdef public _TwoPartyVatNetwork network
-    cdef public object restorer, _stream
+    cdef public object _stream
+    cdef public _Restorer restorer
     cdef public _FdAsyncIoStream stream
 
-    def __init__(self, stream, Restorer restorer):
+    def __init__(self, stream, restorer):
         self._stream = stream
         self.stream = _FdAsyncIoStream(stream.fileno())
-        self.restorer = restorer
+        Py_INCREF(self._stream)
+        Py_INCREF(self.stream) # TODO: attach this to onDrained, also figure out what's leaking
+        self.restorer = _convert_restorer(restorer)
         self.network = _TwoPartyVatNetwork()._init(deref(self.stream.thisptr), capnp.SERVER)
-        self.thisptr = new RpcSystem(makeRpcServer(deref(self.network.thisptr), deref(restorer.thisptr)))
+        self.thisptr = new RpcSystem(makeRpcServer(deref(self.network.thisptr), deref(self.restorer.thisptr)))
+        Py_INCREF(self.restorer) # TODO: attach this to onDrained, also figure out what's leaking
+        Py_INCREF(self.network) # TODO: attach this to onDrained, also figure out what's leaking
 
     def __dealloc__(self):
         del self.thisptr
@@ -1751,9 +1837,17 @@ cdef _new_message(self, kwargs):
         _from_dict(msg, kwargs)
     return msg
 
+class _RestorerImpl(object):
+    pass
+
 class _StructModule(object):
-    def __init__(self, schema):
+    def __init__(self, schema, name):
+        def blank_init(server_self):
+            pass
+        def _restore(self, obj):
+            return self.restore(obj.as_struct(self.schema))
         self.schema = schema
+        self.Restorer = type(name + '.Restorer', (_RestorerImpl,), {'schema':schema, '_restore':_restore})
 
     def read(self, file, traversal_limit_in_words = None, nesting_limit = None):
         """Returns a Reader for the unpacked object read from file.
@@ -1876,7 +1970,7 @@ class _InterfaceModule(object):
         def server_init(server_self):
             pass
         self.schema = schema
-        self.Server = type(name, (_DynamicCapabilityServer,), {'__init__': server_init, 'schema':schema})
+        self.Server = type(name + '.Server', (_DynamicCapabilityServer,), {'__init__': server_init, 'schema':schema})
 
     def _new_client(self, server):
         return _DynamicCapabilityClient()._init_vals(self.schema, server)
@@ -1955,7 +2049,7 @@ cdef class SchemaParser:
                 schema = nodeSchema.get_nested(node.name)
                 proto = schema.get_proto()
                 if proto.isStruct:
-                    local_module = _StructModule(schema.as_struct())
+                    local_module = _StructModule(schema.as_struct(), node.name)
                     class Reader(_DynamicStructReader):
                         """An abstract base class.  Readers are 'instances' of this class."""
                         __metaclass__ = _StructABCMeta

examples/c++.capnp

@@ -1,27 +0,0 @@
-# Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice, this
-#    list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-#    this list of conditions and the following disclaimer in the documentation
-#    and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-@0xbdf87d7bb8304e81;
-$namespace("capnp::annotations");
-
-annotation namespace(file): Text;

examples/calculator-client.py

@@ -6,7 +6,6 @@ import socket
 import capnp
 
 import calculator_capnp
-import rpc_capnp
 
 class PowerFunction(calculator_capnp.Calculator.Function.Server):
     '''An implementation of the Function interface wrapping pow().  Note that
@@ -18,7 +17,7 @@ class PowerFunction(calculator_capnp.Calculator.Function.Server):
         return pow(params[0], params[1])
 
 def parse_args():
-    parser = argparse.ArgumentParser('Connects to the Calculator server at the given address and does some RPCs')
+    parser = argparse.ArgumentParser(usage='Connects to the Calculator server at the given address and does some RPCs')
     parser.add_argument("host", help="HOST:PORT")
 
     return parser.parse_args()
@@ -27,7 +26,7 @@ def main():
     host, port = parse_args().host.split(':')
 
     sock = socket.create_connection((host, port))
-    client = capnp.RpcClient(sock)
+    client = capnp.TwoPartyClient(sock)
 
     # Pass "calculator" to ez_restore (there's also a `restore` function that takes a struct or AnyPointer as an argument), and then cast the returned capability to it's proper type. This casting is due to capabilities not having a reference to their schema
     calculator = client.ez_restore('calculator').cast_as(calculator_capnp.Calculator)

test/test_rpc.py

@@ -12,17 +12,60 @@ class Server(test_capability_capnp.TestInterface.Server):
     def foo(self, i, j, **kwargs):
         return str(i * 5 + self.val)
 
-def test_simple_rpc():
+class TypelessRestorer:
-    def _restore(ref_id):
+    def restore(self, ref_id):
         return Server(100)
 
+def restore_func(ref_id):
+    return Server(100)
+
+class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer):
+    def restore(self, ref_id):
+        assert ref_id.tag == 'testInterface'
+        return Server(100)
+
+def test_simple_rpc():
+
     read, write = socket.socketpair(socket.AF_UNIX)
 
-    restorer = capnp.Restorer(test_capability_capnp.TestSturdyRefObjectId, _restore)
+    restorer = SimpleRestorer()
-    server = capnp.RpcServer(write, restorer)
+    server = capnp.TwoPartyServer(write, restorer)
-    client = capnp.RpcClient(read)
+    client = capnp.TwoPartyClient(read)
 
-    ref = test_capability_capnp.TestSturdyRefObjectId.new_message()
+    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
+    cap = client.restore(ref)
+    cap = cap.cast_as(test_capability_capnp.TestInterface)
+
+    remote = cap.foo(i=5)
+    response = remote.wait()
+
+    assert response.x == '125'
+
+def test_simple_rpc_typeless_restorer():
+
+    read, write = socket.socketpair(socket.AF_UNIX)
+
+    restorer = TypelessRestorer()
+    server = capnp.TwoPartyServer(write, restorer)
+    client = capnp.TwoPartyClient(read)
+
+    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
+    cap = client.restore(ref)
+    cap = cap.cast_as(test_capability_capnp.TestInterface)
+
+    remote = cap.foo(i=5)
+    response = remote.wait()
+
+    assert response.x == '125'
+
+def test_simple_rpc_restore_func():
+
+    read, write = socket.socketpair(socket.AF_UNIX)
+
+    server = capnp.TwoPartyServer(write, restore_func)
+    client = capnp.TwoPartyClient(read)
+
+    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
     cap = client.restore(ref)
     cap = cap.cast_as(test_capability_capnp.TestInterface)