Refactor layout of pxd files

capnp/helpers/capabilityHelper.h

@@ -0,0 +1,154 @@
+#pragma once
+
+#include "capnp/dynamic.h"
+#include <stdexcept>
+#include "Python.h"
+#include <iostream>
+
+extern "C" {
+   void wrap_remote_call(PyObject * func, capnp::Response<capnp::DynamicStruct> &);
+   PyObject * wrap_dynamic_struct_reader(capnp::DynamicStruct::Reader &);
+   ::kj::Promise<void> * call_server_method(PyObject * py_server, char * name, capnp::CallContext< capnp::DynamicStruct, capnp::DynamicStruct> & context);
+   PyObject * wrap_kj_exception(kj::Exception &);
+   PyObject * wrap_kj_exception_for_reraise(kj::Exception &);
+   PyObject * get_exception_info(PyObject *, PyObject *, PyObject *);
+ }
+
+void reraise_kj_exception() {
+  try {
+    if (PyErr_Occurred())
+      ; // let the latest Python exn pass through and ignore the current one
+    else
+      throw;
+  }
+  catch (kj::Exception& exn) {
+    auto obj = wrap_kj_exception_for_reraise(exn);
+    PyErr_SetObject((PyObject*)obj->ob_type, obj);
+  }
+  catch (const std::exception& exn) {
+    PyErr_SetString(PyExc_RuntimeError, exn.what());
+  }
+  catch (...)
+  {
+    PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
+  }
+}
+
+void check_py_error() {
+    PyObject * err = PyErr_Occurred();
+    if(err) {
+        PyObject * ptype, *pvalue, *ptraceback;
+        PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+
+        PyObject * info = get_exception_info(ptype, pvalue, ptraceback);
+
+        PyObject * py_filename = PyTuple_GetItem(info, 0);
+        kj::String filename(kj::heapString(PyBytes_AsString(py_filename)));
+
+        PyObject * py_line = PyTuple_GetItem(info, 1);
+        int line = PyInt_AsLong(py_line);
+
+        PyObject * py_description = PyTuple_GetItem(info, 2);
+        kj::String description(kj::heapString(PyBytes_AsString(py_description)));
+
+        Py_DECREF(ptype);
+        Py_DECREF(pvalue);
+        Py_DECREF(ptraceback);
+        Py_DECREF(info);
+        PyErr_Clear();
+
+        throw kj::Exception(kj::Exception::Nature::OTHER, kj::Exception::Durability::PERMANENT, kj::mv(filename), line, kj::mv(description));
+    }
+}
+
+// TODO: need to decref error_func as well on successful run
+PyObject * wrapPyFunc(PyObject * func, PyObject * arg) {
+    PyObject * result = PyObject_CallFunctionObjArgs(func, arg, NULL);
+    Py_DECREF(func);
+
+    check_py_error();
+    return result;
+}
+
+PyObject * wrapPyFuncNoArg(PyObject * func) {
+    PyObject * result = PyObject_CallFunctionObjArgs(func, NULL);
+    Py_DECREF(func);
+
+    check_py_error();
+    return result;
+}
+
+void wrapRemoteCall(PyObject * func, capnp::Response<capnp::DynamicStruct> & arg) {
+    wrap_remote_call(func, arg);
+
+    check_py_error();
+}
+
+::kj::Promise<PyObject *> then(kj::Promise<PyObject *> & promise, PyObject * func, PyObject * error_func) {
+  if(error_func == Py_None)
+    return promise.then([func](PyObject * arg) { return wrapPyFunc(func, arg); } );
+  else
+    return promise.then([func](PyObject * arg) { return wrapPyFunc(func, arg); } 
+                                     , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
+}
+
+::kj::Promise<void> then(::capnp::RemotePromise< ::capnp::DynamicStruct> & promise, PyObject * func, PyObject * error_func) {
+  if(error_func == Py_None)
+    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { wrapRemoteCall(func, arg); } );
+  else
+    return promise.then([func](capnp::Response<capnp::DynamicStruct>&& arg) { wrapRemoteCall(func, arg); } 
+                                     , [error_func](kj::Exception arg) { wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
+}
+
+::kj::Promise<PyObject *> then(kj::Promise<void> & promise, PyObject * func, PyObject * error_func) {
+  if(error_func == Py_None)
+    return promise.then([func]() { return wrapPyFuncNoArg(func); } );
+  else
+    return promise.then([func]() { return wrapPyFuncNoArg(func); } 
+                                     , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } );
+}
+
+class PythonInterfaceDynamicImpl final: public capnp::DynamicCapability::Server {
+public:
+  PyObject * py_server;
+
+  PythonInterfaceDynamicImpl(capnp::InterfaceSchema & schema, PyObject * _py_server)
+      : capnp::DynamicCapability::Server(schema), py_server(_py_server) {
+        Py_INCREF(_py_server);
+      }
+
+  ~PythonInterfaceDynamicImpl() {
+    Py_DECREF(py_server);
+  }
+
+  kj::Promise<void> call(capnp::InterfaceSchema::Method method,
+                         capnp::CallContext< capnp::DynamicStruct, capnp::DynamicStruct> context) {
+    auto methodName = method.getProto().getName();
+
+    kj::Promise<void> * promise = call_server_method(py_server, const_cast<char *>(methodName.cStr()), context);
+
+    check_py_error();
+
+    if(promise == nullptr)
+      return kj::READY_NOW;
+
+    kj::Promise<void> ret(kj::mv(*promise));
+    delete promise;
+    return ret;
+  }
+};
+
+capnp::DynamicCapability::Client new_client(capnp::InterfaceSchema & schema, PyObject * server) {
+  return capnp::DynamicCapability::Client(kj::heap<PythonInterfaceDynamicImpl>(schema, server));
+}
+capnp::DynamicValue::Reader new_server(capnp::InterfaceSchema & schema, PyObject * server) {
+  return capnp::DynamicValue::Reader(kj::heap<PythonInterfaceDynamicImpl>(schema, server));
+}
+
+capnp::Capability::Client server_to_client(capnp::InterfaceSchema & schema, PyObject * server) {
+  return kj::heap<PythonInterfaceDynamicImpl>(schema, server);
+}
+
+::kj::Promise<PyObject *> convert_to_pypromise(capnp::RemotePromise<capnp::DynamicStruct> & promise) {
+    return promise.then([](capnp::Response<capnp::DynamicStruct>&& response) { return wrap_dynamic_struct_reader(response); } );
+}

capnp/helpers/checkCompiler.h

@@ -0,0 +1,11 @@
+#ifdef __GNUC__
+  #if __clang__
+    #if __cplusplus >= 201103L && !__has_include(<initializer_list>)
+      #warning "Your compiler supports C++11 but your C++ standard library does not.  If your system has libc++ installed (as should be the case on e.g. Mac OSX), try adding -stdlib=libc++ to your CFLAGS (ignore the other warning that says to use CXXFLAGS)."
+    #endif
+  #endif
+#endif
+
+#include "capnp/dynamic.h"
+
+static_assert(CAPNP_VERSION >= 3000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.3 and then re-install this python library");

capnp/helpers/fixMaybe.h

@@ -0,0 +1,22 @@
+#include "kj/common.h"
+#include <stdexcept>
+
+template<typename T>
+T fixMaybe(::kj::Maybe<T> val) {
+  KJ_IF_MAYBE(new_val, val) {
+    return *new_val;
+  } else {
+    throw std::invalid_argument("member was null");
+  }
+}
+
+template<typename T>
+const char * getEnumString(T & val) {
+
+  auto maybe_val = val.which();
+  KJ_IF_MAYBE(new_val, maybe_val) {
+    return new_val->getProto().getName().cStr();;
+  } else {
+    return "";
+  }
+}

capnp/helpers/helpers.pxd

@@ -0,0 +1,29 @@
+from .capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, PyPromise, VoidPromise, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork
+
+from non_circular cimport reraise_kj_exception
+
+from cpython.ref cimport PyObject
+
+cdef extern from "helpers/fixMaybe.h":
+    EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception
+    char * getEnumString(DynamicStruct.Reader val)
+    char * getEnumString(DynamicStruct.Builder val)
+    char * getEnumString(Request val)
+
+cdef extern from "helpers/capabilityHelper.h":
+    # PyPromise evalLater(EventLoop &, PyObject * func)
+    # PyPromise there(EventLoop & loop, PyPromise & promise, PyObject * func, PyObject * error_func)
+    PyPromise then(PyPromise & promise, PyObject * func, PyObject * error_func)
+    VoidPromise then(RemotePromise & promise, PyObject * func, PyObject * error_func)
+    PyPromise then(VoidPromise & promise, PyObject * func, PyObject * error_func)
+    DynamicCapability.Client new_client(InterfaceSchema&, PyObject *)
+    DynamicValue.Reader new_server(InterfaceSchema&, PyObject *)
+    Capability.Client server_to_client(InterfaceSchema&, PyObject *)
+    PyPromise convert_to_pypromise(RemotePromise&)
+
+cdef extern from "helpers/rpcHelper.h":
+    cdef cppclass PyRestorer:
+        PyRestorer(PyObject *, StructSchema&)
+    Capability.Client restoreHelper(RpcSystem&, MessageBuilder&)
+    Capability.Client restoreHelper(RpcSystem&, MessageReader&)
+    RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&)

capnp/helpers/non_circular.pxd

@@ -0,0 +1,8 @@
+from cpython.ref cimport PyObject
+
+cdef extern from "helpers/capabilityHelper.h":
+    cppclass PythonInterfaceDynamicImpl:
+        PythonInterfaceDynamicImpl(PyObject *)
+
+cdef extern from "helpers/capabilityHelper.h":
+    void reraise_kj_exception()

capnp/helpers/rpcHelper.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include "capnp/dynamic.h"
+#include "capnp/rpc-twoparty.h"
+#include "Python.h"
+#include "capabilityHelper.h"
+
+extern "C" {
+   capnp::Capability::Client * call_py_restorer(PyObject *, capnp::DynamicStruct::Reader &);
+}
+
+class PyRestorer final: public capnp::SturdyRefRestorer<capnp::AnyPointer> {
+public:
+  PyRestorer(PyObject * _py_restorer, capnp::StructSchema& _schema): py_restorer(_py_restorer), schema(_schema) {
+    // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere.
+    // Py_INCREF(py_restorer);
+  }
+
+  // ~PyRestorer() {
+  //   Py_DECREF(py_restorer);
+  // }
+  
+  capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override {
+    auto reader = objectId.getAs<capnp::DynamicStruct>(schema);
+    capnp::Capability::Client * ret = call_py_restorer(py_restorer, reader);
+    check_py_error();
+    capnp::Capability::Client stack_ret(*ret);
+    delete ret;
+
+    return stack_ret;
+  }
+
+private:
+  PyObject * py_restorer;
+  capnp::StructSchema schema;
+};
+
+capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::MessageBuilder & objectId) {  capnp::MallocMessageBuilder hostIdMessage(8);
+    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
+    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
+    return client.restore(hostId, objectId.getRoot<capnp::AnyPointer>());
+}
+
+
+capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::MessageReader & objectId) {  capnp::MallocMessageBuilder hostIdMessage(8);
+    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
+    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
+    return client.restore(hostId, objectId.getRoot<capnp::AnyPointer>());
+}
+
+template <typename SturdyRefHostId, typename ProvisionId,
+          typename RecipientId, typename ThirdPartyCapId, typename JoinAnswer>
+capnp::RpcSystem<SturdyRefHostId> makeRpcClientWithRestorer(
+    capnp::VatNetwork<SturdyRefHostId, ProvisionId, RecipientId, ThirdPartyCapId, JoinAnswer>& network,
+    PyRestorer& restorer) {
+    using namespace capnp;
+  return RpcSystem<SturdyRefHostId>(network,
+      kj::Maybe<SturdyRefRestorer<AnyPointer>&>(restorer));
+}