IQ.Lvbs/pycapnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minimal pycapnp

Browse Source
This commit is contained in:
Adeeb Shihadeh
2026-09-21 18:57:37 -07:00
parent a0cb5cdf06
commit 5befba15f3
125 changed files with 423 additions and 14793 deletions
Download Patch File Download Diff File Expand all files Collapse all files

199
capnp/helpers/capabilityHelper.cpp
View File

@@ -1,199 +0,0 @@
#include "capnp/helpers/capabilityHelper.h"
#include "capnp/lib/capnp_api.h"
::kj::Promise<kj::Own<PyRefCounter>> convert_to_pypromise(capnp::RemotePromise<capnp::DynamicStruct> promise) {
return promise.then([](capnp::Response<capnp::DynamicStruct>&& response) {
return stealPyRef(wrap_dynamic_struct_reader(response)); } );
}
void c_reraise_kj_exception() {
GILAcquire gil;
try {
if (PyErr_Occurred())
; // let the latest Python exn pass through and ignore the current one
else
throw;
}
catch (kj::Exception& exn) {
auto obj = wrap_kj_exception_for_reraise(exn);
if (obj == nullptr) {
return;
}
PyErr_SetObject((PyObject*)obj->ob_type, obj);
Py_DECREF(obj);
}
catch (const std::exception& exn) {
PyErr_SetString(PyExc_RuntimeError, exn.what());
}
catch (...)
{
PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
}
}
void check_py_error() {
GILAcquire gil;
PyObject * err = PyErr_Occurred();
if(err) {
PyObject * ptype, *pvalue, *ptraceback;
PyErr_Fetch(&ptype, &pvalue, &ptraceback);
if(ptype == NULL || pvalue == NULL || ptraceback == NULL)
throw kj::Exception(kj::Exception::Type::FAILED, kj::heapString("capabilityHelper.h"), 44, kj::heapString("Unknown error occurred"));
PyObject * info = get_exception_info(ptype, pvalue, ptraceback);
PyObject * py_filename = PyTuple_GetItem(info, 0);
kj::String filename(kj::heapString(PyBytes_AsString(py_filename)));
PyObject * py_line = PyTuple_GetItem(info, 1);
int line = PyLong_AsLong(py_line);
PyObject * py_description = PyTuple_GetItem(info, 2);
kj::String description(kj::heapString(PyBytes_AsString(py_description)));
Py_DECREF(ptype);
Py_DECREF(pvalue);
Py_DECREF(ptraceback);
Py_DECREF(info);
PyErr_Clear();
throw kj::Exception(kj::Exception::Type::FAILED, kj::mv(filename), line, kj::mv(description));
}
}
kj::Promise<kj::Own<PyRefCounter>> wrapPyFunc(kj::Own<PyRefCounter> func, kj::Own<PyRefCounter> arg) {
GILAcquire gil;
PyObject * result = PyObject_CallFunctionObjArgs(func->obj, arg->obj, NULL);
check_py_error();
return stealPyRef(result);
}
::kj::Promise<kj::Own<PyRefCounter>> then(kj::Promise<kj::Own<PyRefCounter>> promise,
kj::Own<PyRefCounter> func, kj::Own<PyRefCounter> error_func) {
if(error_func->obj == Py_None)
return promise.then([func=kj::mv(func)](kj::Own<PyRefCounter> arg) mutable {
return wrapPyFunc(kj::mv(func), kj::mv(arg)); } );
else
return promise.then
([func=kj::mv(func)](kj::Own<PyRefCounter> arg) mutable {
return wrapPyFunc(kj::mv(func), kj::mv(arg)); },
[error_func=kj::mv(error_func)](kj::Exception arg) mutable {
return wrapPyFunc(kj::mv(error_func), stealPyRef(wrap_kj_exception(arg))); } );
}
kj::Promise<void> PythonInterfaceDynamicImpl::call(capnp::InterfaceSchema::Method method,
capnp::CallContext< capnp::DynamicStruct,
capnp::DynamicStruct> context) {
auto methodName = method.getProto().getName();
kj::Promise<void> * promise = call_server_method(this->py_server->obj,
const_cast<char *>(methodName.cStr()),
context,
this->kj_loop->obj);
check_py_error();
if(promise == nullptr)
return kj::READY_NOW;
kj::Promise<void> ret(kj::mv(*promise));
delete promise;
return ret;
};
class ReadPromiseAdapter {
public:
ReadPromiseAdapter(kj::PromiseFulfiller<size_t>& fulfiller, PyObject* protocol,
void* buffer, size_t minBytes, size_t maxBytes)
: protocol(protocol) {
_asyncio_stream_read_start(protocol, buffer, minBytes, maxBytes, fulfiller);
}
~ReadPromiseAdapter() {
_asyncio_stream_read_stop(protocol);
}
private:
PyObject* protocol;
};
class WritePromiseAdapter {
public:
WritePromiseAdapter(kj::PromiseFulfiller<void>& fulfiller, PyObject* protocol,
kj::ArrayPtr<const kj::ArrayPtr<const kj::byte>> pieces)
: protocol(protocol) {
_asyncio_stream_write_start(protocol, pieces, fulfiller);
}
~WritePromiseAdapter() {
_asyncio_stream_write_stop(protocol);
}
private:
PyObject* protocol;
};
PyAsyncIoStream::~PyAsyncIoStream() {
_asyncio_stream_close(protocol->obj);
}
kj::Promise<size_t> PyAsyncIoStream::tryRead(void* buffer, size_t minBytes, size_t maxBytes) {
return kj::newAdaptedPromise<size_t, ReadPromiseAdapter>(protocol->obj, buffer, minBytes, maxBytes);
}
kj::Promise<void> PyAsyncIoStream::write(const void* buffer, size_t size) {
KJ_UNIMPLEMENTED("No use-case AsyncIoStream::write was found yet.");
}
kj::Promise<void> PyAsyncIoStream::write(kj::ArrayPtr<const kj::ArrayPtr<const kj::byte>> pieces) {
return kj::newAdaptedPromise<void, WritePromiseAdapter>(protocol->obj, pieces);
}
kj::Promise<void> PyAsyncIoStream::whenWriteDisconnected() {
// TODO: Possibly connect this to protocol.connection_lost?
return kj::NEVER_DONE;
}
void PyAsyncIoStream::shutdownWrite() {
_asyncio_stream_shutdown_write(protocol->obj);
}
class TaskToPromiseAdapter {
public:
TaskToPromiseAdapter(kj::PromiseFulfiller<void>& fulfiller,
kj::Own<PyRefCounter> task, PyObject* callback)
: task(kj::mv(task)) {
promise_task_add_done_callback(this->task->obj, callback, fulfiller);
}
~TaskToPromiseAdapter() {
promise_task_cancel(this->task->obj);
}
private:
kj::Own<PyRefCounter> task;
};
kj::Promise<void> taskToPromise(kj::Own<PyRefCounter> task, PyObject* callback) {
return kj::newAdaptedPromise<void, TaskToPromiseAdapter>(kj::mv(task), callback);
}
::kj::Promise<kj::Own<PyRefCounter>> tryReadMessage(kj::AsyncIoStream& stream, capnp::ReaderOptions opts) {
return capnp::tryReadMessage(stream, opts)
.then([](kj::Maybe<kj::Own<capnp::MessageReader>> maybeReader) -> kj::Promise<kj::Own<PyRefCounter>> {
KJ_IF_MAYBE(reader, maybeReader) {
PyObject* pyreader = make_async_message_reader(kj::mv(*reader));
check_py_error();
return kj::heap<PyRefCounter>(pyreader);
} else {
return kj::heap<PyRefCounter>(Py_None);
}
});
}
void init_capnp_api() {
import_capnp__lib__capnp();
}

139
capnp/helpers/capabilityHelper.h
View File

@@ -1,139 +0,0 @@
#pragma once
#include "capnp/dynamic.h"
#include <kj/async-io.h>
#include <capnp/serialize-async.h>
#include <stdexcept>
#include "Python.h"
class GILAcquire {
public:
GILAcquire() : gstate(PyGILState_Ensure()) {}
~GILAcquire() {
PyGILState_Release(gstate);
}
PyGILState_STATE gstate;
};
class GILRelease {
public:
GILRelease() {
Py_UNBLOCK_THREADS
}
~GILRelease() {
Py_BLOCK_THREADS
}
PyThreadState *_save; // The macros above read/write from this variable
};
class PyRefCounter {
public:
PyObject * obj;
PyRefCounter(PyObject * o) : obj(o) {
GILAcquire gil;
Py_INCREF(obj);
}
PyRefCounter(const PyRefCounter & ref) : obj(ref.obj) {
GILAcquire gil;
Py_INCREF(obj);
}
~PyRefCounter() {
GILAcquire gil;
Py_DECREF(obj);
}
};
inline kj::Own<PyRefCounter> stealPyRef(PyObject* o) {
auto ret = kj::heap<PyRefCounter>(o);
Py_DECREF(o);
return ret;
}
::kj::Promise<kj::Own<PyRefCounter>> convert_to_pypromise(capnp::RemotePromise<capnp::DynamicStruct> promise);
inline ::kj::Promise<kj::Own<PyRefCounter>> convert_to_pypromise(kj::Promise<void> promise) {
return promise.then([]() {
GILAcquire gil;
return kj::heap<PyRefCounter>(Py_None);
});
}
void c_reraise_kj_exception();
void check_py_error();
::kj::Promise<kj::Own<PyRefCounter>> then(kj::Promise<kj::Own<PyRefCounter>> promise,
kj::Own<PyRefCounter> func, kj::Own<PyRefCounter> error_func);
class PythonInterfaceDynamicImpl final: public capnp::DynamicCapability::Server {
public:
kj::Own<PyRefCounter> py_server;
kj::Own<PyRefCounter> kj_loop;
#if (CAPNP_VERSION_MAJOR < 1)
PythonInterfaceDynamicImpl(capnp::InterfaceSchema & schema,
kj::Own<PyRefCounter> _py_server,
kj::Own<PyRefCounter> kj_loop)
: capnp::DynamicCapability::Server(schema),
py_server(kj::mv(_py_server)), kj_loop(kj::mv(kj_loop)) { }
#else
PythonInterfaceDynamicImpl(capnp::InterfaceSchema & schema,
kj::Own<PyRefCounter> _py_server,
kj::Own<PyRefCounter> kj_loop)
: capnp::DynamicCapability::Server(schema, { true }),
py_server(kj::mv(_py_server)), kj_loop(kj::mv(kj_loop)) { }
#endif
~PythonInterfaceDynamicImpl() {
}
kj::Promise<void> call(capnp::InterfaceSchema::Method method,
capnp::CallContext< capnp::DynamicStruct, capnp::DynamicStruct> context);
};
inline void allowCancellation(capnp::CallContext<capnp::DynamicStruct, capnp::DynamicStruct> context) {
#if (CAPNP_VERSION_MAJOR < 1)
context.allowCancellation();
#endif
}
class PyAsyncIoStream: public kj::AsyncIoStream {
public:
kj::Own<PyRefCounter> protocol;
PyAsyncIoStream(kj::Own<PyRefCounter> protocol) : protocol(kj::mv(protocol)) {}
~PyAsyncIoStream();
kj::Promise<size_t> tryRead(void* buffer, size_t minBytes, size_t maxBytes);
kj::Promise<void> write(const void* buffer, size_t size);
kj::Promise<void> write(kj::ArrayPtr<const kj::ArrayPtr<const kj::byte>> pieces);
kj::Promise<void> whenWriteDisconnected();
void shutdownWrite();
};
template <typename T>
inline void rejectDisconnected(kj::PromiseFulfiller<T>& fulfiller, kj::StringPtr message) {
fulfiller.reject(KJ_EXCEPTION(DISCONNECTED, message));
}
inline void rejectVoidDisconnected(kj::PromiseFulfiller<void>& fulfiller, kj::StringPtr message) {
fulfiller.reject(KJ_EXCEPTION(DISCONNECTED, message));
}
inline kj::Exception makeException(kj::StringPtr message) {
return KJ_EXCEPTION(FAILED, message);
}
kj::Promise<void> taskToPromise(kj::Own<PyRefCounter> coroutine, PyObject* callback);
::kj::Promise<kj::Own<PyRefCounter>> tryReadMessage(kj::AsyncIoStream& stream, capnp::ReaderOptions opts);
void init_capnp_api();

5
capnp/helpers/checkCompiler.h
View File

@@ -1,8 +1,3 @@
#ifdef _MSC_VER
#pragma comment(lib, "Ws2_32.lib")
#pragma comment(lib, "advapi32.lib")
#endif
#include "capnp/dynamic.h"
static_assert(CAPNP_VERSION >= 8000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.8 and then re-install this python library");

12
capnp/helpers/deserialize.h
View File

@@ -1,12 +0,0 @@
#pragma once
#include "capnp/dynamic.h"
#include "capnp/schema.capnp.h"
/// @brief Convert the dynamic struct to a Node::Reader
::capnp::schema::Node::Reader toReader(capnp::DynamicStruct::Reader reader)
{
// requires an intermediate step to AnyStruct before going directly to Node::Reader,
// since there exists no direct conversion from DynamicStruct::Reader to Node::Reader.
return reader.as<capnp::AnyStruct>().as<capnp::schema::Node>();
}

31
capnp/helpers/exception.cpp Normal file
View File

@@ -0,0 +1,31 @@
#include "capnp/helpers/exception.h"
#include "capnp/lib/capnp_api.h"
void c_reraise_kj_exception() {
GILAcquire gil;
try {
if (PyErr_Occurred())
; // let the latest Python exn pass through and ignore the current one
else
throw;
}
catch (kj::Exception& exn) {
auto obj = wrap_kj_exception_for_reraise(exn);
if (obj == nullptr) {
return;
}
PyErr_SetObject((PyObject*)obj->ob_type, obj);
Py_DECREF(obj);
}
catch (const std::exception& exn) {
PyErr_SetString(PyExc_RuntimeError, exn.what());
}
catch (...)
{
PyErr_SetString(PyExc_RuntimeError, "Unknown exception");
}
}
void init_capnp_api() {
import_capnp__lib__capnp();
}

18
capnp/helpers/exception.h Normal file
View File

@@ -0,0 +1,18 @@
#pragma once
#include <Python.h>
#include <kj/exception.h>
#include <stdexcept>
class GILAcquire {
public:
GILAcquire() : gstate(PyGILState_Ensure()) {}
~GILAcquire() {
PyGILState_Release(gstate);
}
PyGILState_STATE gstate;
};
void c_reraise_kj_exception();
void init_capnp_api();

29
capnp/helpers/helpers.pxd
View File

@@ -1,34 +1,9 @@
from capnp.includes.capnp_cpp cimport (
Maybe, PyPromise, VoidPromise, RemotePromise,
DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability,
RpcSystem, MessageBuilder, Own, PyRefCounter, Node, DynamicStruct, CallContext
)
from capnp.includes.schema_cpp cimport ByteArray
from capnp.includes.capnp_cpp cimport Maybe, EnumSchema, StructSchema
from non_circular cimport c_reraise_kj_exception as reraise_kj_exception
from cpython.ref cimport PyObject
cdef extern from "capnp/helpers/fixMaybe.h":
EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception
StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception
cdef extern from "capnp/helpers/capabilityHelper.h":
PyPromise then(PyPromise promise, Own[PyRefCounter] func, Own[PyRefCounter] error_func)
PyPromise convert_to_pypromise(RemotePromise)
PyPromise convert_to_pypromise(VoidPromise)
VoidPromise taskToPromise(Own[PyRefCounter] coroutine, PyObject* callback)
void allowCancellation(CallContext context) except +reraise_kj_exception nogil
cdef extern from "capnp/helpers/exception.h":
void init_capnp_api()
cdef extern from "capnp/helpers/rpcHelper.h":
Own[Capability.Client] bootstrapHelper(RpcSystem&) except +reraise_kj_exception
Own[Capability.Client] bootstrapHelperServer(RpcSystem&) except +reraise_kj_exception
cdef extern from "capnp/helpers/serialize.h":
ByteArray messageToPackedBytes(MessageBuilder &, size_t wordCount) except +reraise_kj_exception
cdef extern from "capnp/helpers/deserialize.h":
Node.Reader toReader(DynamicStruct.Reader reader) except +reraise_kj_exception

8
capnp/helpers/non_circular.pxd
View File

@@ -1,8 +1,2 @@
from cpython.ref cimport PyObject
from libcpp cimport bool
cdef extern from "capnp/helpers/capabilityHelper.h":
cdef extern from "capnp/helpers/exception.h":
void c_reraise_kj_exception()
cdef cppclass PyRefCounter:
PyRefCounter(PyObject *)
PyObject * obj

21
capnp/helpers/rpcHelper.h
View File

@@ -1,21 +0,0 @@
#pragma once
#include "capnp/dynamic.h"
#include <capnp/rpc.capnp.h>
#include "capnp/rpc-twoparty.h"
#include "Python.h"
#include "capabilityHelper.h"
kj::Own<capnp::Capability::Client> bootstrapHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
capnp::MallocMessageBuilder hostIdMessage(8);
auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
return kj::heap<capnp::Capability::Client>(client.bootstrap(hostId));
}
kj::Own<capnp::Capability::Client> bootstrapHelperServer(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
capnp::MallocMessageBuilder hostIdMessage(8);
auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
hostId.setSide(capnp::rpc::twoparty::Side::CLIENT);
return kj::heap<capnp::Capability::Client>(client.bootstrap(hostId));
}

14
capnp/helpers/serialize.h
View File

@@ -1,14 +0,0 @@
#pragma once
#include "kj/io.h"
#include "capnp/dynamic.h"
#include "capnp/serialize-packed.h"
kj::Array< ::capnp::byte> messageToPackedBytes(capnp::MessageBuilder & message, size_t wordCount)
{
kj::Array<capnp::byte> result = kj::heapArray<capnp::byte>(wordCount * 8);
kj::ArrayOutputStream out(result.asPtr());
capnp::writePackedMessage(out, message);
return heapArray(out.getArray()); // TODO: make this non-copying somehow
}