diff --git a/README.md b/README.md index a1ee2e3..9faa12d 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ An error like: capnp/capnp.cpp:312:10: fatal error: 'capnp/dynamic.h' file not found #include "capnp/dynamic.h" -Means you haven't installed the Cap'n Proto C++ library. Please follow the directions at the [official installation docs](http://kentonv.github.io/capnproto/install.html) +Means your sytem can't find the installed the Cap'n Proto C++ library. If you haven't installed it yet, please follow the directions at the [official installation docs](http://kentonv.github.io/capnproto/install.html). Otherwise trying `LDFLAGS=-L/usr/local/lib CPPFLAGS=-I/usr/local/include pip install pycapnp` may fix the problem for you. [![Build Status](https://travis-ci.org/jparyani/pycapnp.png?branch=develop)](https://travis-ci.org/jparyani/pycapnp) diff --git a/benchmark/addressbook.capnp.py b/benchmark/addressbook.capnp.py index b21975a..69417bd 100644 --- a/benchmark/addressbook.capnp.py +++ b/benchmark/addressbook.capnp.py @@ -2,12 +2,16 @@ from __future__ import print_function import os import capnp +try: + profile +except: + profile = lambda func: func this_dir = os.path.dirname(__file__) addressbook = capnp.load(os.path.join(this_dir, 'addressbook.capnp')) print = lambda *x: x - +@profile def writeAddressBook(): addressBook = addressbook.AddressBook.new_message() people = addressBook.init('people', 2) @@ -32,20 +36,57 @@ def writeAddressBook(): msg_bytes = addressBook.to_bytes() return msg_bytes - +@profile def printAddressBook(msg_bytes): addressBook = addressbook.AddressBook.from_bytes(msg_bytes) for person in addressBook.people: - print(person.name, ':', person.email) + person.name, person.email for phone in person.phones: - print(phone.type, ':', phone.number) - print() + phone.type, phone.number + +@profile +def writeAddressBookDict(): + addressBook = addressbook.AddressBook.new_message() + people = addressBook.init('people', 2) + + alice = people[0] + alice.id = 123 + alice.name = 'Alice' + alice.email = 'alice@example.com' + alicePhones = alice.init('phones', 1) + alicePhones[0].number = "555-1212" + alicePhones[0].type = 'mobile' + + bob = people[1] + bob.id = 456 + bob.name = 'Bob' + bob.email = 'bob@example.com' + bobPhones = bob.init('phones', 2) + bobPhones[0].number = "555-4567" + bobPhones[0].type = 'home' + bobPhones[1].number = "555-7654" + bobPhones[1].type = 'work' + + msg = addressBook.to_dict() + return msg +@profile +def printAddressBookDict(msg): + addressBook = addressbook.AddressBook.new_message(**msg) + + for person in addressBook.people: + person.name, person.email + for phone in person.phones: + phone.type, phone.number if __name__ == '__main__': + # for i in range(10000): + # msg_bytes = writeAddressBook() + + # printAddressBook(msg_bytes) for i in range(10000): - msg_bytes = writeAddressBook() + msg = writeAddressBookDict() - printAddressBook(msg_bytes) + printAddressBookDict(msg) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 15e6dde..b2d645b 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,7 +35,7 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,4,1) +bundled_version = (0,5,0) libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp diff --git a/capnp/__init__.py b/capnp/__init__.py index 0685f93..2a04e92 100644 --- a/capnp/__init__.py +++ b/capnp/__init__.py @@ -1,7 +1,7 @@ """A python library wrapping the Cap'n Proto C++ library Example Usage:: - + import capnp addressbook = capnp.load('addressbook.capnp') diff --git a/capnp/_gen.py b/capnp/_gen.py new file mode 100644 index 0000000..5cb3eee --- /dev/null +++ b/capnp/_gen.py @@ -0,0 +1,64 @@ +from __future__ import print_function + +import capnp +import schema_capnp +import sys +from jinja2 import Environment, PackageLoader +import os + +def find_type(code, id): + for node in code['nodes']: + if node['id'] == id: + return node + + return None + +def main(): + env = Environment(loader=PackageLoader('capnp', 'templates')) + env.filters['format_name'] = lambda name: name[name.find(':')+1:] + + code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) + code=code.to_dict() + code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] + for node in code['nodes']: + displayName = node['displayName'] + parent, path = displayName.split(':') + node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['module_name'] = path.replace('.', '_') + node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['schema'] = '_{}_Schema'.format(node['module_name']) + is_union = False + for field in node['struct']['fields']: + if field['discriminantValue'] != 65535: + is_union = True + field['c_name'] = field['name'][0].upper() + field['name'][1:] + if 'slot' in field: + field['type'] = field['slot']['type'].keys()[0] + if not isinstance(field['slot']['type'][field['type']], dict): + continue + sub_type = field['slot']['type'][field['type']].get('typeId', None) + if sub_type: + field['sub_type'] = find_type(code, sub_type) + sub_type = field['slot']['type'][field['type']].get('elementType', None) + if sub_type: + field['sub_type'] = sub_type + else: + field['type'] = find_type(code, field['group']['typeId']) + node['is_union'] = is_union + + include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) + module = env.get_template('module.pyx') + + for f in code['requestedFiles']: + filename = f['filename'].replace('.', '_') + '_cython.pyx' + + file_code = dict(code) + file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] + with open(filename, 'w') as out: + out.write(module.render(code=file_code, file=f, include_dir=include_dir)) + + setup = env.get_template('setup.py.tmpl') + with open('setup_capnp.py', 'w') as out: + out.write(setup.render(code=code)) + print('You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.') + print() diff --git a/capnp/c++.capnp b/capnp/c++.capnp index 7f306a7..2bda547 100644 --- a/capnp/c++.capnp +++ b/capnp/c++.capnp @@ -1,27 +1,26 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. +# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors +# Licensed under the MIT License: # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: # -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. @0xbdf87d7bb8304e81; $namespace("capnp::annotations"); annotation namespace(file): Text; +annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text; diff --git a/capnp/helpers/asyncHelper.h b/capnp/helpers/asyncHelper.h index 9d3860a..7d1b7ba 100644 --- a/capnp/helpers/asyncHelper.h +++ b/capnp/helpers/asyncHelper.h @@ -2,6 +2,7 @@ #include "kj/async.h" #include "Python.h" +#include "capabilityHelper.h" class PyEventPort: public kj::EventPort { public: @@ -9,15 +10,18 @@ public: // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere. // Py_INCREF(py_event_port); } - virtual void wait() { + virtual bool wait() { + GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("wait"), NULL); } - virtual void poll() { + virtual bool poll() { + GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("poll"), NULL); } virtual void setRunnable(bool runnable) { + GILAcquire gil; PyObject * arg = Py_False; if (runnable) arg = Py_True; @@ -29,5 +33,25 @@ private: }; void waitNeverDone(kj::WaitScope & scope) { + GILRelease gil; kj::NEVER_DONE.wait(scope); } + +kj::Timer * getTimer(kj::AsyncIoContext * context) { + return &context->lowLevelProvider->getTimer(); +} + +void waitVoidPromise(kj::Promise * promise, kj::WaitScope & scope) { + GILRelease gil; + promise->wait(scope); +} + +PyObject * waitPyPromise(kj::Promise * promise, kj::WaitScope & scope) { + GILRelease gil; + return promise->wait(scope); +} + +capnp::Response< ::capnp::DynamicStruct> * waitRemote(capnp::RemotePromise< ::capnp::DynamicStruct> * promise, kj::WaitScope & scope) { + GILRelease gil; + return new capnp::Response< ::capnp::DynamicStruct>(promise->wait(scope)); +} diff --git a/capnp/helpers/capabilityHelper.h b/capnp/helpers/capabilityHelper.h index 4a33977..72ee96a 100644 --- a/capnp/helpers/capabilityHelper.h +++ b/capnp/helpers/capabilityHelper.h @@ -3,7 +3,6 @@ #include "capnp/dynamic.h" #include #include "Python.h" -#include extern "C" { PyObject * wrap_remote_call(PyObject * func, capnp::Response &); @@ -17,12 +16,38 @@ extern "C" { ::capnp::RemotePromise< ::capnp::DynamicStruct> * extract_remote_promise(PyObject *); } +class GILAcquire { +public: + GILAcquire() : gstate(PyGILState_Ensure()) {} + ~GILAcquire() { + PyGILState_Release(gstate); + } + + PyGILState_STATE gstate; +}; + +class GILRelease { +public: + GILRelease() { + Py_UNBLOCK_THREADS + } + ~GILRelease() { + Py_BLOCK_THREADS + } + + PyThreadState *_save; // The macros above read/write from this variable +}; + ::kj::Promise convert_to_pypromise(capnp::RemotePromise & promise) { return promise.then([](capnp::Response&& response) { return wrap_dynamic_struct_reader(response); } ); } ::kj::Promise convert_to_pypromise(kj::Promise & promise) { - return promise.then([]() { Py_RETURN_NONE;} ); + return promise.then([]() { + GILAcquire gil; + Py_INCREF( Py_None ); + return Py_None; + }); } template @@ -31,6 +56,7 @@ template } void reraise_kj_exception() { + GILAcquire gil; try { if (PyErr_Occurred()) ; // let the latest Python exn pass through and ignore the current one @@ -51,12 +77,13 @@ void reraise_kj_exception() { } void check_py_error() { + GILAcquire gil; PyObject * err = PyErr_Occurred(); if(err) { PyObject * ptype, *pvalue, *ptraceback; PyErr_Fetch(&ptype, &pvalue, &ptraceback); if(ptype == NULL || pvalue == NULL || ptraceback == NULL) - throw kj::Exception(kj::Exception::Nature::OTHER, kj::Exception::Durability::PERMANENT, kj::heapString("capabilityHelper.h"), 44, kj::heapString("Unknown error occurred")); + throw kj::Exception(kj::Exception::Type::FAILED, kj::heapString("capabilityHelper.h"), 44, kj::heapString("Unknown error occurred")); PyObject * info = get_exception_info(ptype, pvalue, ptraceback); @@ -75,11 +102,12 @@ void check_py_error() { Py_DECREF(info); PyErr_Clear(); - throw kj::Exception(kj::Exception::Nature::OTHER, kj::Exception::Durability::PERMANENT, kj::mv(filename), line, kj::mv(description)); + throw kj::Exception(kj::Exception::Type::FAILED, kj::mv(filename), line, kj::mv(description)); } } kj::Promise wrapPyFunc(PyObject * func, PyObject * arg) { + GILAcquire gil; auto arg_promise = extract_promise(arg); if(arg_promise == NULL) { @@ -102,6 +130,7 @@ kj::Promise wrapPyFunc(PyObject * func, PyObject * arg) { } kj::Promise wrapPyFuncNoArg(PyObject * func) { + GILAcquire gil; PyObject * result = PyObject_CallFunctionObjArgs(func, NULL); check_py_error(); @@ -116,6 +145,7 @@ kj::Promise wrapPyFuncNoArg(PyObject * func) { } kj::Promise wrapRemoteCall(PyObject * func, capnp::Response & arg) { + GILAcquire gil; PyObject * ret = wrap_remote_call(func, arg); check_py_error(); @@ -133,7 +163,7 @@ kj::Promise wrapRemoteCall(PyObject * func, capnp::Response wrapRemoteCall(PyObject * func, capnp::Response&& arg) { return wrapRemoteCall(func, arg); } ); else - return promise.then([func](capnp::Response&& arg) { return wrapRemoteCall(func, arg); } + return promise.then([func](capnp::Response&& arg) { return wrapRemoteCall(func, arg); } , [error_func](kj::Exception arg) { return wrapPyFunc(error_func, wrap_kj_exception(arg)); } ); } @@ -149,7 +179,7 @@ kj::Promise wrapRemoteCall(PyObject * func, capnp::Response(schema, server); -} \ No newline at end of file +} diff --git a/capnp/helpers/checkCompiler.h b/capnp/helpers/checkCompiler.h index d494b8a..ed8d427 100644 --- a/capnp/helpers/checkCompiler.h +++ b/capnp/helpers/checkCompiler.h @@ -8,4 +8,4 @@ #include "capnp/dynamic.h" -static_assert(CAPNP_VERSION >= 4000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.4 and then re-install this python library"); +static_assert(CAPNP_VERSION >= 5000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.5 and then re-install this python library"); diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index d665558..d0b1edc 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -1,4 +1,4 @@ -from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet +from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer from capnp.includes.schema_cpp cimport ByteArray @@ -6,11 +6,11 @@ from non_circular cimport reraise_kj_exception from cpython.ref cimport PyObject -cdef extern from "../helpers/fixMaybe.h": +cdef extern from "capnp/helpers/fixMaybe.h": EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception -cdef extern from "../helpers/capabilityHelper.h": +cdef extern from "capnp/helpers/capabilityHelper.h": # PyPromise evalLater(EventLoop &, PyObject * func) # PyPromise there(EventLoop & loop, PyPromise & promise, PyObject * func, PyObject * error_func) PyPromise then(PyPromise & promise, PyObject * func, PyObject * error_func) @@ -24,7 +24,7 @@ cdef extern from "../helpers/capabilityHelper.h": PyPromise convert_to_pypromise(VoidPromise&) VoidPromise convert_to_voidpromise(PyPromise&) -cdef extern from "../helpers/rpcHelper.h": +cdef extern from "capnp/helpers/rpcHelper.h": Capability.Client restoreHelper(RpcSystem&) Capability.Client restoreHelper(RpcSystem&, MessageBuilder&) Capability.Client restoreHelper(RpcSystem&, MessageReader&) @@ -33,8 +33,12 @@ cdef extern from "../helpers/rpcHelper.h": RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&) PyPromise connectServer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr) -cdef extern from "../helpers/serialize.h": +cdef extern from "capnp/helpers/serialize.h": ByteArray messageToPackedBytes(MessageBuilder &, size_t wordCount) -cdef extern from "../helpers/asyncHelper.h": +cdef extern from "capnp/helpers/asyncHelper.h": void waitNeverDone(WaitScope&) + Response * waitRemote(RemotePromise *, WaitScope&) + PyObject * waitPyPromise(PyPromise *, WaitScope&) + void waitVoidPromise(VoidPromise *, WaitScope&) + Timer * getTimer(AsyncIoContext *) except +reraise_kj_exception diff --git a/capnp/helpers/non_circular.pxd b/capnp/helpers/non_circular.pxd index 49e5507..58771dc 100644 --- a/capnp/helpers/non_circular.pxd +++ b/capnp/helpers/non_circular.pxd @@ -1,20 +1,20 @@ from cpython.ref cimport PyObject -cdef extern from "../helpers/capabilityHelper.h": +cdef extern from "capnp/helpers/capabilityHelper.h": cppclass PythonInterfaceDynamicImpl: PythonInterfaceDynamicImpl(PyObject *) -cdef extern from "../helpers/capabilityHelper.h": +cdef extern from "capnp/helpers/capabilityHelper.h": void reraise_kj_exception() cdef cppclass PyRefCounter: PyRefCounter(PyObject *) -cdef extern from "../helpers/rpcHelper.h": +cdef extern from "capnp/helpers/rpcHelper.h": cdef cppclass PyRestorer: PyRestorer(PyObject *) cdef cppclass ErrorHandler: pass -cdef extern from "../helpers/asyncHelper.h": +cdef extern from "capnp/helpers/asyncHelper.h": cdef cppclass PyEventPort: - PyEventPort(PyObject *) \ No newline at end of file + PyEventPort(PyObject *) diff --git a/capnp/helpers/rpcHelper.h b/capnp/helpers/rpcHelper.h index ca16353..a7d54bd 100644 --- a/capnp/helpers/rpcHelper.h +++ b/capnp/helpers/rpcHelper.h @@ -20,8 +20,9 @@ public: // ~PyRestorer() { // Py_DECREF(py_restorer); // } - + capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override { + GILAcquire gil; capnp::Capability::Client * ret = call_py_restorer(py_restorer, objectId); check_py_error(); capnp::Capability::Client stack_ret(*ret); @@ -63,11 +64,13 @@ capnp::Capability::Client restoreHelper(capnp::RpcSystem& client) { - capnp::MallocMessageBuilder message; - capnp::rpc::SturdyRef::Builder ref = message.getRoot(); - auto hostId = ref.getHostId().initAs(); + capnp::MallocMessageBuilder hostIdMessage(8); + auto hostId = hostIdMessage.initRoot(); hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, ref.getObjectId()); + + capnp::MallocMessageBuilder blankMessage(8); + auto objectId = blankMessage.getRoot(); + return client.restore(hostId, objectId); } template makeRpcClientWithRestorer( capnp::VatNetwork& network, PyRestorer& restorer) { using namespace capnp; - return RpcSystem(network, - kj::Maybe&>(restorer)); + return RpcSystem(network, restorer); } struct ServerContext { @@ -113,17 +115,17 @@ void acceptLoop(kj::TaskSet & tasks, PyRestorer & restorer, kj::Own connectServer(kj::TaskSet & tasks, PyRestorer & restorer, kj::AsyncIoContext * context, kj::StringPtr bindAddress) { - auto paf = kj::newPromiseAndFulfiller(); + auto paf = kj::newPromiseAndFulfiller(); auto portPromise = paf.promise.fork(); tasks.add(context->provider->getNetwork().parseAddress(bindAddress) .then(kj::mvCapture(paf.fulfiller, - [&](kj::Own>&& portFulfiller, + [&](kj::Own>&& portFulfiller, kj::Own&& addr) { auto listener = addr->listen(); portFulfiller->fulfill(listener->getPort()); acceptLoop(tasks, restorer, kj::mv(listener)); }))); - return portPromise.addBranch().then([&](uint32_t port) { return PyLong_FromUnsignedLong(port); }); + return portPromise.addBranch().then([&](unsigned int port) { return PyLong_FromUnsignedLong(port); }); } diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index 0c05b8a..4e8dd8f 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -1,7 +1,7 @@ # schema.capnp.cpp.pyx # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 -cdef extern from "../helpers/checkCompiler.h": +cdef extern from "capnp/helpers/checkCompiler.h": pass from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader @@ -39,8 +39,7 @@ cdef extern from "kj/exception.h" namespace " ::kj": Exception(Exception) char* getFile() int getLine() - int getNature() - int getDurability() + int getType() StringPtr getDescription() cdef extern from "kj/memory.h" namespace " ::kj": @@ -51,7 +50,7 @@ cdef extern from "kj/memory.h" namespace " ::kj": Own[PyRefCounter] makePyRefCounter" ::kj::heap< PyRefCounter >"(PyObject *) cdef extern from "kj/async.h" namespace " ::kj": - cdef cppclass Promise[T]: + cdef cppclass Promise[T] nogil: Promise() Promise(Promise) Promise(T) @@ -100,6 +99,16 @@ cdef extern from "kj/array.h" namespace " ::kj": ctypedef Promise[PyArray] PyPromiseArray +cdef extern from "kj/time.h" namespace " ::kj": + cdef cppclass Duration: + Duration(int64_t) + # cdef cppclass TimePoint: + # TimePoint(Duration) + cdef cppclass Timer: + # int64_t now() + # VoidPromise atTime(TimePoint time) + VoidPromise afterDelay(Duration delay) + cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass AsyncIoStream: pass @@ -107,6 +116,7 @@ cdef extern from "kj/async-io.h" namespace " ::kj": # Own[AsyncInputStream] wrapInputFd(int) # Own[AsyncOutputStream] wrapOutputFd(int) Own[AsyncIoStream] wrapSocketFd(int) + Timer& getTimer() except +reraise_kj_exception cdef cppclass AsyncIoProvider: pass cdef cppclass WaitScope: @@ -122,7 +132,42 @@ cdef extern from "kj/async-io.h" namespace " ::kj": AsyncIoContext setupAsyncIo() +cdef extern from "capnp/schema.capnp.h" namespace " ::capnp": + enum TypeWhich" ::capnp::schema::Type::Which": + TypeWhichVOID " ::capnp::schema::Type::Which::VOID" + TypeWhichBOOL " ::capnp::schema::Type::Which::BOOL" + TypeWhichINT8 " ::capnp::schema::Type::Which::INT8" + TypeWhichINT16 " ::capnp::schema::Type::Which::INT16" + TypeWhichINT32 " ::capnp::schema::Type::Which::INT32" + TypeWhichINT64 " ::capnp::schema::Type::Which::INT64" + TypeWhichUINT8 " ::capnp::schema::Type::Which::UINT8" + TypeWhichUINT16 " ::capnp::schema::Type::Which::UINT16" + TypeWhichUINT32 " ::capnp::schema::Type::Which::UINT32" + TypeWhichUINT64 " ::capnp::schema::Type::Which::UINT64" + TypeWhichFLOAT32 " ::capnp::schema::Type::Which::FLOAT32" + TypeWhichFLOAT64 " ::capnp::schema::Type::Which::FLOAT64" + TypeWhichTEXT " ::capnp::schema::Type::Which::TEXT" + TypeWhichDATA " ::capnp::schema::Type::Which::DATA" + TypeWhichLIST " ::capnp::schema::Type::Which::LIST" + TypeWhichENUM " ::capnp::schema::Type::Which::ENUM" + TypeWhichSTRUCT " ::capnp::schema::Type::Which::STRUCT" + TypeWhichINTERFACE " ::capnp::schema::Type::Which::INTERFACE" + TypeWhichANY_POINTER " ::capnp::schema::Type::Which::ANY_POINTER" + cdef extern from "capnp/schema.h" namespace " ::capnp": + cdef cppclass SchemaType" ::capnp::Type": + SchemaType() + SchemaType(TypeWhich) + cbool isList() + cbool isEnum() + cbool isStruct() + cbool isInterface() + + StructSchema asStruct() except +reraise_kj_exception + EnumSchema asEnum() except +reraise_kj_exception + InterfaceSchema asInterface() except +reraise_kj_exception + ListSchema asList() except +reraise_kj_exception + cdef cppclass Schema: Node.Reader getProto() except +reraise_kj_exception StructSchema asStruct() except +reraise_kj_exception @@ -132,11 +177,17 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": InterfaceSchema asInterface() except +reraise_kj_exception cdef cppclass InterfaceSchema(Schema): + cppclass SuperclassList: + uint size() + InterfaceSchema operator[](uint index) + cppclass Method: InterfaceNode.Method.Reader getProto() InterfaceSchema getContainingInterface() uint16_t getOrdinal() uint getIndex() + StructSchema getParamType() + StructSchema getResultType() cppclass MethodList: uint size() @@ -146,6 +197,7 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": Maybe[Method] findMethodByName(StringPtr name) Method getMethodByName(StringPtr name) bint extends(InterfaceSchema other) + SuperclassList getSuperclasses() # kj::Maybe findSuperclass(uint64_t typeId) const; cdef cppclass StructSchema(Schema): @@ -153,6 +205,7 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": StructNode.Member.Reader getProto() StructSchema getContainingStruct() uint getIndex() + SchemaType getType() cppclass FieldList: uint size() @@ -184,6 +237,15 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": Enumerant getEnumerantByName(char * name) Node.Reader getProto() + cdef cppclass ListSchema: + SchemaType getElementType() + + ListSchema listSchemaOfStruct" ::capnp::ListSchema::of"(StructSchema) + ListSchema listSchemaOfEnum" ::capnp::ListSchema::of"(EnumSchema) + ListSchema listSchemaOfInterface" ::capnp::ListSchema::of"(InterfaceSchema) + ListSchema listSchemaOfList" ::capnp::ListSchema::of"(ListSchema) + ListSchema listSchemaOfType" ::capnp::ListSchema::of"(SchemaType) + cdef cppclass ConstSchema: pass @@ -214,8 +276,11 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": cdef cppclass DynamicStruct: cppclass Reader: DynamicValueForward.Reader get(char *) except +reraise_kj_exception + DynamicValueForward.Reader getByField"get"(StructSchema.Field) except +reraise_kj_exception bint has(char *) except +reraise_kj_exception + bint hasByField"has"(StructSchema.Field) except +reraise_kj_exception StructSchema getSchema() + uint64_t getId"getSchema().getProto().getId"() Maybe[StructSchema.Field] which() MessageSize totalSize() cppclass Pipeline: @@ -228,11 +293,17 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicStruct_Builder() DynamicStruct_Builder(DynamicStruct_Builder &) DynamicValueForward.Builder get(char *) except +reraise_kj_exception + DynamicValueForward.Builder getByField"get"(StructSchema.Field) except +reraise_kj_exception bint has(char *) except +reraise_kj_exception + bint hasByField"has"(StructSchema.Field) except +reraise_kj_exception void set(char *, DynamicValueForward.Reader) except +reraise_kj_exception + void setByField"set"(StructSchema.Field, DynamicValueForward.Reader) except +reraise_kj_exception DynamicValueForward.Builder init(char *, uint size) except +reraise_kj_exception DynamicValueForward.Builder init(char *) except +reraise_kj_exception + DynamicValueForward.Builder initByField"init"(StructSchema.Field, uint size) except +reraise_kj_exception + DynamicValueForward.Builder initByField"init"(StructSchema.Field) except +reraise_kj_exception StructSchema getSchema() + uint64_t getId"getSchema().getProto().getId"() Maybe[StructSchema.Field] which() void adopt(char *, DynamicOrphan) except +reraise_kj_exception DynamicOrphan disown(char *) @@ -263,9 +334,12 @@ cdef extern from "capnp/capability.h" namespace " ::capnp": cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp": cdef cppclass RpcSystem" ::capnp::RpcSystem": RpcSystem(RpcSystem&&) - enum Side" ::capnp::rpc::twoparty::Side": - CLIENT" ::capnp::rpc::twoparty::Side::CLIENT" - SERVER" ::capnp::rpc::twoparty::Side::SERVER" + + cdef cppclass Side" ::capnp::rpc::twoparty::Side": + pass + cdef Side CLIENT" ::capnp::rpc::twoparty::Side::CLIENT" + cdef Side SERVER" ::capnp::rpc::twoparty::Side::SERVER" + cdef cppclass TwoPartyVatNetwork: TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side) VoidPromise onDisconnect() @@ -286,30 +360,11 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": Maybe[StructSchema.Field] which() RemotePromise send() -cdef extern from "capnp/any.h" namespace " ::capnp": - cdef cppclass AnyPointer: - cppclass Reader: - DynamicStruct.Reader getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) - StringPtr getAsText"getAs< ::capnp::Text>"() - cppclass Builder: - Builder(Builder) - DynamicStruct_Builder getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) - StringPtr getAsText"getAs< ::capnp::Text>"() - void setAsStruct"setAs< ::capnp::DynamicStruct>"(DynamicStruct.Reader&) except +reraise_kj_exception - void setAsText"setAs< ::capnp::Text>"(char*) except +reraise_kj_exception - cdef extern from "capnp/dynamic.h" namespace " ::capnp": cdef cppclass DynamicEnum: uint16_t getRaw() Maybe[EnumSchema.Enumerant] getEnumerant() - cdef cppclass DynamicObject: - cppclass Reader: - DynamicStruct.Reader as(StructSchema schema) - cppclass Builder: - DynamicObject.Reader asReader() - # DynamicList::Reader as(ListSchema schema) const; - cdef cppclass DynamicList: cppclass Reader: DynamicValueForward.Reader operator[](uint) except +reraise_kj_exception @@ -325,6 +380,28 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": DynamicOrphan disown(uint) StructSchema getStructElementType'getSchema().getStructElementType'() +cdef extern from "capnp/any.h" namespace " ::capnp": + cdef cppclass AnyPointer: + cppclass Reader: + DynamicStruct.Reader getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) except +reraise_kj_exception + DynamicCapability.Client getAsCapability"getAs< ::capnp::DynamicCapability>"(InterfaceSchema) except +reraise_kj_exception + DynamicList.Reader getAsList"getAs< ::capnp::DynamicList>"(ListSchema) except +reraise_kj_exception + StringPtr getAsText"getAs< ::capnp::Text>"() except +reraise_kj_exception + cppclass Builder: + Builder(Builder) + DynamicStruct_Builder getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) except +reraise_kj_exception + DynamicCapability.Client getAsCapability"getAs< ::capnp::DynamicCapability>"(InterfaceSchema) except +reraise_kj_exception + DynamicList.Builder getAsList"getAs< ::capnp::DynamicList>"(ListSchema) except +reraise_kj_exception + StringPtr getAsText"getAs< ::capnp::Text>"() except +reraise_kj_exception + void setAsStruct"setAs< ::capnp::DynamicStruct>"(DynamicStruct.Reader&) except +reraise_kj_exception + void setAsText"setAs< ::capnp::Text>"(char*) except +reraise_kj_exception + AnyPointer.Reader asReader() except +reraise_kj_exception + void set(AnyPointer.Reader) except +reraise_kj_exception + DynamicStruct_Builder initAsStruct"initAs< ::capnp::DynamicStruct>"(StructSchema) except +reraise_kj_exception + DynamicList.Builder initAsList"initAs< ::capnp::DynamicList>"(ListSchema, uint) except +reraise_kj_exception + + +cdef extern from "capnp/dynamic.h" namespace " ::capnp": cdef cppclass DynamicValue: cppclass Reader: Reader() diff --git a/capnp/includes/types.pxd b/capnp/includes/types.pxd index 10567ce..c6dd3ba 100644 --- a/capnp/includes/types.pxd +++ b/capnp/includes/types.pxd @@ -1,5 +1,4 @@ from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF -from cpython.exc cimport PyErr_Clear from libc.stdint cimport * ctypedef unsigned int uint ctypedef uint8_t byte diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 05dd1e3..1f5ddea 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -1,19 +1,123 @@ from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp -from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder +from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode from capnp.includes.types cimport * from capnp.helpers.non_circular cimport reraise_kj_exception from capnp.helpers cimport helpers + +cdef class _StructSchemaField: + cdef C_StructSchema.Field thisptr + cdef object _parent + cdef _init(self, C_StructSchema.Field other, parent=?) + + +cdef class _DynamicOrphan: + cdef C_DynamicOrphan thisptr + cdef public object _parent + + cdef _init(self, C_DynamicOrphan other, object parent) + + cdef C_DynamicOrphan move(self) + cpdef get(self) + + cdef class _DynamicStructReader: cdef C_DynamicStruct.Reader thisptr cdef public object _parent cdef public bint is_root cdef object _obj_to_pin + cdef object _schema - cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=?) + cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=?, bint tryRegistry=?) - cpdef _which(self) + cpdef _get(self, field) + cpdef _has(self, field) + cpdef _DynamicEnumField _which(self) + cpdef _which_str(self) + cpdef _get_by_field(self, _StructSchemaField field) + cpdef _has_by_field(self, _StructSchemaField field) cpdef as_builder(self, num_first_segment_words=?) + + +cdef class _DynamicStructBuilder: + cdef DynamicStruct_Builder thisptr + cdef public object _parent + cdef public bint is_root + cdef bint _is_written + cdef object _schema + + cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?) + + cdef _check_write(self) + cpdef to_bytes(_DynamicStructBuilder self) + cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) + cpdef to_bytes_packed(_DynamicStructBuilder self) + + cpdef _get(self, field) + cpdef _set(self, field, value) + cpdef _has(self, field) + cpdef init(self, field, size=?) + cpdef _get_by_field(self, _StructSchemaField field) + cpdef _set_by_field(self, _StructSchemaField field, value) + cpdef _has_by_field(self, _StructSchemaField field) + cpdef _init_by_field(self, _StructSchemaField field, size=?) + cpdef init_resizable_list(self, field) + cpdef _DynamicEnumField _which(self) + cpdef _which_str(self) + cpdef adopt(self, field, _DynamicOrphan orphan) + cpdef disown(self, field) + + cpdef as_reader(self) + cpdef copy(self, num_first_segment_words=?) + +cdef class _DynamicEnumField: + cdef object thisptr + + cdef _init(self, proto) + cpdef _str(self) + +cdef class _Schema: + cdef C_Schema thisptr + + cdef _init(self, C_Schema other) + + cpdef as_const_value(self) + cpdef as_struct(self) + cpdef as_interface(self) + cpdef as_enum(self) + cpdef get_dependency(self, id) + cpdef get_proto(self) + +cdef class _InterfaceSchema: + cdef C_InterfaceSchema thisptr + cdef object __method_names, __method_names_inherited, __methods, __methods_inherited + cdef _init(self, C_InterfaceSchema other) + cpdef get_dependency(self, id) + +cdef class _DynamicEnum: + cdef capnp.DynamicEnum thisptr + cdef public object _parent + + cdef _init(self, capnp.DynamicEnum other, object parent) + cpdef _as_str(self) + +cdef class _DynamicListBuilder: + cdef C_DynamicList.Builder thisptr + cdef public object _parent + cdef _init(self, C_DynamicList.Builder other, object parent) + + cpdef _get(self, int64_t index) + cpdef _set(self, index, value) + + cpdef adopt(self, index, _DynamicOrphan orphan) + cpdef disown(self, index) + +cdef to_python_reader(C_DynamicValue.Reader self, object parent) +cdef to_python_builder(C_DynamicValue.Builder self, object parent) +cdef _to_dict(msg, bint verbose, bint ordered) +cdef _from_list(_DynamicListBuilder msg, list d) +cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent) +cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 28dcb9d..2f76d3f 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2,6 +2,7 @@ # distutils: language = c++ # distutils: extra_compile_args = --std=c++11 # distutils: libraries = capnpc capnp-rpc capnp kj-async kj +# distutils: include_dirs = . # cython: c_string_type = str # cython: c_string_encoding = default # cython: embedsignature = True @@ -13,6 +14,7 @@ from capnp.helpers.helpers cimport makeRpcClientWithRestorer from libc.stdlib cimport malloc, free from libc.string cimport memcpy from cython.operator cimport dereference as deref +from cpython.exc cimport PyErr_Clear from types import ModuleType as _ModuleType import os as _os @@ -26,17 +28,26 @@ from operator import attrgetter as _attrgetter import threading as _threading import socket as _socket import random as _random +import collections as _collections _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR _CAPNP_VERSION_MICRO = capnp.CAPNP_VERSION_MICRO _CAPNP_VERSION = capnp.CAPNP_VERSION +cdef dict _type_registry = {} + +def register_type(id, klass): + _type_registry[id] = klass + +def deregister_all_types(): + _type_registry = {} + # By making it public, we'll be able to call it from capabilityHelper.h -cdef public object wrap_dynamic_struct_reader(Response & r): +cdef public object wrap_dynamic_struct_reader(Response & r) with gil: return _Response()._init_childptr(new Response(moveResponse(r)), None) -cdef public PyObject * wrap_remote_call(PyObject * func, Response & r) except *: +cdef public PyObject * wrap_remote_call(PyObject * func, Response & r) except * with gil: response = _Response()._init_childptr(new Response(moveResponse(r)), None) func_obj = func @@ -47,7 +58,7 @@ cdef public PyObject * wrap_remote_call(PyObject * func, Response & r) except *: cdef _find_field_order(struct_node): return [f.name for f in sorted(struct_node.fields, key=_attrgetter('codeOrder'))] -cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_name, CallContext & _context) except *: +cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_name, CallContext & _context) except * with gil: server = _server method_name = _method_name @@ -94,7 +105,7 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_ ret = (ret,) names = _find_field_order(context.results.schema.node.struct) if len(ret) > len(names): - raise ValueError('Too many values returned from `%s`. Expected %d and got %d' % (method_name, len(names), len(ret))) + raise KjException('Too many values returned from `%s`. Expected %d and got %d' % (method_name, len(names), len(ret))) results = context.results for arg_name, arg_val in zip(names, ret): @@ -102,7 +113,7 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_ return NULL -cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except *: +cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except * with gil: restorer = _restorer reader = _DynamicObjectReader()._init(_reader, None) @@ -113,10 +124,10 @@ cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_Dynam return new C_Capability.Client(helpers.server_to_client(schema.thisptr, server)) -cdef public convert_array_pyobject(PyArray & arr): +cdef public convert_array_pyobject(PyArray & arr) with gil: return [arr[i] for i in range(arr.size())] -cdef public PyPromise * extract_promise(object obj): +cdef public PyPromise * extract_promise(object obj) with gil: if type(obj) is Promise: promise = obj @@ -127,7 +138,7 @@ cdef public PyPromise * extract_promise(object obj): return NULL -cdef public RemotePromise * extract_remote_promise(object obj): +cdef public RemotePromise * extract_remote_promise(object obj) with gil: if type(obj) is _RemotePromise: promise = <_RemotePromise>obj promise.is_consumed = True @@ -150,16 +161,12 @@ def _make_enum(enum_name, *sequential, **named): enums['reverse_mapping'] = reverse return type(enum_name, (), enums) -_Nature = _make_enum('_Nature', - PRECONDITION = 0, - LOCAL_BUG = 1, - OS_ERROR = 2, - NETWORK_FAILURE = 3, +_Type = _make_enum('_Type', + FAILED = 0, + OVERLOADED = 1, + DISCONNECTED = 2, + UNIMPLEMENTED = 3, OTHER = 4) -_Durability = _make_enum('_Durability', - PERMANENT = 0, - TEMPORARY = 1, - OVERLOADED = 2) cdef class _KjExceptionWrapper: cdef capnp.Exception * thisptr @@ -177,14 +184,10 @@ cdef class _KjExceptionWrapper: property line: def __get__(self): return self.thisptr.getLine() - property nature: + property type: def __get__(self): - cdef int temp = self.thisptr.getNature() - return _Nature.reverse_mapping[temp] - property durability: - def __get__(self): - cdef int temp = self.thisptr.getDurability() - return _Durability.reverse_mapping[temp] + cdef int temp = self.thisptr.getType() + return _Type.reverse_mapping[temp] property description: def __get__(self): return self.thisptr.getDescription().cStr() @@ -195,10 +198,9 @@ cdef class _KjExceptionWrapper: # Extension classes can't inherit from Exception, so we're going to proxy wrap kj::Exception, and forward all calls to it from this Python class class KjException(Exception): - '''KjException is a wrapper of the internal C++ exception type. There are 2 enums, `Nature` and `Durability`, listed below, and a bunch of fields''' + '''KjException is a wrapper of the internal C++ exception type. There is an enum named `Type` listed below, and a bunch of fields''' - Nature = _make_enum('Nature', **{x : x for x in _Nature.reverse_mapping.values()}) - Durability = _make_enum('Durability', **{x : x for x in _Durability.reverse_mapping.values()}) + Type = _make_enum('Type', **{x : x for x in _Type.reverse_mapping.values()}) def __init__(self, message=None, nature=None, durability=None, wrapper=None): if wrapper is not None: @@ -216,17 +218,11 @@ class KjException(Exception): def line(self): return self.wrapper.line @property - def nature(self): + def type(self): if self.wrapper is not None: - return self.wrapper.nature + return self.wrapper.type else: - return self.nature - @property - def durability(self): - if self.wrapper is not None: - return self.wrapper.durability - else: - return self.durability + return self.type @property def description(self): if self.wrapper is not None: @@ -237,42 +233,32 @@ class KjException(Exception): def __str__(self): return self.message -cdef public object wrap_kj_exception(capnp.Exception & exception): + def _to_python(self): + message = self.message + if self.wrapper.type == 'FAILED': + if 'has no such' in self.message: + return AttributeError(message) + return self + +cdef public object wrap_kj_exception(capnp.Exception & exception) with gil: PyErr_Clear() wrapper = _KjExceptionWrapper()._init(exception) ret = KjException(wrapper=wrapper) return ret -cdef public object wrap_kj_exception_for_reraise(capnp.Exception & exception): +cdef public object wrap_kj_exception_for_reraise(capnp.Exception & exception) with gil: wrapper = _KjExceptionWrapper()._init(exception) - wrapper_msg = str(wrapper) - - nature = wrapper.nature - - if wrapper.nature == 'PRECONDITION': - if 'has no such member' in wrapper_msg: - return AttributeError(wrapper_msg) - else: - return ValueError(wrapper_msg) - # elif wrapper.nature == 'LOCAL_BUG': - # return ValueError(str(wrapper)) - if wrapper.nature == 'OS_ERROR': - return OSError(wrapper_msg) - if wrapper.nature == 'NETWORK_FAILURE': - return IOError(wrapper_msg) - ret = KjException(wrapper=wrapper) return ret -cdef public object get_exception_info(object exc_type, object exc_obj, object exc_tb): +cdef public object get_exception_info(object exc_type, object exc_obj, object exc_tb) with gil: try: return (exc_tb.tb_frame.f_code.co_filename.encode(), exc_tb.tb_lineno, (repr(exc_type) + ':' + str(exc_obj)).encode()) except: return (b'', 0, b"Couldn't determine python exception") - ctypedef fused _DynamicStructReaderOrBuilder: _DynamicStructReader _DynamicStructBuilder @@ -388,12 +374,15 @@ cdef class _DynamicListReader: self._parent = parent return self - def __getitem__(self, index): - size = self.thisptr.size() + cpdef _get(self, int64_t index): + return to_python_reader(self.thisptr[index], self._parent) + + def __getitem__(self, int64_t index): + cdef uint size = self.thisptr.size() if index >= size: raise IndexError('Out of bounds') index = index % size - return to_python_reader(self.thisptr[index], self._parent) + return self._get(index) def __len__(self): return self.thisptr.size() @@ -449,6 +438,9 @@ cdef class _DynamicResizableListBuilder: self._list.append((orphan, orphan_val)) return orphan_val + cpdef _get(self, index): + return self._list[index][1] + def __getitem__(self, index): return self._list[index][1] @@ -487,24 +479,22 @@ cdef class _DynamicListBuilder: for phone in phones: print phone.number """ - cdef C_DynamicList.Builder thisptr - cdef public object _parent cdef _init(self, C_DynamicList.Builder other, object parent): self.thisptr = other self._parent = parent return self - cdef _get(self, index): + cpdef _get(self, int64_t index): return to_python_builder(self.thisptr[index], self._parent) - def __getitem__(self, index): - size = self.thisptr.size() + def __getitem__(self, int64_t index): + cdef uint size = self.thisptr.size() if index >= size: raise IndexError('Out of bounds') index = index % size return self._get(index) - cdef _set(self, index, value): + cpdef _set(self, index, value): _setDynamicField(self.thisptr, index, value, self._parent) def __setitem__(self, index, value): @@ -574,9 +564,9 @@ cdef class _List_NestedNode_Reader: # # elif type == capnp.TYPE_STRUCT: # # return _DynamicStructReader()._init(self.asStruct(), parent) # elif type == capnp.TYPE_UNKNOWN: -# raise ValueError("Cannot convert type to Python. Type is unknown by capnproto library") +# raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") # else: -# raise ValueError("Cannot convert type to Python. Type is unhandled by capnproto library") +# raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") cdef to_python_reader(C_DynamicValue.Reader self, object parent): cdef int type = self.getType() @@ -607,9 +597,9 @@ cdef to_python_reader(C_DynamicValue.Reader self, object parent): elif type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init(self.asCapability(), parent) elif type == capnp.TYPE_UNKNOWN: - raise ValueError("Cannot convert type to Python. Type is unknown by capnproto library") + raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: - raise ValueError("Cannot convert type to Python. Type is unhandled by capnproto library") + raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") cdef to_python_builder(C_DynamicValue.Builder self, object parent): cdef int type = self.getType() @@ -640,9 +630,9 @@ cdef to_python_builder(C_DynamicValue.Builder self, object parent): elif type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init(self.asCapability(), parent) elif type == capnp.TYPE_UNKNOWN: - raise ValueError("Cannot convert type to Python. Type is unknown by capnproto library") + raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: - raise ValueError("Cannot convert type to Python. Type is unhandled by capnproto library") + raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") cdef C_DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value): return C_DynamicValue.Reader(value.thisptr.asReader()) @@ -671,6 +661,17 @@ cdef _setBaseString(_DynamicSetterClasses thisptr, field, value): cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.set(field, temp) +cdef _setBytesField(DynamicStruct_Builder thisptr, _StructSchemaField field, value): + cdef capnp.StringPtr temp_string = capnp.StringPtr(value, len(value)) + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) + thisptr.setByField(field.thisptr, temp) + +cdef _setBaseStringField(DynamicStruct_Builder thisptr, _StructSchemaField field, value): + encoded_value = value.encode() + cdef capnp.StringPtr temp_string = capnp.StringPtr(encoded_value, len(encoded_value)) + cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) + thisptr.setByField(field.thisptr, temp) + cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): cdef C_DynamicValue.Reader temp value_type = type(value) @@ -697,10 +698,10 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): elif value_type is dict: if _DynamicSetterClasses is DynamicStruct_Builder: builder = to_python_builder(thisptr.get(field), parent) - _from_dict(builder, value) + builder.from_dict(value) else: builder = to_python_builder(thisptr[field], parent) - _from_dict(builder, value) + builder.from_dict(value) elif value is None: temp = C_DynamicValue.Reader(VOID) thisptr.set(field, temp) @@ -715,54 +716,159 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) else: - raise ValueError("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) + raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) -cdef _to_dict(msg, bint verbose): +cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent): + cdef C_DynamicValue.Reader temp + value_type = type(value) + + if value_type is int or value_type is long: + if value < 0: + temp = C_DynamicValue.Reader(value) + else: + temp = C_DynamicValue.Reader(value) + thisptr.setByField(field.thisptr, temp) + elif value_type is float: + temp = C_DynamicValue.Reader(value) + thisptr.setByField(field.thisptr, temp) + elif value_type is bool: + temp = C_DynamicValue.Reader(value) + thisptr.setByField(field.thisptr, temp) + elif value_type is bytes: + _setBytesField(thisptr, field, value) + elif isinstance(value, basestring): + _setBaseStringField(thisptr, field, value) + elif value_type is list: + builder = to_python_builder(thisptr.init(field.proto.name, len(value)), parent) + _from_list(builder, value) + elif value_type is dict: + builder = to_python_builder(thisptr.getByField(field.thisptr), parent) + builder.from_dict(value) + elif value is None: + temp = C_DynamicValue.Reader(VOID) + thisptr.setByField(field.thisptr, temp) + elif value_type is _DynamicStructBuilder: + thisptr.setByField(field.thisptr, _extract_dynamic_struct_builder(value)) + elif value_type is _DynamicStructReader: + thisptr.setByField(field.thisptr, _extract_dynamic_struct_reader(value)) + elif value_type is _DynamicCapabilityClient: + thisptr.setByField(field.thisptr, _extract_dynamic_client(value)) + elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer): + thisptr.setByField(field.thisptr, _extract_dynamic_server(value)) + elif value_type is _DynamicEnum: + thisptr.setByField(field.thisptr, _extract_dynamic_enum(value)) + else: + raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) + +cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent): + cdef C_DynamicValue.Reader temp + value_type = type(value) + + if value_type is int or value_type is long: + if value < 0: + temp = C_DynamicValue.Reader(value) + else: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is float: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is bool: + temp = C_DynamicValue.Reader(value) + thisptr.set(field, temp) + elif value_type is bytes: + _setBytes(thisptr, field, value) + elif isinstance(value, basestring): + _setBaseString(thisptr, field, value) + elif value_type is list: + builder = to_python_builder(thisptr.init(field, len(value)), parent) + _from_list(builder, value) + elif value_type is dict: + builder = to_python_builder(thisptr.get(field), parent) + builder.from_dict(value) + elif value is None: + temp = C_DynamicValue.Reader(VOID) + thisptr.set(field, temp) + elif value_type is _DynamicStructBuilder: + thisptr.set(field, _extract_dynamic_struct_builder(value)) + elif value_type is _DynamicStructReader: + thisptr.set(field, _extract_dynamic_struct_reader(value)) + elif value_type is _DynamicCapabilityClient: + thisptr.set(field, _extract_dynamic_client(value)) + elif value_type is _DynamicCapabilityServer or isinstance(value, _DynamicCapabilityServer): + thisptr.set(field, _extract_dynamic_server(value)) + elif value_type is _DynamicEnum: + thisptr.set(field, _extract_dynamic_enum(value)) + else: + raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) + +cdef _DynamicListBuilder temp_list_b +cdef _DynamicListReader temp_list_r +cdef _DynamicResizableListBuilder temp_list_rb +cdef _DynamicStructBuilder temp_msg_b +cdef _DynamicStructReader temp_msg_r +cdef _to_dict(msg, bint verbose, bint ordered): msg_type = type(msg) - if msg_type is _DynamicListBuilder or msg_type is _DynamicListReader or msg_type is _DynamicResizableListBuilder: - return [_to_dict(x, verbose) for x in msg] + if msg_type is _DynamicListBuilder: + temp_list_b = msg + return [_to_dict(temp_list_b._get(i), verbose, ordered) for i in range(len(msg))] + elif msg_type is _DynamicListReader: + temp_list_r = msg + return [_to_dict(temp_list_r._get(i), verbose, ordered) for i in range(len(msg))] + elif msg_type is _DynamicResizableListBuilder: + temp_list_rb = msg + return [_to_dict(temp_list_rb._get(i), verbose, ordered) for i in range(len(msg))] - if msg_type is _DynamicStructBuilder or msg_type is _DynamicStructReader: - ret = {} + if msg_type is _DynamicStructBuilder: + temp_msg_b = msg + if ordered: + ret = _collections.OrderedDict() + else: + ret = {} try: - which = msg.which() - ret[which] = _to_dict(getattr(msg, which), verbose) - except ValueError: + which = temp_msg_b.which() + ret[which] = _to_dict(temp_msg_b._get(which), verbose, ordered) + except KjException: pass - for field in msg.schema.non_union_fields: - if verbose or msg._has(field): - ret[field] = _to_dict(getattr(msg, field), verbose) + for field in temp_msg_b.schema.non_union_fields: + if verbose or temp_msg_b._has(field): + ret[field] = _to_dict(temp_msg_b._get(field), verbose, ordered) return ret + elif msg_type is _DynamicStructReader: + temp_msg_r = msg + if ordered: + ret = _collections.OrderedDict() + else: + ret = {} + try: + which = temp_msg_r.which() + ret[which] = _to_dict(temp_msg_r._get(which), verbose, ordered) + except KjException: + pass + + for field in temp_msg_r.schema.non_union_fields: + if verbose or temp_msg_r._has(field): + ret[field] = _to_dict(temp_msg_r._get(field), verbose, ordered) + + return ret + + if isinstance(msg, (_DynamicStructBuilder, _DynamicStructReader)): + return msg.to_dict(verbose, ordered) if msg_type is _DynamicEnum: return str(msg) return msg - -cdef _from_dict(_DynamicStructBuilder msg, dict d): - for key, val in d.iteritems(): - if key != 'which': - try: - msg._set(key, val) - except Exception as e: - if 'expected isSetInUnion(field)' in str(e): - msg.init(key) - msg._set(key, val) - cdef _from_list(_DynamicListBuilder msg, list d): cdef size_t count = 0 - for val in d: - msg._set(count, val) - count += 1 + for i in range(len(d)): + msg._set(i, d[i]) cdef class _DynamicEnum: - cdef capnp.DynamicEnum thisptr - cdef public object _parent - cdef _init(self, capnp.DynamicEnum other, object parent): self.thisptr = other self._parent = parent @@ -802,8 +908,6 @@ cdef class _DynamicEnum: return left >= right cdef class _DynamicEnumField: - cdef object thisptr - cdef _init(self, proto): self.thisptr = proto return self @@ -813,9 +917,12 @@ cdef class _DynamicEnumField: def __get__(self): return self.thisptr.discriminantValue - def __str__(self): + cpdef _str(self): return self.thisptr.name + def __str__(self): + return self._str() + def __repr__(self): return '<%s which-enum>' % str(self) @@ -849,8 +956,11 @@ cdef class _MessageSize: self.word_count = word_count self.cap_count = cap_count -def _struct_reducer(schema_id, data): - return _global_schema_parser.modules_by_id[schema_id].from_bytes(data) +if getattr(_sys, 'subversion', [''])[0] == 'PyPy': + from pickle_helper import _struct_reducer +else: + def _struct_reducer(schema_id, data): + return _global_schema_parser.modules_by_id[schema_id].from_bytes(data) cdef class _DynamicStructReader: """Reads Cap'n Proto structs @@ -861,30 +971,54 @@ cdef class _DynamicStructReader: print person.name # using . syntax print getattr(person, 'field-with-hyphens') # for names that are invalid for python, use getattr """ - cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=False): + cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=False, bint tryRegistry = True): self.thisptr = other self._parent = parent self.is_root = isRoot + self._schema = None + + if tryRegistry and len(_type_registry) > 0: + registered_type = _type_registry.get(self.thisptr.getId(), None) + if registered_type: + return registered_type[0](self) return self - def __getattr__(self, field): + cpdef _get(self, field): return to_python_reader(self.thisptr.get(field), self._parent) - def _has(self, field): + def __getattr__(self, field): + try: + return self._get(field) + except KjException as e: + raise e._to_python() + + cpdef _get_by_field(self, _StructSchemaField field): + return to_python_reader(self.thisptr.getByField(field.thisptr), self._parent) + + cpdef _has(self, field): return self.thisptr.has(field) - cpdef _which(self): + cpdef _has_by_field(self, _StructSchemaField field): + return self.thisptr.hasByField(field.thisptr) + + cpdef _which_str(self): + try: + return helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr() + except: + raise KjException("Attempted to call which on a non-union type") + + cpdef _DynamicEnumField _which(self): """Returns the enum corresponding to the union in this struct :rtype: :class:`_DynamicEnumField` :return: A string/enum corresponding to what field is set in the union - :Raises: :exc:`exceptions.ValueError` if this struct doesn't contain a union + :Raises: :exc:`KjException` if this struct doesn't contain a union """ try: which = _DynamicEnumField()._init(_StructSchemaField()._init(helpers.fixMaybe(self.thisptr.which()), self).proto) except: - raise ValueError("Attempted to call which on a non-union type") + raise KjException("Attempted to call which on a non-union type") return which @@ -894,7 +1028,7 @@ cdef class _DynamicStructReader: :rtype: :class:`_DynamicEnumField` :return: A string/enum corresponding to what field is set in the union - :Raises: :exc:`exceptions.ValueError` if this struct doesn't contain a union + :Raises: :exc:`KjException` if this struct doesn't contain a union """ def __get__(_DynamicStructReader self): return self._which() @@ -902,7 +1036,9 @@ cdef class _DynamicStructReader: property schema: """A property that returns the _StructSchema object matching this reader""" def __get__(self): - return _StructSchema()._init(self.thisptr.getSchema()) + if self._schema is None: + self._schema = _StructSchema()._init(self.thisptr.getSchema()) + return self._schema def __dir__(self): return list(self.schema.fieldnames) @@ -913,8 +1049,8 @@ cdef class _DynamicStructReader: def __repr__(self): return '<%s reader %s>' % (self.schema.node.displayName, strStructReader(self.thisptr).cStr()) - def to_dict(self, verbose=False): - return _to_dict(self, verbose) + def to_dict(self, verbose=False, ordered=False): + return _to_dict(self, verbose, ordered) cpdef as_builder(self, num_first_segment_words=None): """A method for casting this Reader to a Builder @@ -934,9 +1070,10 @@ cdef class _DynamicStructReader: size = self.thisptr.totalSize() return _MessageSize(size.wordCount, size.capCount) - def __reduce__(self): + def __reduce_ex__(self, proto): return _struct_reducer, (self.schema.node.id, self.as_builder().to_bytes()) + cdef class _DynamicStructBuilder: """Builds Cap'n Proto structs @@ -950,20 +1087,22 @@ cdef class _DynamicStructBuilder: setattr(person, 'field-with-hyphens', 'foo') # for names that are invalid for python, use setattr print getattr(person, 'field-with-hyphens') # for names that are invalid for python, use getattr """ - cdef DynamicStruct_Builder thisptr - cdef public object _parent - cdef public bint is_root - cdef bint _is_written - cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot = False): + cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot = False, bint tryRegistry = True): self.thisptr = other self._parent = parent self.is_root = isRoot self._is_written = False + self._schema = None + + if tryRegistry and len(_type_registry) > 0: + registered_type = _type_registry.get(self.thisptr.getId(), None) + if registered_type: + return registered_type[1](self) return self cdef _check_write(self): if not self.is_root: - raise ValueError("You can only call write() on the message's root struct.") + raise KjException("You can only call write() on the message's root struct.") if self._is_written: _warnings.warn("This message has already been written once. Be very careful that you're not setting Text/Struct/List fields more than once, since that will cause memory leaks (both in memory and in the serialized data). You can disable this warning by setting the `_is_written` field of this object to False after every write.") @@ -978,7 +1117,7 @@ cdef class _DynamicStructBuilder: :rtype: void - :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct. + :Raises: :exc:`KjException` if this isn't the message's root struct. """ self._check_write() _write_message_to_fd(file.fileno(), self._parent) @@ -995,7 +1134,7 @@ cdef class _DynamicStructBuilder: :rtype: void - :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct. + :Raises: :exc:`KjException` if this isn't the message's root struct. """ self._check_write() _write_packed_message_to_fd(file.fileno(), self._parent) @@ -1008,7 +1147,7 @@ cdef class _DynamicStructBuilder: :rtype: bytes - :Raises: :exc:`exceptions.ValueError` if this isn't the message's root struct. + :Raises: :exc:`KjException` if this isn't the message's root struct. """ self._check_write() cdef _MessageBuilder builder = self._parent @@ -1041,23 +1180,36 @@ cdef class _DynamicStructBuilder: self._is_written = True return ret - cdef _get(self, field): - cdef C_DynamicValue.Builder value = self.thisptr.get(field) + cpdef _get(self, field): + return to_python_builder(self.thisptr.get(field), self._parent) - return to_python_builder(value, self._parent) + cpdef _get_by_field(self, _StructSchemaField field): + return to_python_builder(self.thisptr.getByField(field.thisptr), self._parent) def __getattr__(self, field): - return self._get(field) + try: + return self._get(field) + except KjException as e: + raise e._to_python() - cdef _set(self, field, value): + cpdef _set(self, field, value): _setDynamicField(self.thisptr, field, value, self._parent) - def __setattr__(self, field, value): - self._set(field, value) + cpdef _set_by_field(self, _StructSchemaField field, value): + _setDynamicFieldWithField(self.thisptr, field, value, self._parent) - def _has(self, field): + def __setattr__(self, field, value): + try: + self._set(field, value) + except KjException as e: + raise e._to_python() + + cpdef _has(self, field): return self.thisptr.has(field) + cpdef _has_by_field(self, _StructSchemaField field): + return self.thisptr.hasByField(field.thisptr) + cpdef init(self, field, size=None): """Method for initializing fields that are of type union/struct/list @@ -1071,13 +1223,33 @@ cdef class _DynamicStructBuilder: :rtype: :class:`_DynamicStructBuilder` or :class:`_DynamicListBuilder` - :Raises: :exc:`exceptions.ValueError` if the field isn't in this struct + :Raises: :exc:`KjException` if the field isn't in this struct """ if size is None: return to_python_builder(self.thisptr.init(field), self._parent) else: return to_python_builder(self.thisptr.init(field, size), self._parent) + cpdef _init_by_field(self, _StructSchemaField field, size=None): + """Method for initializing fields that are of type union/struct/list + + Typically, you don't have to worry about initializing structs/unions, so this method is mainly for lists. + + :type field: str + :param field: The field name to initialize + + :type size: int + :param size: The size of the list to initiialize. This should be None for struct/union initialization. + + :rtype: :class:`_DynamicStructBuilder` or :class:`_DynamicListBuilder` + + :Raises: :exc:`KjException` if the field isn't in this struct + """ + if size is None: + return to_python_builder(self.thisptr.initByField(field.thisptr), self._parent) + else: + return to_python_builder(self.thisptr.initByField(field.thisptr, size), self._parent) + cpdef init_resizable_list(self, field): """Method for initializing fields that are of type list (of structs) @@ -1090,22 +1262,28 @@ cdef class _DynamicStructBuilder: :rtype: :class:`_DynamicResizableListBuilder` - :Raises: :exc:`exceptions.ValueError` if the field isn't in this struct + :Raises: :exc:`KjException` if the field isn't in this struct """ return _DynamicResizableListBuilder(self, field, _StructSchema()._init((self.thisptr.get(field)).asList().getStructElementType())) - cpdef _which(self): + cpdef _which_str(self): + try: + return helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr() + except: + raise KjException("Attempted to call which on a non-union type") + + cpdef _DynamicEnumField _which(self): """Returns the enum corresponding to the union in this struct :rtype: :class:`_DynamicEnumField` :return: A string/enum corresponding to what field is set in the union - :Raises: :exc:`exceptions.ValueError` if this struct doesn't contain a union + :Raises: :exc:`KjException` if this struct doesn't contain a union """ try: which = _DynamicEnumField()._init(_StructSchemaField()._init(helpers.fixMaybe(self.thisptr.which()), self).proto) except: - raise ValueError("Attempted to call which on a non-union type") + raise KjException("Attempted to call which on a non-union type") return which @@ -1115,7 +1293,7 @@ cdef class _DynamicStructBuilder: :rtype: :class:`_DynamicEnumField` :return: A string/enum corresponding to what field is set in the union - :Raises: :exc:`exceptions.ValueError` if this struct doesn't contain a union + :Raises: :exc:`KjException` if this struct doesn't contain a union """ def __get__(_DynamicStructBuilder self): return self._which() @@ -1176,7 +1354,9 @@ cdef class _DynamicStructBuilder: property schema: """A property that returns the _StructSchema object matching this writer""" def __get__(self): - return _StructSchema()._init(self.thisptr.getSchema()) + if self._schema is None: + self._schema = _StructSchema()._init(self.thisptr.getSchema()) + return self._schema def __dir__(self): return list(self.schema.fieldnames) @@ -1187,15 +1367,27 @@ cdef class _DynamicStructBuilder: def __repr__(self): return '<%s builder %s>' % (self.schema.node.displayName, strStructBuilder(self.thisptr).cStr()) - def to_dict(self, verbose=False): - return _to_dict(self, verbose) + def to_dict(self, verbose=False, ordered=False): + return _to_dict(self, verbose, ordered) + + def from_dict(self, dict d): + for key, val in d.iteritems(): + if key != 'which': + try: + self._set(key, val) + except Exception as e: + if 'expected isSetInUnion(field)' in str(e): + self.init(key) + self._set(key, val) + else: + raise property total_size: def __get__(self): size = self.thisptr.totalSize() return _MessageSize(size.wordCount, size.capCount) - def __reduce__(self): + def __reduce_ex__(self, proto): return _struct_reducer, (self.schema.node.id, self.to_bytes()) cdef class _DynamicStructPipeline: @@ -1221,12 +1413,15 @@ cdef class _DynamicStructPipeline: elif type == capnp.TYPE_STRUCT: return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) elif type == capnp.TYPE_UNKNOWN: - raise ValueError("Cannot convert type to Python. Type is unknown by capnproto library") + raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: - raise ValueError("Cannot convert type to Python. Type is unhandled by capnproto library") + raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") def __getattr__(self, field): - return self._get(field) + try: + return self._get(field) + except KjException as e: + raise e._to_python() property schema: """A property that returns the _StructSchema object matching this reader""" @@ -1242,12 +1437,10 @@ cdef class _DynamicStructPipeline: # def __repr__(self): # return '<%s reader %s>' % (self.schema.node.displayName, strStructReader(self.thisptr).cStr()) - def to_dict(self, verbose=False): - return _to_dict(self, verbose) + def to_dict(self, verbose=False, ordered=False): + return _to_dict(self, verbose, ordered) cdef class _DynamicOrphan: - cdef C_DynamicOrphan thisptr - cdef public object _parent cdef _init(self, C_DynamicOrphan other, object parent): self.thisptr = moveOrphan(other) self._parent = parent @@ -1287,6 +1480,24 @@ cdef class _DynamicObjectReader: return _DynamicStructReader()._init(self.thisptr.getAs(s.thisptr), self._parent) + cpdef as_interface(self, schema) except +reraise_kj_exception: + cdef _InterfaceSchema s + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent) + + cpdef as_list(self, schema) except +reraise_kj_exception: + cdef ListSchema s + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + return _DynamicListReader()._init(self.thisptr.getAsList(s.thisptr), self._parent) + cpdef as_text(self) except +reraise_kj_exception: return (self.thisptr.getAsText().cStr())[:] @@ -1311,12 +1522,47 @@ cdef class _DynamicObjectBuilder: return _DynamicStructBuilder()._init(self.thisptr.getAs(s.thisptr), self._parent) + cpdef as_interface(self, schema) except +reraise_kj_exception: + cdef _InterfaceSchema s + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent) + + cpdef as_list(self, schema) except +reraise_kj_exception: + cdef ListSchema s + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + return _DynamicListBuilder()._init(self.thisptr.getAsList(s.thisptr), self._parent) + + cpdef set(self, other): + "Set value of this object with the value of another AnyPointer::Reader. Don't use this for structs" + cdef _DynamicObjectReader reader = other + self.thisptr.set(reader.thisptr) + cpdef set_as_text(self, text): self.thisptr.setAsText(text) + cpdef init_as_list(self, schema, size): + cdef ListSchema s + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + return _DynamicListBuilder()._init(self.thisptr.initAsList(s.thisptr, size), self._parent) + cpdef as_text(self) except +reraise_kj_exception: return (self.thisptr.getAsText().cStr())[:] + cpdef as_reader(self): + return _DynamicObjectReader()._init(self.thisptr.asReader(), self._parent) + cdef class _EventLoop: cdef capnp.AsyncIoContext * thisptr @@ -1339,6 +1585,7 @@ cdef class _EventLoop: cdef _EventLoop C_DEFAULT_EVENT_LOOP = _EventLoop() _C_DEFAULT_EVENT_LOOP_LOCAL = None +_THREAD_LOCAL_EVENT_LOOPS = [] cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): 'Optimization for not having to deal with threadlocal event loops unless we need to' @@ -1352,30 +1599,62 @@ cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): _C_DEFAULT_EVENT_LOOP_LOCAL.loop = _EventLoop() return _C_DEFAULT_EVENT_LOOP_LOCAL.loop - raise RuntimeError("You don't have any EventLoops running. Please make sure to add one") + raise KjException("You don't have any EventLoops running. Please make sure to add one") -# cpdef remove_event_loop(): -# 'Remove the global event loop' -# global C_DEFAULT_EVENT_LOOP -# C_DEFAULT_EVENT_LOOP._remove() -# C_DEFAULT_EVENT_LOOP = None +cdef class _Timer: + cdef capnp.Timer * thisptr -# cpdef create_event_loop(is_thread_local=True): -# '''Create a new global event loop. This will not remove the previous -# EventLoop for you, so make sure to do that first''' -# global C_DEFAULT_EVENT_LOOP -# global _C_DEFAULT_EVENT_LOOP_LOCAL -# if is_thread_local: -# if _C_DEFAULT_EVENT_LOOP_LOCAL is None: -# _C_DEFAULT_EVENT_LOOP_LOCAL = _threading.local() -# _C_DEFAULT_EVENT_LOOP_LOCAL.loop = _EventLoop() -# else: -# C_DEFAULT_EVENT_LOOP = _EventLoop() + cdef _init(self, capnp.Timer * timer): + self.thisptr = timer + return self -# cpdef reset_event_loop(): -# global C_DEFAULT_EVENT_LOOP -# C_DEFAULT_EVENT_LOOP._remove() -# C_DEFAULT_EVENT_LOOP = _EventLoop() + cpdef after_delay(self, time) except +reraise_kj_exception: + return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Duration(time))) + +def getTimer(): + return _Timer()._init(helpers.getTimer(C_DEFAULT_EVENT_LOOP_GETTER().thisptr)) + +cpdef remove_event_loop(ignore_errors=False): + 'Remove the global event loop' + global C_DEFAULT_EVENT_LOOP + global _THREAD_LOCAL_EVENT_LOOPS + global _C_DEFAULT_EVENT_LOOP_LOCAL + + if C_DEFAULT_EVENT_LOOP: + try: + C_DEFAULT_EVENT_LOOP._remove() + except: + if not ignore_errors: + raise + C_DEFAULT_EVENT_LOOP = None + if len(_THREAD_LOCAL_EVENT_LOOPS) > 0: + for loop in _THREAD_LOCAL_EVENT_LOOPS: + try: + loop._remove() + except: + if not ignore_errors: + raise + _THREAD_LOCAL_EVENT_LOOPS = [] + _C_DEFAULT_EVENT_LOOP_LOCAL = None + +cpdef create_event_loop(threaded=True): + '''Create a new global event loop. This will not remove the previous + EventLoop for you, so make sure to do that first''' + global C_DEFAULT_EVENT_LOOP + global _C_DEFAULT_EVENT_LOOP_LOCAL + if threaded: + if _C_DEFAULT_EVENT_LOOP_LOCAL is None: + _C_DEFAULT_EVENT_LOOP_LOCAL = _threading.local() + loop = _EventLoop() + _C_DEFAULT_EVENT_LOOP_LOCAL.loop = loop + _THREAD_LOCAL_EVENT_LOOPS.append(loop) + else: + C_DEFAULT_EVENT_LOOP = _EventLoop() + +cpdef reset_event_loop(): + global C_DEFAULT_EVENT_LOOP + C_DEFAULT_EVENT_LOOP._remove() + C_DEFAULT_EVENT_LOOP = _EventLoop() def wait_forever(): cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() @@ -1425,7 +1704,7 @@ cdef class Promise: else: self.is_consumed = False self._obj = obj - Py_INCREF(obj) + Py_INCREF(obj) # TODO: MEM: fix leak self.thisptr = new PyPromise(obj) self._event_loop = C_DEFAULT_EVENT_LOOP_GETTER() @@ -1441,9 +1720,9 @@ cdef class Promise: cpdef wait(self) except +reraise_kj_exception: if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - ret = self.thisptr.wait(deref(self._event_loop.thisptr).waitScope) + ret = helpers.waitPyPromise(self.thisptr, deref(self._event_loop.thisptr).waitScope) Py_DECREF(ret) self.is_consumed = True @@ -1452,21 +1731,40 @@ cdef class Promise: cpdef then(self, func, error_func=None) except +reraise_kj_exception: if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - self.is_consumed = True + argspec = None + try: + argspec = _inspect.getargspec(func) + except: + pass + if argspec: + args_length = len(argspec.args) if argspec.args else 0 + defaults_length = len(argspec.defaults) if argspec.defaults else 0 + if args_length - defaults_length != 1: + raise KjException('Function passed to `then` call must take exactly one argument') - return Promise()._init(helpers.then(deref(self.thisptr), func, error_func).attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), self) + cdef Promise new_promise = Promise()._init(helpers.then(deref(self.thisptr), func, error_func), self) + return Promise()._init(new_promise.thisptr.attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), new_promise) def attach(self, *args): if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') ret = Promise()._init(self.thisptr.attach(capnp.makePyRefCounter(args)), self) self.is_consumed = True return ret + cpdef cancel(self, numParents=1) except +reraise_kj_exception: + if numParents > 0 and hasattr(self._parent, 'cancel'): + self._parent.cancel(numParents - 1) + + self.is_consumed = True + del self.thisptr + self.thisptr = NULL + + cdef class _VoidPromise: cdef VoidPromise * thisptr cdef public bint is_consumed @@ -1488,31 +1786,52 @@ cdef class _VoidPromise: cpdef wait(self) except +reraise_kj_exception: if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') + + helpers.waitVoidPromise(self.thisptr, deref(self._event_loop.thisptr).waitScope) - self.thisptr.wait(deref(self._event_loop.thisptr).waitScope) self.is_consumed = True cpdef then(self, func, error_func=None) except +reraise_kj_exception: if self.is_consumed: - raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - return Promise()._init(helpers.then(deref(self.thisptr), func, error_func).attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), self) + argspec = None + try: + argspec = _inspect.getargspec(func) + except: + pass + if argspec: + args_length = len(argspec.args) if argspec.args else 0 + defaults_length = len(argspec.defaults) if argspec.defaults else 0 + if args_length - defaults_length != 0: + raise KjException('Function passed to `then` call must take no arguments') + + cdef Promise new_promise = Promise()._init(helpers.then(deref(self.thisptr), func, error_func), self) + return Promise()._init(new_promise.thisptr.attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), new_promise) cpdef as_pypromise(self) except +reraise_kj_exception: if self.is_consumed: - raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') return Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)), self) def attach(self, *args): if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') ret = _VoidPromise()._init(self.thisptr.attach(capnp.makePyRefCounter(args)), self) self.is_consumed = True return ret + cpdef cancel(self, numParents=1) except +reraise_kj_exception: + if numParents > 0 and hasattr(self._parent, 'cancel'): + self._parent.cancel(numParents - 1) + + self.is_consumed = True + del self.thisptr + self.thisptr = NULL + cdef class _RemotePromise: cdef RemotePromise * thisptr cdef public bint is_consumed @@ -1534,26 +1853,38 @@ cdef class _RemotePromise: cpdef wait(self) except +reraise_kj_exception: if self.is_consumed: - raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - ret = _Response()._init_child(self.thisptr.wait(deref(self._event_loop.thisptr).waitScope), self._parent) + ret = _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent) self.is_consumed = True return ret cpdef as_pypromise(self) except +reraise_kj_exception: if self.is_consumed: - raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') return Promise()._init(helpers.convert_to_pypromise(deref(self.thisptr)), self) cpdef then(self, func, error_func=None) except +reraise_kj_exception: if self.is_consumed: - raise RuntimeError('Promise was already used in a consuming operation. You can no longer use this Promise object') + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') + + argspec = None + try: + argspec = _inspect.getargspec(func) + except: + pass + if argspec: + args_length = len(argspec.args) if argspec.args else 0 + defaults_length = len(argspec.defaults) if argspec.defaults else 0 + if args_length - defaults_length != 1: + raise KjException('Function passed to `then` call must take exactly one argument') Py_INCREF(func) Py_INCREF(error_func) - return Promise()._init(helpers.then(deref(self.thisptr), func, error_func).attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), self) + cdef Promise new_promise = Promise()._init(helpers.then(deref(self.thisptr), func, error_func), self) + return Promise()._init(new_promise.thisptr.attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), new_promise) cpdef _get(self, field) except +reraise_kj_exception: cdef int type = (self.thisptr.get(field)).getType() @@ -1562,12 +1893,15 @@ cdef class _RemotePromise: elif type == capnp.TYPE_STRUCT: return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) elif type == capnp.TYPE_UNKNOWN: - raise ValueError("Cannot convert type to Python. Type is unknown by capnproto library") + raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: - raise ValueError("Cannot convert type to Python. Type is unhandled by capnproto library") + raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") def __getattr__(self, field): - return self._get(field) + try: + return self._get(field) + except KjException as e: + raise e._to_python() property schema: """A property that returns the _StructSchema object matching this reader""" @@ -1577,12 +1911,20 @@ cdef class _RemotePromise: def __dir__(self): return list(self.schema.fieldnames) - def to_dict(self, verbose=False): - return _to_dict(self, verbose) + def to_dict(self, verbose=False, ordered=False): + return _to_dict(self, verbose, ordered) + + cpdef cancel(self, numParents=1) except +reraise_kj_exception: + if numParents > 0 and hasattr(self._parent, 'cancel'): + self._parent.cancel(numParents - 1) + + self.is_consumed = True + del self.thisptr + self.thisptr = NULL # def attach(self, *args): # if self.is_consumed: - # raise ValueError('Promise was already used in a consuming operation. You can no longer use this Promise object') + # raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') # ret = _RemotePromise()._init(self.thisptr.attach(capnp.makePyRefCounter(args)), self) # self.is_consumed = True @@ -1603,7 +1945,7 @@ cpdef join_promises(promises) except +reraise_kj_exception: pyPromise = promise.as_pypromise() new_promises_append(pyPromise) else: - raise ValueError('One of the promises passed to `join_promises` had a non promise value of: ' + str(promise)) + raise KjException('One of the promises passed to `join_promises` had a non promise value of: ' + str(promise)) heap.add(movePromise(deref(pyPromise.thisptr))) pyPromise.is_consumed = True @@ -1611,16 +1953,21 @@ cpdef join_promises(promises) except +reraise_kj_exception: cdef class _Request(_DynamicStructBuilder): cdef Request * thisptr_child + cdef public bint is_consumed cdef _init_child(self, Request other, parent): self.thisptr_child = new Request(moveRequest(other)) self._init(deref(self.thisptr_child), parent) + self.is_consumed = False return self def __dealloc__(self): del self.thisptr_child cpdef send(self): + if self.is_consumed: + raise KjException('Request has already been sent. You can only send a request once.') + self.is_consumed = True return _RemotePromise()._init(self.thisptr_child.send(), self._parent) cdef class _Response(_DynamicStructReader): @@ -1654,11 +2001,14 @@ cdef class _DynamicCapabilityServer: self.server = server def __getattr__(self, field): - return getattr(self.server, field) + try: + return getattr(self.server, field) + except KjException as e: + raise e._to_python() cdef class _DynamicCapabilityClient: cdef C_DynamicCapability.Client thisptr - cdef public object _server, _parent + cdef public object _server, _parent, _cached_schema cdef _init(self, C_DynamicCapability.Client other, object parent): self.thisptr = other @@ -1678,14 +2028,13 @@ cdef class _DynamicCapabilityClient: cpdef _find_method_args(self, method_name): s = self.schema - meth = None - for meth in s.node.interface.methods: - if meth.name == method_name: - break + meth = s.methods_inherited.get(method_name, None) + if meth is None: + raise AttributeError("Method named %s not found." % method_name) - params = s.get_dependency(meth.paramStructType).node + params = meth.param_type.node if params.scopeId != 0: - raise ValueError("Cannot call method `%s` with positional args, since its param struct is not implicitly defined and thus does not have a set order of arguments" % method_name) + raise KjException("Cannot call method `%s` with positional args, since its param struct is not implicitly defined and thus does not have a set order of arguments" % method_name) return _find_field_order(params.struct) @@ -1693,7 +2042,7 @@ cdef class _DynamicCapabilityClient: if args is not None: arg_names = self._find_method_args(name) if len(args) > len(arg_names): - raise ValueError('Too many arguments passed to `%s`. Expected %d and got %d' % (name, len(arg_names), len(args))) + raise KjException('Too many arguments passed to `%s`. Expected %d and got %d' % (name, len(arg_names), len(args))) for arg_name, arg_val in zip(arg_names, args): _setDynamicField(deref(request), arg_name, arg_val, self) @@ -1726,10 +2075,18 @@ cdef class _DynamicCapabilityClient: return self._send_helper(name, word_count, args, kwargs) def __getattr__(self, name): - if name.endswith('_request'): - short_name = name[:-8] - return _partial(self._request, short_name) - return _partial(self._send, name) + try: + if name.endswith('_request'): + short_name = name[:-8] + if short_name not in self.schema.method_names_inherited: + raise AttributeError('Method named %s not found' % short_name) + return _partial(self._request, short_name) + + if name not in self.schema.method_names_inherited: + raise AttributeError('Method named %s not found' % name) + return _partial(self._send, name) + except KjException as e: + raise e._to_python() cpdef upcast(self, schema) except +reraise_kj_exception: cdef _InterfaceSchema s @@ -1751,10 +2108,12 @@ cdef class _DynamicCapabilityClient: property schema: """A property that returns the _InterfaceSchema object matching this client""" def __get__(self): - return _InterfaceSchema()._init(self.thisptr.getSchema()) + if self._cached_schema is None: + self._cached_schema = _InterfaceSchema()._init(self.thisptr.getSchema()) + return self._cached_schema def __dir__(self): - return list(self.schema.method_names) + return list(self.schema.method_names_inherited) cdef class _CapabilityClient: cdef C_Capability.Client * thisptr @@ -1813,7 +2172,7 @@ cdef _Restorer _convert_restorer(restorer): elif callable(restorer): return _Restorer(restorer) else: - raise ValueError("Restorer object ({}) isn't able to be used as a restore".format(str(restorer))) + raise KjException("Restorer object ({}) isn't able to be used as a restore".format(str(restorer))) cdef class TwoPartyClient: cdef RpcSystem * thisptr @@ -1870,9 +2229,9 @@ cdef class TwoPartyClient: return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), object_reader.thisptr), self) else: if not hasattr(objectId, 'is_root'): - raise ValueError("objectId was not a valid Cap'n Proto struct") + raise KjException("objectId was not a valid Cap'n Proto struct") if not objectId.is_root: - raise ValueError("objectId must be the root of a Cap'n Proto message, ie. addressbook_capnp.Person.new_message()") + raise KjException("objectId must be the root of a Cap'n Proto message, ie. addressbook_capnp.Person.new_message()") try: builder = objectId._parent @@ -1884,7 +2243,7 @@ cdef class TwoPartyClient: elif reader is not None: return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(reader.thisptr)), self) else: - raise ValueError("objectId unexpectedly was not convertible to the proper type") + raise KjException("objectId unexpectedly was not convertible to the proper type") cpdef ez_restore(self, textId) except +reraise_kj_exception: # ez-rpc from the C++ API uses Text under the hood @@ -1947,7 +2306,7 @@ cdef class TwoPartyServer: cpdef run_forever(self): if self.port_promise is None: - raise ValueError("You must pass a string as the socket parameter in __init__ to use this function") + raise KjException("You must pass a string as the socket parameter in __init__ to use this function") wait_forever() @@ -1992,7 +2351,6 @@ cdef class PromiseFulfillerPair: deref(deref(self.thisptr).fulfiller).fulfill() cdef class _Schema: - cdef C_Schema thisptr cdef _init(self, C_Schema other): self.thisptr = other return self @@ -2010,6 +2368,8 @@ cdef class _Schema: return _EnumSchema()._init(self.thisptr.asEnum()) cpdef get_dependency(self, id): + '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' + _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) return _Schema()._init(self.thisptr.getDependency(id)) cpdef get_proto(self): @@ -2022,12 +2382,16 @@ cdef class _Schema: cdef class _StructSchema: cdef C_StructSchema thisptr - cdef object __fieldnames, __union_fields, __non_union_fields + cdef object __fieldnames, __union_fields, __non_union_fields, __fields, __getters + cdef list __fields_list cdef _init(self, C_StructSchema other): self.thisptr = other self.__fieldnames = None self.__union_fields = None self.__non_union_fields = None + self.__fields = None + self.__fields_list = None + self.__getters = None return self property fieldnames: @@ -2063,12 +2427,36 @@ cdef class _StructSchema: for i in xrange(nfields)) return self.__non_union_fields + property fields: + """All of the _StructSchemaField in this schema as a dict""" + def __get__(self): + if self.__fields is not None: + return self.__fields + fieldlist = self.thisptr.getFields() + nfields = fieldlist.size() + self.__fields = {fieldlist[i].getProto().getName().cStr() : _StructSchemaField()._init(fieldlist[i], self) + for i in xrange(nfields)} + return self.__fields + + property fields_list: + """All of the _StructSchemaField in this schema as a list""" + def __get__(self): + if self.__fields_list is not None: + return self.__fields_list + fieldlist = self.thisptr.getFields() + nfields = fieldlist.size() + self.__fields_list = [_StructSchemaField()._init(fieldlist[i], self) + for i in xrange(nfields)] + return self.__fields_list + property node: """The raw schema node""" def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) cpdef get_dependency(self, id): + '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' + _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) return _Schema()._init(self.thisptr.getDependency(id)) def __richcmp__(_StructSchema self, _StructSchema other, mode): @@ -2082,9 +2470,20 @@ cdef class _StructSchema: def __repr__(self): return '' % self.node.displayName +cdef typeAsSchema(capnp.SchemaType fieldType): + # TODO(soon): make sure this is memory safe + if fieldType.isInterface(): + return _InterfaceSchema()._init(fieldType.asInterface()) + elif fieldType.isStruct(): + return _StructSchema()._init(fieldType.asStruct()) + elif fieldType.isEnum(): + return _EnumSchema()._init(fieldType.asEnum()) + elif fieldType.isList(): + return ListSchema()._init(fieldType.asList()) + else: + raise KjException("Schema type is unknown") + cdef class _StructSchemaField: - cdef C_StructSchema.Field thisptr - cdef object _parent cdef _init(self, C_StructSchema.Field other, parent=None): self.thisptr = other self._parent = parent @@ -2095,13 +2494,28 @@ cdef class _StructSchemaField: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) + property schema: + """The schema of this field, or None if it's a type without a schema""" + def __get__(self): + return typeAsSchema(self.thisptr.getType()) + def __repr__(self): return '' % self.proto.name -cdef class _InterfaceSchema: - cdef C_InterfaceSchema thisptr - cdef object __method_names +cdef class _InterfaceMethod: + cdef C_InterfaceSchema.Method thisptr + cdef _init(self, C_InterfaceSchema.Method other): + self.thisptr = other + return self + + property param_type: + """The type of this method's parameter struct""" + def __get__(self): + # TODO(soon): make sure this is memory safe + return _StructSchema()._init(self.thisptr.getParamType()) + +cdef class _InterfaceSchema: cdef _init(self, C_InterfaceSchema other): self.thisptr = other return self @@ -2117,12 +2531,64 @@ cdef class _InterfaceSchema: for i in xrange(nfields)) return self.__method_names + property method_names_inherited: + """A set of the function names in the interface, including inherited methods""" + def __get__(self): + if self.__method_names_inherited is not None: + return self.__method_names_inherited + + fieldlist = self.thisptr.getMethods() + nfields = fieldlist.size() + self.__method_names_inherited = set(fieldlist[i].getProto().getName().cStr() + for i in xrange(nfields)) + for interface in self.superclasses: + self.__method_names_inherited |= interface.method_names_inherited + + return self.__method_names_inherited + + property methods: + """A mapping of method names to their respective _InterfaceMethod""" + def __get__(self): + if self.__methods is not None: + return self.__methods + + fieldlist = self.thisptr.getMethods() + nfields = fieldlist.size() + # TODO(soon): make sure this is memory safe + self.__methods = {fieldlist[i].getProto().getName().cStr() : _InterfaceMethod()._init(fieldlist[i]) + for i in xrange(nfields)} + return self.__methods + + property methods_inherited: + """A mapping of method names to their respective _InterfaceMethod, including inherited methods""" + def __get__(self): + if self.__methods_inherited is not None: + return self.__methods_inherited + + fieldlist = self.thisptr.getMethods() + nfields = fieldlist.size() + # TODO(soon): make sure this is memory safe + self.__methods_inherited = {fieldlist[i].getProto().getName().cStr() : _InterfaceMethod()._init(fieldlist[i]) + for i in xrange(nfields)} + for interface in self.superclasses: + self.__methods_inherited.update(interface.methods_inherited) + + return self.__methods_inherited + + property superclasses: + """A list of superclasses for this interface""" + def __get__(self): + cdef C_InterfaceSchema.SuperclassList classes = self.thisptr.getSuperclasses() + return [_InterfaceSchema()._init(classes[i]) for i in range(classes.size())] + property node: """The raw schema node""" def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) cpdef get_dependency(self, id): + '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' + _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) return _Schema()._init(self.thisptr.getDependency(id)) def __repr__(self): @@ -2151,6 +2617,130 @@ cdef class _EnumSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) +cdef class _SchemaType: + cdef capnp.SchemaType thisptr + +types = _ModuleType('capnp.types') +cdef _SchemaType _void = _SchemaType() +_void.thisptr = capnp.SchemaType(capnp.TypeWhichVOID) +types.Void = _void + +cdef _SchemaType _bool = _SchemaType() +_bool.thisptr = capnp.SchemaType(capnp.TypeWhichBOOL) +types.Bool = _bool + +cdef _SchemaType _int8 = _SchemaType() +_int8.thisptr = capnp.SchemaType(capnp.TypeWhichINT8) +types.Int8 = _int8 + +cdef _SchemaType _int16 = _SchemaType() +_int16.thisptr = capnp.SchemaType(capnp.TypeWhichINT16) +types.Int16 = _int16 + +cdef _SchemaType _int32 = _SchemaType() +_int32.thisptr = capnp.SchemaType(capnp.TypeWhichINT32) +types.Int32 = _int32 + +cdef _SchemaType _int64 = _SchemaType() +_int64.thisptr = capnp.SchemaType(capnp.TypeWhichINT64) +types.Int64 = _int64 + +cdef _SchemaType _uint8 = _SchemaType() +_uint8.thisptr = capnp.SchemaType(capnp.TypeWhichUINT8) +types.UInt8 = _uint8 + +cdef _SchemaType _uint16 = _SchemaType() +_uint16.thisptr = capnp.SchemaType(capnp.TypeWhichUINT16) +types.UInt16 = _uint16 + +cdef _SchemaType _uint32 = _SchemaType() +_uint32.thisptr = capnp.SchemaType(capnp.TypeWhichUINT32) +types.UInt32 = _uint32 + +cdef _SchemaType _uint64 = _SchemaType() +_uint64.thisptr = capnp.SchemaType(capnp.TypeWhichUINT64) +types.UInt64 = _uint64 + +cdef _SchemaType _float32 = _SchemaType() +_float32.thisptr = capnp.SchemaType(capnp.TypeWhichFLOAT32) +types.Float32 = _float32 + +cdef _SchemaType _float64 = _SchemaType() +_float64.thisptr = capnp.SchemaType(capnp.TypeWhichFLOAT64) +types.Float64 = _float64 + +cdef _SchemaType _text = _SchemaType() +_text.thisptr = capnp.SchemaType(capnp.TypeWhichTEXT) +types.Text = _text + +cdef _SchemaType _data = _SchemaType() +_data.thisptr = capnp.SchemaType(capnp.TypeWhichDATA) +types.Data = _data + +# cdef _SchemaType _list = _SchemaType() +# _list.thisptr = capnp.SchemaType(capnp.TypeWhichLIST) +# types.list = _list + +cdef _SchemaType _enum = _SchemaType() +_enum.thisptr = capnp.SchemaType(capnp.TypeWhichENUM) +types.Enum = _enum + +# cdef _SchemaType _struct = _SchemaType() +# _struct.thisptr = capnp.SchemaType(capnp.TypeWhichSTRUCT) +# types.struct = _struct + +# cdef _SchemaType _interface = _SchemaType() +# _interface.thisptr = capnp.SchemaType(capnp.TypeWhichINTERFACE) +# types.interface = _interface + +cdef _SchemaType _any_pointer = _SchemaType() +_any_pointer.thisptr = capnp.SchemaType(capnp.TypeWhichANY_POINTER) +types.AnyPointer = _any_pointer + +cdef class ListSchema: + cdef C_ListSchema thisptr + + def __init__(self, schema=None): + cdef _StructSchema ss + cdef _EnumSchema es + cdef _InterfaceSchema iis + cdef ListSchema ls + cdef _SchemaType st + + if schema is not None: + if hasattr(schema, 'schema'): + s = schema.schema + else: + s = schema + + typeSchema = type(s) + if typeSchema is _StructSchema: + ss = s + self.thisptr = capnp.listSchemaOfStruct(ss.thisptr) + elif typeSchema is _EnumSchema: + es = s + self.thisptr = capnp.listSchemaOfEnum(es.thisptr) + elif typeSchema is _InterfaceSchema: + iis = s + self.thisptr = capnp.listSchemaOfInterface(iis.thisptr) + elif typeSchema is ListSchema: + ls = s + self.thisptr = capnp.listSchemaOfList(ls.thisptr) + elif typeSchema is _SchemaType: + st = s + self.thisptr = capnp.listSchemaOfType(st.thisptr) + else: + raise KjException("Unknown schema type") + + cdef _init(self, C_ListSchema other): + self.thisptr = other + return self + + property elementType: + """The schema of the element type of this list""" + def __get__(self): + return typeAsSchema(self.thisptr.getElementType()) + cdef class _ParsedSchema(_Schema): cdef C_ParsedSchema thisptr_child cdef _init_child(self, C_ParsedSchema other): @@ -2170,7 +2760,7 @@ cdef _new_message(self, kwargs, num_first_segment_words): builder = _MallocMessageBuilder(num_first_segment_words) msg = builder.init_root(self.schema) if kwargs is not None: - _from_dict(msg, kwargs) + msg.from_dict(kwargs) return msg class _RestorerImpl(object): @@ -2188,19 +2778,20 @@ class _StructModule(object): self.Restorer = type(name + '.Restorer', (_RestorerImpl,), {'schema':schema, '_restore':_restore}) # Add enums for union fields - for field in schema.node.struct.fields: + for field, raw_field in zip(schema.node.struct.fields, schema.fields_list): if field.which() == 'group': - name = field.name.capitalize() - union_schema = schema.get_dependency(field.group.typeId).node.struct + name = field.name[0].upper() + field.name[1:] + raw_schema = raw_field.schema + field_schema = raw_schema.node.struct - if union_schema.discriminantCount == 0: - continue - - union_module = _StructModuleWhich() - setattr(union_module, 'schema', union_schema) - for union_field in union_schema.fields: - setattr(union_module, union_field.name, union_field.discriminantValue) - setattr(self, name, union_module) + if field_schema.discriminantCount == 0: + sub_module = _StructModule(raw_schema, name) + else: + sub_module = _StructModuleWhich() + setattr(sub_module, 'schema', raw_schema) + for union_field in field_schema.fields: + setattr(sub_module, union_field.name, union_field.discriminantValue) + setattr(self, name, sub_module) def read(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the unpacked object read from file. @@ -2402,7 +2993,7 @@ cdef class SchemaParser: :Raises: - :exc:`exceptions.IOError` if `file_name` doesn't exist - - :exc:`exceptions.RuntimeError` if the Cap'n Proto C++ library has any problems loading the schema + - :exc:`KjException` if the Cap'n Proto C++ library has any problems loading the schema """ def _load(nodeSchema, module): @@ -2467,6 +3058,7 @@ cdef class SchemaParser: abs_path = _os.path.abspath(file_name) module.__path__ = _os.path.dirname(abs_path) module.__file__ = abs_path + module.schema = fileSchema return module @@ -2611,9 +3203,6 @@ cdef class _MessageReader: def __init__(self): raise NotImplementedError("This is an abstract base class") - cpdef _get_root_node(self): - return _NodeReader().init(self.thisptr.getRootNode()) - cpdef get_root(self, schema) except +reraise_kj_exception: """A method for instantiating Cap'n Proto structs @@ -2797,7 +3386,7 @@ cdef class _MultipleMessageReader: try: reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) return reader.get_root(self.schema) - except ValueError as e: + except KjException as e: if 'EOF' in str(e): raise StopIteration else: @@ -2828,7 +3417,7 @@ cdef class _MultiplePackedMessageReader: try: reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) return reader.get_root(self.schema) - except ValueError as e: + except KjException as e: if 'EOF' in str(e): raise StopIteration else: @@ -2893,7 +3482,7 @@ cdef class _FlatMessageBuilder(_MessageBuilder): cdef Py_ssize_t sz PyObject_AsWriteBuffer(buf, &ptr, &sz) if sz % 8 != 0: - raise ValueError("input length must be a multiple of eight bytes") + raise KjException("input length must be a multiple of eight bytes") self._object_to_pin = buf self.thisptr = new schema_cpp.FlatMessageBuilder(schema_cpp.WordArrayPtr(ptr, sz//8)) @@ -2992,7 +3581,7 @@ def load(file_name, display_name=None, imports=[]): :return: A module corresponding to the loaded schema. You can access parsed schemas and constants with . syntax - :Raises: :exc:`exceptions.ValueError` if `file_name` doesn't exist + :Raises: :exc:`KjException` if `file_name` doesn't exist """ global _global_schema_parser @@ -3044,6 +3633,12 @@ class _Importer: module_name = module_name[:-len('_capnp')] capnp_module_name = module_name + self.extension + has_underscores = False + + if '_' in capnp_module_name: + capnp_module_name_dashes = capnp_module_name.replace('_', '-') + capnp_module_name_spaces = capnp_module_name.replace('_', ' ') + has_underscores = True if package_path: paths = package_path @@ -3062,8 +3657,14 @@ class _Importer: path = _os.getcwd() elif not is_abs(path): path = abspath(path) + if is_file(path+sep+capnp_module_name): return _Loader(fullname, join_path(path, capnp_module_name), self.additional_paths) + if has_underscores: + if is_file(path+sep+capnp_module_name_dashes): + return _Loader(fullname, join_path(path, capnp_module_name_dashes), self.additional_paths) + if is_file(path+sep+capnp_module_name_spaces): + return _Loader(fullname, join_path(path, capnp_module_name_spaces), self.additional_paths) _importer = None diff --git a/capnp/lib/pickle_helper.py b/capnp/lib/pickle_helper.py new file mode 100644 index 0000000..c28d46a --- /dev/null +++ b/capnp/lib/pickle_helper.py @@ -0,0 +1,6 @@ +import capnp + + +def _struct_reducer(schema_id, data): + 'Hack to deal with pypy not allowing reduce functions to be "built-in" methods (ie. compiled from a .pyx)' + return capnp._global_schema_parser.modules_by_id[schema_id].from_bytes(data) diff --git a/capnp/schema.capnp b/capnp/schema.capnp new file mode 100644 index 0000000..bb3532e --- /dev/null +++ b/capnp/schema.capnp @@ -0,0 +1,383 @@ +# Copyright (c) 2013, Kenton Varda +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +using Cxx = import "c++.capnp"; + +@0xa93fc509624c72d9; +$Cxx.namespace("capnp::schema"); + +using Id = UInt64; +# The globally-unique ID of a file, type, or annotation. + +struct Node { + id @0 :Id; + + displayName @1 :Text; + # Name to present to humans to identify this Node. You should not attempt to parse this. Its + # format could change. It is not guaranteed to be unique. + # + # (On Zooko's triangle, this is the node's nickname.) + + displayNamePrefixLength @2 :UInt32; + # If you want a shorter version of `displayName` (just naming this node, without its surrounding + # scope), chop off this many characters from the beginning of `displayName`. + + scopeId @3 :Id; + # ID of the lexical parent node. Typically, the scope node will have a NestedNode pointing back + # at this node, but robust code should avoid relying on this (and, in fact, group nodes are not + # listed in the outer struct's nestedNodes, since they are listed in the fields). `scopeId` is + # zero if the node has no parent, which is normally only the case with files, but should be + # allowed for any kind of node (in order to make runtime type generation easier). + + nestedNodes @4 :List(NestedNode); + # List of nodes nested within this node, along with the names under which they were declared. + + struct NestedNode { + name @0 :Text; + # Unqualified symbol name. Unlike Node.name, this *can* be used programmatically. + # + # (On Zooko's triangle, this is the node's petname according to its parent scope.) + + id @1 :Id; + # ID of the nested node. Typically, the target node's scopeId points back to this node, but + # robust code should avoid relying on this. + } + + annotations @5 :List(Annotation); + # Annotations applied to this node. + + union { + # Info specific to each kind of node. + + file @6 :Void; + + struct :group { + dataWordCount @7 :UInt16; + # Size of the data section, in words. + + pointerCount @8 :UInt16; + # Size of the pointer section, in pointers (which are one word each). + + preferredListEncoding @9 :ElementSize; + # The preferred element size to use when encoding a list of this struct. If this is anything + # other than `inlineComposite` then the struct is one word or less in size and is a candidate + # for list packing optimization. + + isGroup @10 :Bool; + # If true, then this "struct" node is actually not an independent node, but merely represents + # some named union or group within a particular parent struct. This node's scopeId refers + # to the parent struct, which may itself be a union/group in yet another struct. + # + # All group nodes share the same dataWordCount and pointerCount as the top-level + # struct, and their fields live in the same ordinal and offset spaces as all other fields in + # the struct. + # + # Note that a named union is considered a special kind of group -- in fact, a named union + # is exactly equivalent to a group that contains nothing but an unnamed union. + + discriminantCount @11 :UInt16; + # Number of fields in this struct which are members of an anonymous union, and thus may + # overlap. If this is non-zero, then a 16-bit discriminant is present indicating which + # of the overlapping fields is active. This can never be 1 -- if it is non-zero, it must be + # two or more. + # + # Note that the fields of an unnamed union are considered fields of the scope containing the + # union -- an unnamed union is not its own group. So, a top-level struct may contain a + # non-zero discriminant count. Named unions, on the other hand, are equivalent to groups + # containing unnamed unions. So, a named union has its own independent schema node, with + # `isGroup` = true. + + discriminantOffset @12 :UInt32; + # If `discriminantCount` is non-zero, this is the offset of the union discriminant, in + # multiples of 16 bits. + + fields @13 :List(Field); + # Fields defined within this scope (either the struct's top-level fields, or the fields of + # a particular group; see `isGroup`). + # + # The fields are sorted by ordinal number, but note that because groups share the same + # ordinal space, the field's index in this list is not necessarily exactly its ordinal. + # On the other hand, the field's position in this list does remain the same even as the + # protocol evolves, since it is not possible to insert or remove an earlier ordinal. + # Therefore, for most use cases, if you want to identify a field by number, it may make the + # most sense to use the field's index in this list rather than its ordinal. + } + + enum :group { + enumerants@14 :List(Enumerant); + # Enumerants ordered by numeric value (ordinal). + } + + interface :group { + methods @15 :List(Method); + # Methods ordered by ordinal. + + extends @31 :List(Id); + # Superclasses of this interface. + } + + const :group { + type @16 :Type; + value @17 :Value; + } + + annotation :group { + type @18 :Type; + + targetsFile @19 :Bool; + targetsConst @20 :Bool; + targetsEnum @21 :Bool; + targetsEnumerant @22 :Bool; + targetsStruct @23 :Bool; + targetsField @24 :Bool; + targetsUnion @25 :Bool; + targetsGroup @26 :Bool; + targetsInterface @27 :Bool; + targetsMethod @28 :Bool; + targetsParam @29 :Bool; + targetsAnnotation @30 :Bool; + } + } +} + +struct Field { + # Schema for a field of a struct. + + name @0 :Text; + + codeOrder @1 :UInt16; + # Indicates where this member appeared in the code, relative to other members. + # Code ordering may have semantic relevance -- programmers tend to place related fields + # together. So, using code ordering makes sense in human-readable formats where ordering is + # otherwise irrelevant, like JSON. The values of codeOrder are tightly-packed, so the maximum + # value is count(members) - 1. Fields that are members of a union are only ordered relative to + # the other members of that union, so the maximum value there is count(union.members). + + annotations @2 :List(Annotation); + + const noDiscriminant :UInt16 = 0xffff; + + discriminantValue @3 :UInt16 = Field.noDiscriminant; + # If the field is in a union, this is the value which the union's discriminant should take when + # the field is active. If the field is not in a union, this is 0xffff. + + union { + slot :group { + # A regular, non-group, non-fixed-list field. + + offset @4 :UInt32; + # Offset, in units of the field's size, from the beginning of the section in which the field + # resides. E.g. for a UInt32 field, multiply this by 4 to get the byte offset from the + # beginning of the data section. + + type @5 :Type; + defaultValue @6 :Value; + + hadExplicitDefault @10 :Bool; + # Whether the default value was specified explicitly. Non-explicit default values are always + # zero or empty values. Usually, whether the default value was explicit shouldn't matter. + # The main use case for this flag is for structs representing method parameters: + # explicitly-defaulted parameters may be allowed to be omitted when calling the method. + } + + group :group { + # A group. + + typeId @7 :Id; + # The ID of the group's node. + } + } + + ordinal :union { + implicit @8 :Void; + explicit @9 :UInt16; + # The original ordinal number given to the field. You probably should NOT use this; if you need + # a numeric identifier for a field, use its position within the field array for its scope. + # The ordinal is given here mainly just so that the original schema text can be reproduced given + # the compiled version -- i.e. so that `capnp compile -ocapnp` can do its job. + } +} + +struct Enumerant { + # Schema for member of an enum. + + name @0 :Text; + + codeOrder @1 :UInt16; + # Specifies order in which the enumerants were declared in the code. + # Like Struct.Field.codeOrder. + + annotations @2 :List(Annotation); +} + +struct Method { + # Schema for method of an interface. + + name @0 :Text; + + codeOrder @1 :UInt16; + # Specifies order in which the methods were declared in the code. + # Like Struct.Field.codeOrder. + + paramStructType @2 :Id; + # ID of the parameter struct type. If a named parameter list was specified in the method + # declaration (rather than a single struct parameter type) then a corresponding struct type is + # auto-generated. Such an auto-generated type will not be listed in the interface's + # `nestedNodes` and its `scopeId` will be zero -- it is completely detached from the namespace. + + resultStructType @3 :Id; + # ID of the return struct type; similar to `paramStructType`. + + annotations @4 :List(Annotation); +} + +struct Type { + # Represents a type expression. + + union { + # The ordinals intentionally match those of Value. + + void @0 :Void; + bool @1 :Void; + int8 @2 :Void; + int16 @3 :Void; + int32 @4 :Void; + int64 @5 :Void; + uint8 @6 :Void; + uint16 @7 :Void; + uint32 @8 :Void; + uint64 @9 :Void; + float32 @10 :Void; + float64 @11 :Void; + text @12 :Void; + data @13 :Void; + + list :group { + elementType @14 :Type; + } + + enum :group { + typeId @15 :Id; + } + struct :group { + typeId @16 :Id; + } + interface :group { + typeId @17 :Id; + } + + anyPointer @18 :Void; + } +} + +struct Value { + # Represents a value, e.g. a field default value, constant value, or annotation value. + + union { + # The ordinals intentionally match those of Type. + + void @0 :Void; + bool @1 :Bool; + int8 @2 :Int8; + int16 @3 :Int16; + int32 @4 :Int32; + int64 @5 :Int64; + uint8 @6 :UInt8; + uint16 @7 :UInt16; + uint32 @8 :UInt32; + uint64 @9 :UInt64; + float32 @10 :Float32; + float64 @11 :Float64; + text @12 :Text; + data @13 :Data; + + list @14 :AnyPointer; + + enum @15 :UInt16; + struct @16 :AnyPointer; + + interface @17 :Void; + # The only interface value that can be represented statically is "null", whose methods always + # throw exceptions. + + anyPointer @18 :AnyPointer; + } +} + +struct Annotation { + # Describes an annotation applied to a declaration. Note AnnotationNode describes the + # annotation's declaration, while this describes a use of the annotation. + + id @0 :Id; + # ID of the annotation node. + + value @1 :Value; +} + +enum ElementSize { + # Possible element sizes for encoded lists. These correspond exactly to the possible values of + # the 3-bit element size component of a list pointer. + + empty @0; # aka "void", but that's a keyword. + bit @1; + byte @2; + twoBytes @3; + fourBytes @4; + eightBytes @5; + pointer @6; + inlineComposite @7; +} + +struct CodeGeneratorRequest { + nodes @0 :List(Node); + # All nodes parsed by the compiler, including for the files on the command line and their + # imports. + + requestedFiles @1 :List(RequestedFile); + # Files which were listed on the command line. + + struct RequestedFile { + id @0 :Id; + # ID of the file. + + filename @1 :Text; + # Name of the file as it appeared on the command-line (minus the src-prefix). You may use + # this to decide where to write the output. + + imports @2 :List(Import); + # List of all imported paths seen in this file. + + struct Import { + id @0 :Id; + # ID of the imported file. + + name @1 :Text; + # Name which *this* file used to refer to the foreign file. This may be a relative name. + # This information is provided because it might be useful for code generation, e.g. to + # generate #include directives in C++. We don't put this in Node.file because this + # information is only meaningful at compile time anyway. + # + # (On Zooko's triangle, this is the import's petname according to the importing file.) + } + } +} diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx new file mode 100644 index 0000000..df45582 --- /dev/null +++ b/capnp/templates/module.pyx @@ -0,0 +1,284 @@ +# addressbook_fast.pyx +# distutils: language = c++ +# distutils: extra_compile_args = --std=c++11 +# distutils: include_dirs = {{include_dir}} +# distutils: libraries = capnpc capnp capnp-rpc +# distutils: sources = {{file.filename}}.cpp +# cython: c_string_type = str +# cython: c_string_encoding = default +# cython: embedsignature = True + +{% macro getter(field, type) -%} + {% if 'uint' in field['type'] -%} +uint64_t get{{field.c_name}}() except +reraise_kj_exception + {% elif 'int' in field['type'] -%} +int64_t get{{field.c_name}}() except +reraise_kj_exception + {% elif 'void' == field['type'] -%} +void get{{field.c_name}}() except +reraise_kj_exception + {% elif 'bool' == field['type'] -%} +cbool get{{field.c_name}}() except +reraise_kj_exception + {% elif 'text' == field['type'] -%} +StringPtr get{{field.c_name}}() except +reraise_kj_exception + {% elif 'data' == field['type'] -%} +Data.{{type}} get{{field.c_name}}() except +reraise_kj_exception + {% else -%} +DynamicValue.{{type}} get{{field.c_name}}() except +reraise_kj_exception + {%- endif %} +{%- endmacro %} +# TODO: add struct/enum/list types + +{% macro getfield(field, type) -%} +cpdef _get_{{field.name}}(self): + {% if 'int' in field['type'] -%} + return self.thisptr_child.get{{field.c_name}}() + {% elif 'void' == field['type'] -%} + self.thisptr_child.get{{field.c_name}}() + return None + {% elif 'bool' == field['type'] -%} + return self.thisptr_child.get{{field.c_name}}() + {% elif 'text' == field['type'] -%} + temp = self.thisptr_child.get{{field.c_name}}() + return (temp.begin())[:temp.size()] + {% elif 'data' == field['type'] -%} + temp = self.thisptr_child.get{{field.c_name}}() + return ((temp.begin())[:temp.size()]) + {% else -%} + cdef DynamicValue.{{type}} temp = self.thisptr_child.get{{field.c_name}}() + return to_python_{{type | lower}}(temp, self._parent) + {% endif -%} +{%- endmacro %} + +{% macro setter(field) -%} + {% if 'int' in field['type'] -%} +void set{{field.c_name}}({{field.type}}_t) except +reraise_kj_exception + {% elif 'bool' == field['type'] -%} +void set{{field.c_name}}(cbool) except +reraise_kj_exception + {% elif 'text' == field['type'] -%} +void set{{field.c_name}}(StringPtr) except +reraise_kj_exception + {% elif 'data' == field['type'] -%} +void set{{field.c_name}}(ArrayPtr[byte]) except +reraise_kj_exception + {% else -%} +void set{{field.c_name}}(DynamicValue.Reader) except +reraise_kj_exception + {%- endif %} +{%- endmacro %} + +{% macro setfield(field) -%} + {% if 'int' in field['type'] -%} +cpdef _set_{{field.name}}(self, {{field.type}}_t value): + self.thisptr_child.set{{field.c_name}}(value) + {% elif 'void' == field['type'] -%} +cpdef _set_{{field.name}}(self, value=None): + pass + {% elif 'bool' == field['type'] -%} +cpdef _set_{{field.name}}(self, bool value): + self.thisptr_child.set{{field.c_name}}(value) + {% elif 'list' == field['type'] -%} +cpdef _set_{{field.name}}(self, list value): + cdef uint i = 0 + self.init("{{field.name}}", len(value)) + cdef _DynamicListBuilder temp = self._get_{{field.name}}() + for elem in value: + {% if 'struct' in field['sub_type'] -%} + temp._get(i).from_dict(elem) + {% else -%} + temp[i] = elem + {% endif -%} + i += 1 + {% elif 'text' == field['type'] -%} +cpdef _set_{{field.name}}(self, value): + cdef StringPtr temp_string + if type(value) is bytes: + temp_string = StringPtr(value, len(value)) + else: + encoded_value = value.encode() + temp_string = StringPtr(encoded_value, len(encoded_value)) + self.thisptr_child.set{{field.c_name}}(temp_string) + {% elif 'data' == field['type'] -%} +cpdef _set_{{field.name}}(self, value): + cdef StringPtr temp_string + if type(value) is bytes: + temp_string = StringPtr(value, len(value)) + else: + encoded_value = value.encode() + temp_string = StringPtr(encoded_value, len(encoded_value)) + self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](temp_string.begin(), temp_string.size())) + {% else -%} +cpdef _set_{{field.name}}(self, value): + _setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent) + {% endif -%} +{%- endmacro %} + +import capnp +import {{file.filename | replace('.', '_')}} + +from capnp.includes.types cimport * +from capnp cimport helpers +from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr, ArrayPtr, Data +from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema + +from capnp.helpers.non_circular cimport reraise_kj_exception + +cdef DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value): + return DynamicValue.Reader(value.thisptr.asReader()) + +cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader value): + return DynamicValue.Reader(value.thisptr) + +cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): + return DynamicValue.Reader(value.thisptr) + +cdef _from_list(_DynamicListBuilder msg, list d): + cdef size_t count = 0 + for val in d: + msg._set(count, val) + count += 1 + +cdef DynamicValue.Reader to_dynamic_value(value): + cdef DynamicValue.Reader temp + cdef StringPtr temp_string + value_type = type(value) + + if value_type is int or value_type is long: + if value < 0: + temp = DynamicValue.Reader(value) + else: + temp = DynamicValue.Reader(value) + elif value_type is float: + temp = DynamicValue.Reader(value) + elif value_type is bool: + temp = DynamicValue.Reader(value) + elif value_type is bytes: + temp_string = StringPtr(value, len(value)) + temp = DynamicValue.Reader(temp_string) + elif isinstance(value, basestring): + encoded_value = value.encode() + temp_string = StringPtr(encoded_value, len(encoded_value)) + temp = DynamicValue.Reader(temp_string) + elif value is None: + temp = DynamicValue.Reader(VOID) + elif value_type is _DynamicStructBuilder: + temp = _extract_dynamic_struct_builder(value) + elif value_type is _DynamicStructReader: + temp = _extract_dynamic_struct_reader(value) + elif value_type is _DynamicEnum: + temp = _extract_dynamic_enum(value) + else: + raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value)))) + + return temp + + +cdef extern from "{{file.filename}}.h": + {%- for node in code.nodes %} + Schema get{{node.module_name}}Schema"capnp::Schema::from<{{node.c_module_path}}>"() + + cdef cppclass {{node.module_name}}"{{node.c_module_path}}": + cppclass Reader: + {%- for field in node.struct.fields %} + {{ getter(field, "Reader")|indent(12)}} + {%- endfor %} + cppclass Builder: + {%- for field in node.struct.fields %} + {{ getter(field, "Builder")|indent(12)}} + {{ setter(field)|indent(12)}} + {%- endfor %} + {%- endfor %} + + cdef cppclass C_DynamicStruct_Reader" ::capnp::DynamicStruct::Reader": + {%- for node in code.nodes %} + {{node.module_name}}.Reader as{{node.module_name}}"as<{{node.c_module_path}}>"() + {%- endfor %} + + cdef cppclass C_DynamicStruct_Builder" ::capnp::DynamicStruct::Builder": + {%- for node in code.nodes %} + {{node.module_name}}.Builder as{{node.module_name}}"as<{{node.c_module_path}}>"() + {%- endfor %} + +{%- for node in code.nodes %} + +{{node.schema}} = _Schema()._init(get{{node.module_name}}Schema()).as_struct() +{{node.module_path}}.schema = {{node.schema}} + +cdef class {{node.module_name}}_Reader(_DynamicStructReader): + cdef {{node.module_name}}.Reader thisptr_child + def __init__(self, _DynamicStructReader struct): + self._init(struct.thisptr, struct._parent, struct.is_root, False) + self.thisptr_child = (struct.thisptr).as{{node.module_name}}() + {% for field in node.struct.fields %} + + {{ getfield(field, "Reader")|indent(4) }} + + property {{field.name}}: + def __get__(self): + return self._get_{{field.name}}() + {%- endfor %} + + def to_dict(self, verbose=False, ordered=False): + ret = { + {% for field in node.struct.fields %} + {% if field.discriminantValue == 65535 %} + '{{field.name}}': _to_dict(self.{{field.name}}, verbose, ordered), + {% endif %} + {%- endfor %} + } + + {% if node.is_union %} + which = self._which_str() + ret[which] = getattr(self, which) + {% endif %} + + return ret + +cdef class {{node.module_name}}_Builder(_DynamicStructBuilder): + cdef {{node.module_name}}.Builder thisptr_child + def __init__(self, _DynamicStructBuilder struct): + self._init(struct.thisptr, struct._parent, struct.is_root, False) + self.thisptr_child = (struct.thisptr).as{{node.module_name}}() + {% for field in node.struct.fields %} + {{ getfield(field, "Builder")|indent(4) }} + {{ setfield(field)|indent(4) }} + + property {{field.name}}: + def __get__(self): + return self._get_{{field.name}}() + def __set__(self, value): + self._set_{{field.name}}(value) + {%- endfor %} + + def to_dict(self, verbose=False, ordered=False): + ret = { + {% for field in node.struct.fields %} + {% if field.discriminantValue == 65535 %} + '{{field.name}}': _to_dict(self.{{field.name}}, verbose, ordered), + {% endif %} + {%- endfor %} + } + + {% if node.is_union %} + which = self._which_str() + ret[which] = getattr(self, which) + {% endif %} + + return ret + + def from_dict(self, dict d): + cdef str key + for key, val in d.iteritems(): + if False: pass + {% for field in node.struct.fields %} + elif key == "{{field.name}}": + try: + self._set_{{field.name}}(val) + except Exception as e: + if 'expected isSetInUnion(field)' in str(e): + self.init(key) + self._set_{{field.name}}(val) + else: + raise + {%- endfor %} + else: + raise ValueError('Key not found in struct: ' + key) + + +capnp.register_type({{node.id}}, ({{node.module_name}}_Reader, {{node.module_name}}_Builder)) +{% endfor %} diff --git a/capnp/templates/setup.py.tmpl b/capnp/templates/setup.py.tmpl new file mode 100644 index 0000000..e1e2e9a --- /dev/null +++ b/capnp/templates/setup.py.tmpl @@ -0,0 +1,33 @@ +#!/usr/bin/env python +from distutils.core import setup +from Cython.Build import cythonize +import os +import re + + +files = [{% for f in code.requestedFiles %}"{{f.filename}}", {% endfor %}] + +for f in files: + cpp_file = f + '.cpp' + cplus_file = f + '.c++' + cpp_mod = os.path.getmtime(cpp_file) + cplus_mod = 0 + try: + cplus_mod = os.path.getmtime(cplus_file) + except: + pass + if not os.path.exists(cpp_file) or cpp_mod < cplus_mod: + if not os.path.exists(cplus_file): + raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.") + os.rename(cplus_file, cpp_file) + + with open(f + '.h', "r") as file: + lines = file.readlines() + with open(f + '.h', "w") as file: + for line in lines: + file.write(re.sub(r'Builder\(\)\s*=\s*delete;', 'Builder() = default;', line)) + +setup( + name="{{code.requestedFiles[0] | replace('.', '_')}}", + ext_modules=cythonize('*_capnp_cython.pyx', language="c++") +) diff --git a/examples/addressbook.capnp b/examples/addressbook.capnp index e1cd77c..95fb26a 100644 --- a/examples/addressbook.capnp +++ b/examples/addressbook.capnp @@ -18,7 +18,6 @@ struct Person { work @2; } } - employment :union { unemployed @4 :Void; employer @5 :Text; @@ -26,6 +25,13 @@ struct Person { selfEmployed @7 :Void; # We assume that a person is only one of these. } + + testGroup :group { + field1 @8 :UInt32; + field2 @9 :UInt32; + field3 @10 :UInt32; + } + extraData @11 :Data; } struct AddressBook { diff --git a/examples/thread.capnp b/examples/thread.capnp new file mode 100644 index 0000000..ae32b8d --- /dev/null +++ b/examples/thread.capnp @@ -0,0 +1,11 @@ +@0xf5745ea9c82baa3a; + +interface Example { + interface StatusSubscriber { + status @0 (value: Bool); + # Call the function on the given parameters. + } + + longRunning @0 () -> (value: Bool); + subscribeStatus @1 (subscriber: StatusSubscriber); +} diff --git a/examples/thread_client.py b/examples/thread_client.py new file mode 100755 index 0000000..7201dd0 --- /dev/null +++ b/examples/thread_client.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import threading +import time +import capnp + +import thread_capnp + +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +def start_status_thread(host): + client = capnp.TwoPartyClient(host) + cap = client.ez_restore('example').cast_as(thread_capnp.Example) + + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + promise.wait() + + +def main(host): + client = capnp.TwoPartyClient(host) + cap = client.ez_restore('example').cast_as(thread_capnp.Example) + + status_thread = threading.Thread(target=start_status_thread, args=(host,)) + status_thread.daemon = True + status_thread.start() + + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + +if __name__ == '__main__': + main(parse_args().host) diff --git a/examples/thread_server.py b/examples/thread_server.py new file mode 100755 index 0000000..04b1e2d --- /dev/null +++ b/examples/thread_server.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import capnp + +import thread_capnp + + +class ExampleImpl(thread_capnp.Example.Server): + + "Implementation of the Example threading Cap'n Proto interface." + + def subscribeStatus(self, subscriber, **kwargs): + return capnp.getTimer().after_delay(10**9) \ + .then(lambda: subscriber.status(True)) \ + .then(lambda _: self.subscribeStatus(subscriber)) + + def longRunning(self, **kwargs): + return capnp.getTimer().after_delay(3 * 10**9) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS may be '*' to bind to all local addresses.\ +:PORT may be omitted to choose a port automatically. ''') + + parser.add_argument("address", help="ADDRESS[:PORT]") + + return parser.parse_args() + + +impl = ExampleImpl() + + +def restore(ref): + assert ref.as_text() == 'example' + return impl + + +def main(): + address = parse_args().address + + server = capnp.TwoPartyServer(address, restore) + server.run_forever() + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 5b743ff..617ceb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +jinja2 >= 2.7.3 cython >= 0.21 setuptools >= 0.8 pytest diff --git a/setup.py b/setup.py index bae80ec..379a84a 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,8 @@ from Cython.Distutils import build_ext as build_ext_c _this_dir = os.path.dirname(__file__) MAJOR = 0 -MINOR = 4 -MICRO = 6 +MINOR = 5 +MICRO = 0 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) # Write version info @@ -114,7 +114,7 @@ setup( name="pycapnp", packages=["capnp"], version=VERSION, - package_data={'capnp': ['*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx']}, + package_data={'capnp': ['*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*']}, ext_modules=cythonize('capnp/lib/*.pyx'), cmdclass = { 'clean': clean, @@ -123,6 +123,9 @@ setup( install_requires=[ 'cython >= 0.21', 'setuptools >= 0.8'], + entry_points={ + 'console_scripts' : ['capnpc-cython = capnp._gen:main'] + }, # PyPi info description="A cython wrapping of the C++ Cap'n Proto library", long_description=long_description, diff --git a/test/addressbook with spaces.capnp b/test/addressbook with spaces.capnp new file mode 100644 index 0000000..f7c611e --- /dev/null +++ b/test/addressbook with spaces.capnp @@ -0,0 +1,39 @@ +@0xc39aee9191aedcf3; + +const qux :UInt32 = 123; + +struct Person { + id @0 :UInt32; + name @1 :Text; + email @2 :Text; + phones @3 :List(PhoneNumber); + + struct PhoneNumber { + number @0 :Text; + type @1 :Type; + + enum Type { + mobile @0; + home @1; + work @2; + } + } + + employment :union { + unemployed @4 :Void; + employer @5 :Employer; + school @6 :Text; + selfEmployed @7 :Void; + # We assume that a person is only one of these. + } +} + +struct Employer { + name @0 :Text; + boss @1 :Person; +} + +struct AddressBook { + people @0 :List(Person); +} + diff --git a/test/addressbook-with-dashes.capnp b/test/addressbook-with-dashes.capnp new file mode 100644 index 0000000..2295e8e --- /dev/null +++ b/test/addressbook-with-dashes.capnp @@ -0,0 +1,39 @@ +@0xd33206731939e03b; + +const qux :UInt32 = 123; + +struct Person { + id @0 :UInt32; + name @1 :Text; + email @2 :Text; + phones @3 :List(PhoneNumber); + + struct PhoneNumber { + number @0 :Text; + type @1 :Type; + + enum Type { + mobile @0; + home @1; + work @2; + } + } + + employment :union { + unemployed @4 :Void; + employer @5 :Employer; + school @6 :Text; + selfEmployed @7 :Void; + # We assume that a person is only one of these. + } +} + +struct Employer { + name @0 :Text; + boss @1 :Person; +} + +struct AddressBook { + people @0 :List(Person); +} + diff --git a/test/annotations.capnp b/test/annotations.capnp new file mode 100644 index 0000000..bc41fa8 --- /dev/null +++ b/test/annotations.capnp @@ -0,0 +1,17 @@ +@0xfb9a160831eee9bb; + +struct AnnotationStruct { + test @0: Int32; +} + +annotation test1(*): Text; +annotation test2(*): AnnotationStruct; +annotation test3(*): List(AnnotationStruct); +annotation test4(*): List(UInt16); + +$test1("TestFile"); + +struct TestAnnotationOne $test1("Test") { } +struct TestAnnotationTwo $test2(test = 100) { } +struct TestAnnotationThree $test3([(test=100), (test=101)]) { } +struct TestAnnotationFour $test4([200, 201]) { } diff --git a/test/object.capnp b/test/object.capnp deleted file mode 100644 index 9bb2b29..0000000 --- a/test/object.capnp +++ /dev/null @@ -1,21 +0,0 @@ -@0x8186ddb142b58556; - -struct Person { - id @0 :UInt32; - name @1 :Text; -} - -struct Place { - id @0 :UInt32; - name @1 :Text; -} - -struct Thing { - id @0 :UInt64; - value @1 :UInt64; -} - -struct TestObject { - object @0 :AnyPointer; -} - diff --git a/test/test_capability.py b/test/test_capability.py index f6f007d..7a21739 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -1,6 +1,7 @@ import pytest import capnp import os +import time import test_capability_capnp as capability @@ -31,7 +32,7 @@ class PipelineServer(capability.TestPipeline.Server): def test_client(): client = capability.TestInterface._new_client(Server()) - + req = client._request('foo') req.i = 5 @@ -39,7 +40,7 @@ def test_client(): response = remote.wait() assert response.x == '26' - + req = client.foo_request() req.i = 5 @@ -48,12 +49,12 @@ def test_client(): assert response.x == '26' - with pytest.raises(ValueError): + with pytest.raises(AttributeError): client.foo2_request() req = client.foo_request() - with pytest.raises(ValueError): + with pytest.raises(Exception): req.i = 'foo' req = client.foo_request() @@ -63,18 +64,18 @@ def test_client(): def test_simple_client(): client = capability.TestInterface._new_client(Server()) - + remote = client._send('foo', i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5, j=True) response = remote.wait() @@ -106,19 +107,19 @@ def test_simple_client(): assert response.x == '5_test' assert response.i == 5 - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, 10) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, True, 100) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(i='foo') - with pytest.raises(ValueError): + with pytest.raises(AttributeError): remote = client.foo2(i=5) - with pytest.raises(AttributeError): + with pytest.raises(Exception): remote = client.foo(baz=5) def test_pipeline(): @@ -148,7 +149,7 @@ class BadServer(capability.TestInterface.Server): def test_exception_client(): client = capability.TestInterface._new_client(BadServer()) - + remote = client._send('foo', i=5) with pytest.raises(capnp.KjException): remote.wait() @@ -254,4 +255,91 @@ def test_tail_call(): assert result.n == 2 assert callee_server.count == 1 - assert caller_server.count == 1 \ No newline at end of file + assert caller_server.count == 1 + + +def test_cancel(): + client = capability.TestInterface._new_client(Server()) + + req = client._request('foo') + req.i = 5 + + remote = req.send() + remote.cancel() + + with pytest.raises(Exception): + remote.wait() + + +def test_timer(): + global test_timer_var + test_timer_var = False + + def set_timer_var(): + global test_timer_var + test_timer_var = True + capnp.getTimer().after_delay(1).then(set_timer_var).wait() + + assert test_timer_var is True + + test_timer_var = False + promise = capnp.Promise(0).then(lambda x: time.sleep(.1)).then(lambda x: time.sleep(.1)).then(lambda x: set_timer_var()) + + canceller = capnp.getTimer().after_delay(1).then(lambda: promise.cancel()) + + joined = capnp.join_promises([canceller, promise]) + joined.wait() + + # faling for now, not sure why... + # assert test_timer_var is False + + +def test_double_send(): + client = capability.TestInterface._new_client(Server()) + + req = client._request('foo') + req.i = 5 + + req.send() + with pytest.raises(Exception): + req.send() + + +def test_then_args(): + capnp.Promise(0).then(lambda x: 1) + + with pytest.raises(Exception): + capnp.Promise(0).then(lambda: 1) + + with pytest.raises(Exception): + capnp.Promise(0).then(lambda x, y: 1) + + capnp.getTimer().after_delay(1).then(lambda: 1) # after_delay is a VoidPromise + + with pytest.raises(Exception): + capnp.getTimer().after_delay(1).then(lambda x: 1) + + client = capability.TestInterface._new_client(Server()) + + client.foo(i=5).then(lambda x: 1) + + with pytest.raises(Exception): + client.foo(i=5).then(lambda: 1) + + with pytest.raises(Exception): + client.foo(i=5).then(lambda x, y: 1) + + +class ExtendsServer(Server): + def qux(self, **kwargs): + pass + + +def test_inheritance(): + client = capability.TestExtends._new_client(ExtendsServer()) + client.qux().wait() + + remote = client.foo(i=5) + response = remote.wait() + + assert response.x == '26' diff --git a/test/test_capability_context.py b/test/test_capability_context.py index 0653d01..4f2c111 100644 --- a/test/test_capability_context.py +++ b/test/test_capability_context.py @@ -31,7 +31,7 @@ class PipelineServer: def test_client_context(capability): client = capability.TestInterface._new_client(Server()) - + req = client._request('foo') req.i = 5 @@ -39,7 +39,7 @@ def test_client_context(capability): response = remote.wait() assert response.x == '26' - + req = client.foo_request() req.i = 5 @@ -48,12 +48,12 @@ def test_client_context(capability): assert response.x == '26' - with pytest.raises(ValueError): + with pytest.raises(AttributeError): client.foo2_request() req = client.foo_request() - with pytest.raises(ValueError): + with pytest.raises(Exception): req.i = 'foo' req = client.foo_request() @@ -63,18 +63,18 @@ def test_client_context(capability): def test_simple_client_context(capability): client = capability.TestInterface._new_client(Server()) - + remote = client._send('foo', i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5, j=True) response = remote.wait() @@ -100,19 +100,19 @@ def test_simple_client_context(capability): assert response.x == 'localhost_test' - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, 10) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, True, 100) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(i='foo') - with pytest.raises(ValueError): + with pytest.raises(AttributeError): remote = client.foo2(i=5) - with pytest.raises(AttributeError): + with pytest.raises(Exception): remote = client.foo(baz=5) def test_pipeline_context(capability): @@ -140,7 +140,7 @@ class BadServer: def test_exception_client_context(capability): client = capability.TestInterface._new_client(BadServer()) - + remote = client._send('foo', i=5) with pytest.raises(capnp.KjException): remote.wait() diff --git a/test/test_capability_old.py b/test/test_capability_old.py index 215fc96..beaa5ca 100644 --- a/test/test_capability_old.py +++ b/test/test_capability_old.py @@ -32,7 +32,7 @@ class PipelineServer: def test_client(capability): client = capability.TestInterface._new_client(Server()) - + req = client._request('foo') req.i = 5 @@ -40,7 +40,7 @@ def test_client(capability): response = remote.wait() assert response.x == '26' - + req = client.foo_request() req.i = 5 @@ -49,12 +49,12 @@ def test_client(capability): assert response.x == '26' - with pytest.raises(ValueError): + with pytest.raises(AttributeError): client.foo2_request() req = client.foo_request() - with pytest.raises(ValueError): + with pytest.raises(Exception): req.i = 'foo' req = client.foo_request() @@ -64,18 +64,18 @@ def test_client(capability): def test_simple_client(capability): client = capability.TestInterface._new_client(Server()) - + remote = client._send('foo', i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5) response = remote.wait() assert response.x == '26' - + remote = client.foo(i=5, j=True) response = remote.wait() @@ -101,19 +101,19 @@ def test_simple_client(capability): assert response.x == 'localhost_test' - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, 10) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(5, True, 100) - with pytest.raises(ValueError): + with pytest.raises(Exception): remote = client.foo(i='foo') - with pytest.raises(ValueError): + with pytest.raises(AttributeError): remote = client.foo2(i=5) - with pytest.raises(AttributeError): + with pytest.raises(Exception): remote = client.foo(baz=5) def test_pipeline(capability): @@ -143,7 +143,7 @@ class BadServer: def test_exception_client(capability): client = capability.TestInterface._new_client(BadServer()) - + remote = client._send('foo', i=5) with pytest.raises(capnp.KjException): remote.wait() @@ -249,4 +249,4 @@ def test_tail_call(capability): assert result.n == 2 assert callee_server.count == 1 - assert caller_server.count == 1 \ No newline at end of file + assert caller_server.count == 1 diff --git a/test/test_load.py b/test/test_load.py index 25e1107..7f6d6af 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -52,12 +52,18 @@ def test_failed_import(): foo.name = 'foo' - with pytest.raises(ValueError): + with pytest.raises(Exception): bar.foo = foo def test_defualt_import_hook(): import addressbook_capnp +def test_dash_import(): + import addressbook_with_dashes_capnp + +def test_spaces_import(): + import addressbook_with_spaces_capnp + def test_add_import_hook(): capnp.add_import_hook([this_dir]) diff --git a/test/test_object.py b/test/test_object.py index 28beb03..f87e124 100644 --- a/test/test_object.py +++ b/test/test_object.py @@ -4,21 +4,48 @@ import os this_dir = os.path.dirname(__file__) -@pytest.fixture -def object(): - return capnp.load(os.path.join(this_dir, 'object.capnp')) -def test_object_basic(object): - obj = object.TestObject.new_message() - person = obj.object.as_struct(object.Person) +@pytest.fixture +def addressbook(): + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + + +def test_object_basic(addressbook): + obj = capnp._MallocMessageBuilder().get_root_as_any() + person = obj.as_struct(addressbook.Person) person.name = 'test' person.id = 1000 - same_person = obj.object.as_struct(object.Person) + same_person = obj.as_struct(addressbook.Person) assert same_person.name == 'test' assert same_person.id == 1000 obj_r = obj.as_reader() - same_person = obj_r.object.as_struct(object.Person) + same_person = obj_r.as_struct(addressbook.Person) assert same_person.name == 'test' assert same_person.id == 1000 + + +def test_object_list(addressbook): + obj = capnp._MallocMessageBuilder().get_root_as_any() + listSchema = capnp.ListSchema(addressbook.Person) + people = obj.init_as_list(listSchema, 2) + person = people[0] + person.name = 'test' + person.id = 1000 + person = people[1] + person.name = 'test2' + person.id = 1001 + + same_person = obj.as_list(listSchema) + assert same_person[0].name == 'test' + assert same_person[0].id == 1000 + assert same_person[1].name == 'test2' + assert same_person[1].id == 1001 + + obj_r = obj.as_reader() + same_person = obj_r.as_list(listSchema) + assert same_person[0].name == 'test' + assert same_person[0].id == 1000 + assert same_person[1].name == 'test2' + assert same_person[1].id == 1001 diff --git a/test/test_regression.py b/test/test_regression.py index c488b75..388bcc5 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -156,7 +156,7 @@ def test_addressbook_resizable(addressbook): bob.employment.unemployed = None people.finish() - + addresses.write(file) @@ -192,6 +192,75 @@ def test_addressbook_resizable(addressbook): f = open('example', 'r') printAddressBook(f) +def test_addressbook_explicit_fields(addressbook): + def writeAddressBook(file): + addresses = addressbook.AddressBook.new_message() + address_fields = addressbook.AddressBook.schema.fields + person_fields = addressbook.Person.schema.fields + phone_fields = addressbook.Person.PhoneNumber.schema.fields + people = addresses._init_by_field(address_fields['people'], 2) + + alice = people[0] + alice._set_by_field(person_fields['id'], 123) + alice._set_by_field(person_fields['name'], 'Alice') + alice._set_by_field(person_fields['email'], 'alice@example.com') + alicePhones = alice._init_by_field(person_fields['phones'], 1) + alicePhones[0]._set_by_field(phone_fields['number'], "555-1212") + alicePhones[0]._set_by_field(phone_fields['type'], 'mobile') + employment = alice._get_by_field(person_fields['employment']) + employment._set_by_field(addressbook.Person.Employment.schema.fields['school'], "MIT") + + bob = people[1] + bob._set_by_field(person_fields['id'], 456) + bob._set_by_field(person_fields['name'], 'Bob') + bob._set_by_field(person_fields['email'], 'bob@example.com') + bobPhones = bob._init_by_field(person_fields['phones'], 2) + bobPhones[0]._set_by_field(phone_fields['number'], "555-4567") + bobPhones[0]._set_by_field(phone_fields['type'], 'home') + bobPhones[1]._set_by_field(phone_fields['number'], "555-7654") + bobPhones[1]._set_by_field(phone_fields['type'], 'work') + employment = bob._get_by_field(person_fields['employment']) + employment._set_by_field(addressbook.Person.Employment.schema.fields['unemployed'], None) + + addresses.write(file) + + + def printAddressBook(file): + addresses = addressbook.AddressBook.read(file) + address_fields = addressbook.AddressBook.schema.fields + person_fields = addressbook.Person.schema.fields + phone_fields = addressbook.Person.PhoneNumber.schema.fields + + people = addresses._get_by_field(address_fields['people']) + + alice = people[0] + assert alice._get_by_field(person_fields['id']) == 123 + assert alice._get_by_field(person_fields['name']) == 'Alice' + assert alice._get_by_field(person_fields['email']) == 'alice@example.com' + alicePhones = alice._get_by_field(person_fields['phones']) + assert alicePhones[0]._get_by_field(phone_fields['number']) == "555-1212" + assert alicePhones[0]._get_by_field(phone_fields['type']) == 'mobile' + employment = alice._get_by_field(person_fields['employment']) + employment._get_by_field(addressbook.Person.Employment.schema.fields['school']) == "MIT" + + bob = people[1] + assert bob._get_by_field(person_fields['id']) == 456 + assert bob._get_by_field(person_fields['name']) == 'Bob' + assert bob._get_by_field(person_fields['email']) == 'bob@example.com' + bobPhones = bob._get_by_field(person_fields['phones']) + assert bobPhones[0]._get_by_field(phone_fields['number']) == "555-4567" + assert bobPhones[0]._get_by_field(phone_fields['type']) == 'home' + assert bobPhones[1]._get_by_field(phone_fields['number']) == "555-7654" + assert bobPhones[1]._get_by_field(phone_fields['type']) == 'work' + employment = bob._get_by_field(person_fields['employment']) + employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) == None + + + f = open('example', 'w') + writeAddressBook(f) + + f = open('example', 'r') + printAddressBook(f) @pytest.fixture def all_types(): diff --git a/test/test_schema.py b/test/test_schema.py new file mode 100644 index 0000000..0e472dd --- /dev/null +++ b/test/test_schema.py @@ -0,0 +1,50 @@ +import pytest +import capnp +import os + +this_dir = os.path.dirname(__file__) + + +@pytest.fixture +def addressbook(): + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + + +@pytest.fixture +def annotations(): + return capnp.load(os.path.join(this_dir, 'annotations.capnp')) + + +def test_basic_schema(addressbook): + assert addressbook.Person.schema.fieldnames[0] == 'id' + + +def test_list_schema(addressbook): + peopleField = addressbook.AddressBook.schema.fields['people'] + personType = peopleField.schema.elementType + + assert personType.node.id == addressbook.Person.schema.node.id + + personListSchema = capnp.ListSchema(addressbook.Person) + + assert personListSchema.elementType.node.id == addressbook.Person.schema.node.id + + +def test_annotations(annotations): + assert annotations.schema.node.annotations[0].value.text == 'TestFile' + + annotation = annotations.TestAnnotationOne.schema.node.annotations[0] + assert annotation.value.text == 'Test' + + annotation = annotations.TestAnnotationTwo.schema.node.annotations[0] + assert annotation.value.struct.as_struct(annotations.AnnotationStruct).test == 100 + + annotation = annotations.TestAnnotationThree.schema.node.annotations[0] + annotation_list = annotation.value.list.as_list(capnp.ListSchema(annotations.AnnotationStruct)) + assert annotation_list[0].test == 100 + assert annotation_list[1].test == 101 + + annotation = annotations.TestAnnotationFour.schema.node.annotations[0] + annotation_list = annotation.value.list.as_list(capnp.ListSchema(capnp.types.UInt16)) + assert annotation_list[0] == 200 + assert annotation_list[1] == 201 diff --git a/test/test_serialization.py b/test/test_serialization.py index 4c6203f..3928c31 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -4,6 +4,7 @@ import os import platform import test_regression import tempfile +import pickle this_dir = os.path.dirname(__file__) @@ -98,3 +99,11 @@ def test_file_and_bytes_packed(all_types): f.seek(0) assert f.read() == msg.to_bytes_packed() + +def test_pickle(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + data = pickle.dumps(msg) + msg2 = pickle.loads(data) + + test_regression.check_all_types(msg2) diff --git a/test/test_struct.py b/test/test_struct.py index c459ff5..f0c3b45 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -37,9 +37,9 @@ def test_which_builder(addressbook): assert bob.employment.which == addressbook.Person.Employment.unemployed assert bob.employment.which == "unemployed" - with pytest.raises(ValueError): + with pytest.raises(Exception): addresses.which - with pytest.raises(ValueError): + with pytest.raises(Exception): addresses.which @@ -71,9 +71,9 @@ def test_which_reader(addressbook): bob = people[1] assert bob.employment.which == "unemployed" - with pytest.raises(ValueError): + with pytest.raises(Exception): addresses.which - with pytest.raises(ValueError): + with pytest.raises(Exception): addresses.which @@ -187,3 +187,34 @@ def test_to_dict_enum(addressbook): field = person.to_dict()['phones'][0]['type'] assert isstr(field) assert field == 'mobile' + + +def test_explicit_field(addressbook): + person = addressbook.Person.new_message(**{'name': 'Test'}) + + name_field = addressbook.Person.schema.fields['name'] + + assert person.name == person._get_by_field(name_field) + assert person.name == person.as_reader()._get_by_field(name_field) + + +def test_to_dict_verbose(addressbook): + person = addressbook.Person.new_message(**{'name': 'Test'}) + + assert person.to_dict(verbose=True)['phones'] == [] + + if sys.version_info >= (2, 7): + assert person.to_dict(verbose=True, ordered=True)['phones'] == [] + + with pytest.raises(KeyError): + assert person.to_dict()['phones'] == [] + + +def test_to_dict_ordered(addressbook): + person = addressbook.Person.new_message(**{'name': 'Alice', 'phones': [{'type': 'mobile', 'number': '555-1212'}], 'id': 123, 'employment': {'school': 'MIT'}, 'email': 'alice@example.com'}) + + if sys.version_info >= (2, 7): + assert list(person.to_dict(ordered=True).keys()) == ['id', 'name', 'email', 'phones', 'employment'] + else: + with pytest.raises(Exception): + person.to_dict(ordered=True) diff --git a/test/test_threads.py b/test/test_threads.py new file mode 100644 index 0000000..8604ecf --- /dev/null +++ b/test/test_threads.py @@ -0,0 +1,66 @@ +import capnp +import pytest +import test_capability_capnp +import socket +import threading +import platform + +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +def test_making_event_loop(): + capnp.remove_event_loop(True) + capnp.create_event_loop() + + capnp.remove_event_loop() + capnp.create_event_loop() + +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +def test_making_threaded_event_loop(): + capnp.remove_event_loop(True) + capnp.create_event_loop(True) + + capnp.remove_event_loop() + capnp.create_event_loop(True) + + +class Server(test_capability_capnp.TestInterface.Server): + + def __init__(self, val=1): + self.val = val + + def foo(self, i, j, **kwargs): + return str(i * 5 + self.val) + + +class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): + + def restore(self, ref_id): + assert ref_id.tag == 'testInterface' + return Server(100) + + +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +def test_using_threads(): + capnp.remove_event_loop(True) + capnp.create_event_loop(True) + + read, write = socket.socketpair(socket.AF_UNIX) + + def run_server(): + restorer = SimpleRestorer() + server = capnp.TwoPartyServer(write, restorer) + capnp.wait_forever() + + server_thread = threading.Thread(target=run_server) + server_thread.daemon = True + server_thread.start() + + client = capnp.TwoPartyClient(read) + + ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') + cap = client.restore(ref) + cap = cap.cast_as(test_capability_capnp.TestInterface) + + remote = cap.foo(i=5) + response = remote.wait() + + assert response.x == '125'