From 3d77722e882e5f083dbb50294987509044d13ce7 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 13 Apr 2015 16:51:38 -0700 Subject: [PATCH 001/126] Add support for unix sockets and improve rpc testing --- capnp/lib/capnp.pyx | 15 ++++++++++----- test/test_rpc_calculator.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 3af300d..8b9f885 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2204,13 +2204,18 @@ cdef class TwoPartyClient: del self.thisptr cpdef _connect(self, host_string): - host, port = host_string.split(':') + if host_string.startswith('unix:'): + path = host_string[5:] + sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) + sock.connect(path) + else: + host, port = host_string.split(':') - sock = _socket.create_connection((host, port)) + sock = _socket.create_connection((host, port)) - # Set TCP_NODELAY on socket to disable Nagle's algorithm. This is not - # neccessary, but it speeds things up. - sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) + # Set TCP_NODELAY on socket to disable Nagle's algorithm. This is not + # neccessary, but it speeds things up. + sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) return sock cpdef restore(self, objectId) except +reraise_kj_exception: diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index 2eccf99..1d7777a 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -2,9 +2,12 @@ import capnp import os import socket import gc +import subprocess +import time import sys # add examples dir to sys.path -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'examples')) +examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') +sys.path.append(examples_dir) import calculator_client import calculator_server @@ -16,6 +19,31 @@ def test_calculator(): calculator_client.main(read) +def run_subprocesses(address): + server = subprocess.Popen([examples_dir + '/calculator_server.py', address]) + time.sleep(.1) # Give the server some small amount of time to start listening + client = subprocess.Popen([examples_dir + '/calculator_client.py', address]) + + ret = client.wait() + server.kill() + assert ret == 0 + + +def test_calculator_tcp(): + address = '127.0.0.1:36431' + run_subprocesses(address) + + +def test_calculator_unix(): + path = '/tmp/pycapnp-test' + try: + os.unlink(path) + except OSError: + pass + + address = 'unix:' + path + run_subprocesses(address) + def test_calculator_gc(): def new_evaluate_impl(old_evaluate_impl): def call(*args, **kwargs): From a5bf532d53db37414d5e6b25f139ef237a854cc1 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 4 May 2015 11:00:25 -0700 Subject: [PATCH 002/126] Add `init` method to lists --- capnp/lib/capnp.pxd | 2 ++ capnp/lib/capnp.pyx | 11 +++++++++++ test/addressbook.capnp | 3 +++ test/test_struct.py | 13 +++++++++++++ 4 files changed, 29 insertions(+) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index d7e3360..4936dd7 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -115,6 +115,8 @@ cdef class _DynamicListBuilder: cpdef adopt(self, index, _DynamicOrphan orphan) cpdef disown(self, index) + cpdef init(self, index, size) + cdef to_python_reader(C_DynamicValue.Reader self, object parent) cdef to_python_builder(C_DynamicValue.Builder self, object parent) cdef _to_dict(msg, bint verbose, bint ordered) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 8b9f885..41f7b50 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -534,6 +534,17 @@ cdef class _DynamicListBuilder: """ return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) + cpdef init(self, index, size): + """A method for initializing an element in a list + + :type index: int + :param index: The index of the element in the list + + :type size: int + :param size: Size of the element to be initialized. + """ + return to_python_builder(self.thisptr.init(index, size), self._parent) + def __str__(self): return printListBuilder(self.thisptr).flatten().cStr() diff --git a/test/addressbook.capnp b/test/addressbook.capnp index b50b68b..576396c 100644 --- a/test/addressbook.capnp +++ b/test/addressbook.capnp @@ -37,3 +37,6 @@ struct AddressBook { people @0 :List(Person); } +struct NestedList { + list @0 :List(List(Int32)); +} diff --git a/test/test_struct.py b/test/test_struct.py index f0c3b45..563ba02 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -218,3 +218,16 @@ def test_to_dict_ordered(addressbook): else: with pytest.raises(Exception): person.to_dict(ordered=True) + +def test_nested_list(addressbook): + struct = addressbook.NestedList.new_message() + struct.init('list', 2) + + struct.list.init(0, 1) + struct.list.init(1, 2) + + struct.list[0][0] = 1 + struct.list[1][0] = 2 + struct.list[1][1] = 3 + + assert struct.to_dict()["list"] == [[1], [2,3]] From 790bdce72ab3f2b6c203f00eaf07e6d605aa631a Mon Sep 17 00:00:00 2001 From: Kamal Marhubi Date: Thu, 28 May 2015 14:25:40 -0400 Subject: [PATCH 003/126] Force use of virtualized Travic CI infrastructure to allow sudo Newly set up projects on Travis CI use their containerized infrastructure, which doesn't allow setuid programs to run. Adding the `sudo: required` line opts out of the containerized infrastructure. The current `setup_travis.sh` uses sudo to call `update-alternatives` which doesn't have an equivalent yet. Tracking bug for that feature: https://github.com/travis-ci/travis-ci/issues/3668 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index e65ce0d..5b28cdc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +# Use older, non-container infrastructure to allow sudo +sudo: required + language: python python: From fc031d3611da180705f05abc68c42b6d9619d85e Mon Sep 17 00:00:00 2001 From: Kamal Marhubi Date: Wed, 6 May 2015 19:43:24 -0400 Subject: [PATCH 004/126] Add python 3.4 to the tox envlist --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 54b3276..e41316b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py32,py33 +envlist = py27,py32,py33,py34 [testenv] deps= From b31b53b6a7a22f3dba0e7d75ffd8abbcf57b8bfa Mon Sep 17 00:00:00 2001 From: Kamal Marhubi Date: Sat, 30 May 2015 19:38:40 -0400 Subject: [PATCH 005/126] Add test for setting a field from a list --- test/test_struct.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_struct.py b/test/test_struct.py index 563ba02..14aaca2 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -107,6 +107,14 @@ def test_builder_set(addressbook): person.foo = 'test' +def test_builder_set_from_list(all_types): + msg = all_types.TestAllTypes.new_message() + + msg.int32List = [0, 1, 2] + + assert list(msg.int32List) == [0, 1, 2] + + def test_null_str(all_types): msg = all_types.TestAllTypes.new_message() From 211192f235bbad77775097cd40204229a4049eaa Mon Sep 17 00:00:00 2001 From: Kamal Marhubi Date: Sat, 30 May 2015 19:42:27 -0400 Subject: [PATCH 006/126] Tidy up _from_list - remove unused variable - switch from `range` to `enumerate` for indexed iteration --- capnp/lib/capnp.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 41f7b50..1f9ab02 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -873,10 +873,10 @@ cdef _to_dict(msg, bint verbose, bint ordered): return msg + cdef _from_list(_DynamicListBuilder msg, list d): - cdef size_t count = 0 - for i in range(len(d)): - msg._set(i, d[i]) + for i, x in enumerate(d): + msg._set(i, x) cdef class _DynamicEnum: From 8f78a7c80b2e9ec037f778eed4ef491d10dd8a07 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 3 Jun 2015 14:35:39 -0700 Subject: [PATCH 007/126] Add null capability test --- test/test_capability.capnp | 6 +++++- test/test_capability.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/test/test_capability.capnp b/test/test_capability.capnp index 87a2e17..5d3c140 100644 --- a/test/test_capability.capnp +++ b/test/test_capability.capnp @@ -76,4 +76,8 @@ interface TestTailCallee { interface TestTailCaller { foo @0 (i :Int32, callee :TestTailCallee) -> TestTailCallee.TailResult; -} \ No newline at end of file +} + +interface TestPassedCap { + foo @0 (cap :TestInterface) -> (x: Text); +} diff --git a/test/test_capability.py b/test/test_capability.py index 7a21739..61b7906 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -343,3 +343,18 @@ def test_inheritance(): response = remote.wait() assert response.x == '26' + + +class TestPassedCap(capability.TestPassedCap.Server): + def foo(self, cap, _context, **kwargs): + def set_result(res): + _context.results.x = res.x + return cap.foo(5).then(set_result) + + +def test_null_cap(): + client = capability.TestPassedCap._new_client(TestPassedCap()) + assert client.foo(Server()).wait().x == '26' + + with pytest.raises(capnp.KjException): + client.foo().wait() From 606a325cfa0a9bed57b3cefe7d539224ef04722e Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 3 Jun 2015 15:29:33 -0700 Subject: [PATCH 008/126] Change sleep timer to be longer for test_rpc_calculator Hopefully this fixes timing issues with the test on travisci --- test/test_rpc_calculator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index 1d7777a..ecf73ec 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -21,7 +21,7 @@ def test_calculator(): def run_subprocesses(address): server = subprocess.Popen([examples_dir + '/calculator_server.py', address]) - time.sleep(.1) # Give the server some small amount of time to start listening + time.sleep(2) # Give the server some small amount of time to start listening client = subprocess.Popen([examples_dir + '/calculator_client.py', address]) ret = client.wait() From 7ff67ebf4aefb190c256166e1aa45b28639eb3f4 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 4 Jun 2015 16:49:46 -0700 Subject: [PATCH 009/126] Add bootstrap method to TwoPartyServer --- capnp/helpers/helpers.pxd | 1 + capnp/helpers/rpcHelper.h | 7 +++++++ capnp/lib/capnp.pyx | 3 +++ 3 files changed, 11 insertions(+) diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index 15b7197..0d9884e 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -31,6 +31,7 @@ cdef extern from "capnp/helpers/rpcHelper.h": Capability.Client restoreHelper(RpcSystem&, AnyPointer.Reader&) Capability.Client restoreHelper(RpcSystem&, AnyPointer.Builder&) Capability.Client bootstrapHelper(RpcSystem&) + Capability.Client bootstrapHelperServer(RpcSystem&) RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&) PyPromise connectServerRestorer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr) PyPromise connectServer(TaskSet &, Capability.Client, AsyncIoContext *, StringPtr) diff --git a/capnp/helpers/rpcHelper.h b/capnp/helpers/rpcHelper.h index cd25a92..54f3081 100644 --- a/capnp/helpers/rpcHelper.h +++ b/capnp/helpers/rpcHelper.h @@ -80,6 +80,13 @@ capnp::Capability::Client bootstrapHelper(capnp::RpcSystem& client) { + capnp::MallocMessageBuilder hostIdMessage(8); + auto hostId = hostIdMessage.initRoot(); + hostId.setSide(capnp::rpc::twoparty::Side::CLIENT); + return client.bootstrap(hostId); +} + template capnp::RpcSystem makeRpcClientWithRestorer( diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 1f9ab02..ffac811 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2352,6 +2352,9 @@ cdef class TwoPartyServer: wait_forever() + cpdef bootstrap(self) except +reraise_kj_exception: + return _CapabilityClient()._init(helpers.bootstrapHelperServer(deref(self.thisptr)), self) + property port: def __get__(self): if self._port is None: From 6f1ce8bcc5fd91c41f6da5160e3c7b164a93fadb Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 8 Jun 2015 13:39:10 -0700 Subject: [PATCH 010/126] Add support for using keyword arguments with a named struct in an RPC --- capnp/lib/capnp.pyx | 2 +- test/test_capability.capnp | 8 ++++++++ test/test_capability.py | 12 ++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index ffac811..2c8158a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2050,7 +2050,7 @@ cdef class _DynamicCapabilityClient: return _find_field_order(params.struct) cdef _set_fields(self, Request * request, name, args, kwargs): - if args is not None: + if args is not None and len(args) > 0: arg_names = self._find_method_args(name) if len(args) > len(arg_names): raise KjException('Too many arguments passed to `%s`. Expected %d and got %d' % (name, len(arg_names), len(args))) diff --git a/test/test_capability.capnp b/test/test_capability.capnp index 5d3c140..816fdf3 100644 --- a/test/test_capability.capnp +++ b/test/test_capability.capnp @@ -81,3 +81,11 @@ interface TestTailCaller { interface TestPassedCap { foo @0 (cap :TestInterface) -> (x: Text); } + +interface TestStructArg { + bar @0 BarParams -> (c: Text); +} +struct BarParams { + a @0 :Text; + b @1 :Int32; +} diff --git a/test/test_capability.py b/test/test_capability.py index 61b7906..1667d7d 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -358,3 +358,15 @@ def test_null_cap(): with pytest.raises(capnp.KjException): client.foo().wait() + + +class TestStructArg(capability.TestStructArg.Server): + def bar(self, a, b, **kwargs): + return a + str(b) + + +def test_struct_args(): + client = capability.TestStructArg._new_client(TestStructArg()) + assert client.bar(a='test', b=1).wait().c == 'test1' + with pytest.raises(capnp.KjException): + assert client.bar('test', 1).wait().c == 'test1' From a4990f08687e5215e84ca4ef24904e026a3d44b8 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 8 Jun 2015 14:59:21 -0700 Subject: [PATCH 011/126] Handle AnyPointers better as arguments to RPC functions --- capnp/includes/capnp_cpp.pxd | 1 + capnp/lib/capnp.pyx | 18 ++++++++++++++++++ test/test_capability.capnp | 4 ++++ test/test_capability.py | 13 +++++++++++++ 4 files changed, 36 insertions(+) diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index c246d0b..7816d20 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -427,6 +427,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": Reader(DynamicStruct.Reader& value) Reader(DynamicCapability.Client& value) Reader(PythonInterfaceDynamicImpl& value) + Reader(AnyPointer.Reader& value) Type getType() int64_t asInt"as"() uint64_t asUint"as"() diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 2c8158a..eb6696a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -661,6 +661,12 @@ cdef C_DynamicValue.Reader _extract_dynamic_server(object value): cdef C_DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): return C_DynamicValue.Reader(value.thisptr) +cdef C_DynamicValue.Reader _extract_any_pointer(_DynamicObjectReader value): + return C_DynamicValue.Reader(value.thisptr) + +cdef C_DynamicValue.Reader _extract_any_pointer_builder(_DynamicObjectBuilder value): + return C_DynamicValue.Reader(value.thisptr.asReader()) + cdef _setBytes(_DynamicSetterClasses thisptr, field, value): cdef capnp.StringPtr temp_string = capnp.StringPtr(value, len(value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) @@ -726,6 +732,10 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) @@ -768,6 +778,10 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField thisptr.setByField(field.thisptr, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.setByField(field.thisptr, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) @@ -810,6 +824,10 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) diff --git a/test/test_capability.capnp b/test/test_capability.capnp index 816fdf3..6770a3a 100644 --- a/test/test_capability.capnp +++ b/test/test_capability.capnp @@ -89,3 +89,7 @@ struct BarParams { a @0 :Text; b @1 :Int32; } + +interface TestGeneric(MyObject) { + foo @0 (a :MyObject) -> (b: Text); +} diff --git a/test/test_capability.py b/test/test_capability.py index 1667d7d..71114ac 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -370,3 +370,16 @@ def test_struct_args(): assert client.bar(a='test', b=1).wait().c == 'test1' with pytest.raises(capnp.KjException): assert client.bar('test', 1).wait().c == 'test1' + + +class TestGeneric(capability.TestGeneric.Server): + def foo(self, a, **kwargs): + return a.as_text() + 'test' + + +def test_generic(): + client = capability.TestGeneric._new_client(TestGeneric()) + + obj = capnp._MallocMessageBuilder().get_root_as_any() + obj.set_as_text("anypointer_") + assert client.foo(obj).wait().b == 'anypointer_test' From 819f21938e9425f23e15bb254478cfd18e1788df Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 16 Jun 2015 11:28:18 -0700 Subject: [PATCH 012/126] Fix warning from PyEventPort --- capnp/helpers/asyncHelper.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/capnp/helpers/asyncHelper.h b/capnp/helpers/asyncHelper.h index 7d1b7ba..8b4e1f1 100644 --- a/capnp/helpers/asyncHelper.h +++ b/capnp/helpers/asyncHelper.h @@ -13,11 +13,13 @@ public: virtual bool wait() { GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("wait"), NULL); + return true; // TODO: get the bool result from python } virtual bool poll() { GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("poll"), NULL); + return true; // TODO: get the bool result from python } virtual void setRunnable(bool runnable) { From bd0576b68a87813361a95b6a32d5b14f5583bb9e Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 16 Jun 2015 11:47:13 -0700 Subject: [PATCH 013/126] Add warnings for using old restorer methods --- capnp/lib/capnp.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index eb6696a..8d84517 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2221,6 +2221,7 @@ cdef class TwoPartyClient: self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) self._restorer = None else: + _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) self._restorer = _convert_restorer(restorer) self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self._network.thisptr), deref(self._restorer.thisptr))) @@ -2248,6 +2249,7 @@ cdef class TwoPartyClient: return sock cpdef restore(self, objectId) except +reraise_kj_exception: + _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) cdef _MessageBuilder builder cdef _MessageReader reader cdef _DynamicObjectBuilder object_builder @@ -2326,6 +2328,7 @@ cdef class TwoPartyServer: schema = bootstrap.schema self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) elif restorer: + _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) self._restorer = _convert_restorer(restorer) self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr))) From 338b58741307f0dc717c7a9b1ed6c3fa47d93e59 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 16 Jun 2015 11:49:40 -0700 Subject: [PATCH 014/126] Update bundled libcapnp to v0.5.2 --- buildutils/bundle.py | 4 ++-- buildutils/setup_travis.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index f1292c4..9271036 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,8 +35,8 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,5,1,2) -libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version) +bundled_version = (0,5,2) +libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp HERE = os.path.dirname(__file__) diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh index 956dcf5..e25ae80 100755 --- a/buildutils/setup_travis.sh +++ b/buildutils/setup_travis.sh @@ -2,7 +2,7 @@ set -exo pipefail -CAPNP_VERSION=0.5.1.2 +CAPNP_VERSION=0.5.2 sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get -qq update From 4b76211b787ad93cd4c506429b710618a1d8c635 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 16 Jun 2015 11:57:33 -0700 Subject: [PATCH 015/126] Bump version to v0.5.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d97a4f4..512a464 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 6 +MICRO = 7 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From e67e3ce04c766325ace5ccaa93d1a4f388bb2463 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 16 Jun 2015 12:08:11 -0700 Subject: [PATCH 016/126] Update changelog for v0.5.7 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab8d43..ad13689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.5.7 (2015-06-16) +- Update bundled libcapnp to v0.5.2 +- Add warnings for using old restorer methods. You should use `bootstrap` instead +- Fix warning from PyEventPort +- Handle AnyPointers better as arguments to RPC functions +- Add support for using keyword arguments with a named struct in an RPC +- Add bootstrap method to TwoPartyServer +- Add `init` method to lists +- Add support for unix sockets in RPC + ## v0.5.6 (2015-04-13) - Fix a serious bug in TwoPartyServer that was preventing it from working when passed a string address. - Fix bugs that were exposed by defining KJDEBUG (thanks @davidcarne for finding this) From b57f0f0df8962c45aec5e3ae1ad10c309f0394ba Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 12 Jul 2015 12:21:29 -0700 Subject: [PATCH 017/126] Add `result_type` to InterfaceMethodSchema --- capnp/lib/capnp.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 8d84517..19b7acf 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2579,6 +2579,12 @@ cdef class _InterfaceMethod: # TODO(soon): make sure this is memory safe return _StructSchema()._init(self.thisptr.getParamType()) + property result_type: + """The type of this method's result struct""" + def __get__(self): + # TODO(soon): make sure this is memory safe + return _StructSchema()._init(self.thisptr.getResultType()) + cdef class _InterfaceSchema: cdef _init(self, C_InterfaceSchema other): self.thisptr = other From 3357771b393d4c4d218dfbc178e4715ca63b97da Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 13 Jul 2015 10:57:55 -0700 Subject: [PATCH 018/126] Fixes for changes in cython v0.22.1 --- capnp/lib/capnp.pxd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 4936dd7..1673365 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -52,9 +52,9 @@ cdef class _DynamicStructBuilder: cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?) cdef _check_write(self) - cpdef to_bytes(_DynamicStructBuilder self) - cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) - cpdef to_bytes_packed(_DynamicStructBuilder self) + cpdef to_bytes(_DynamicStructBuilder self) except +reraise_kj_exception + cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception + cpdef to_bytes_packed(_DynamicStructBuilder self) except +reraise_kj_exception cpdef _get(self, field) cpdef _set(self, field, value) @@ -102,7 +102,7 @@ cdef class _DynamicEnum: cdef public object _parent cdef _init(self, capnp.DynamicEnum other, object parent) - cpdef _as_str(self) + cpdef _as_str(self) except +reraise_kj_exception cdef class _DynamicListBuilder: cdef C_DynamicList.Builder thisptr From 4e6f3818554f2d70aa10dcd8aa965706a157ad2c Mon Sep 17 00:00:00 2001 From: Sergei Dyshel Date: Wed, 21 Oct 2015 15:02:06 +0300 Subject: [PATCH 019/126] Fix KjException init (missing wrapper). --- capnp/lib/capnp.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 19b7acf..ac6e4dd 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -207,6 +207,7 @@ class KjException(Exception): self.wrapper = wrapper self.message = str(wrapper) else: + self.wrapper = None self.message = message self.nature = nature self.durability = durability From 8436a1908df4992241719338f3197c7cbdccb9f6 Mon Sep 17 00:00:00 2001 From: Klee Dienes Date: Sat, 14 Nov 2015 18:49:08 -0500 Subject: [PATCH 020/126] Add reraise_kj_exception to the prettyPrint functions. --- capnp/lib/capnp.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index ac6e4dd..d9f671b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -302,11 +302,11 @@ cdef extern from "" namespace "std": capnp.AsyncIoContext moveAsyncContext"std::move"(capnp.AsyncIoContext) cdef extern from "" namespace " ::capnp": - StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) - StringTree printStructBuilder" ::capnp::prettyPrint"(DynamicStruct_Builder) - StringTree printRequest" ::capnp::prettyPrint"(Request &) - StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) - StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) + StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) except +reraise_kj_exception + StringTree printStructBuilder" ::capnp::prettyPrint"(DynamicStruct_Builder) except +reraise_kj_exception + StringTree printRequest" ::capnp::prettyPrint"(Request &) except +reraise_kj_exception + StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) except +reraise_kj_exception + StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) except +reraise_kj_exception cdef class _NodeReader: cdef C_Node.Reader thisptr From b863e9b9d6f31078fc16de976423009d664af0c2 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Dec 2015 16:01:28 -0800 Subject: [PATCH 021/126] Fix mistakenly discarding the file parameter on reads Fixes #82 --- capnp/lib/capnp.pyx | 46 +++++++++++++++++++++++------------------ test/test_regression.py | 2 +- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index d9f671b..59fd519 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2877,7 +2877,7 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: :class:`_DynamicStructReader`""" - reader = _StreamFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit) + reader = _StreamFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns an iterable, that when traversed will return Readers for messages. @@ -2892,7 +2892,7 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultipleMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultipleMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit) return reader def read_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object read from file. @@ -2907,7 +2907,7 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: :class:`_DynamicStructReader`""" - reader = _PackedFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit) + reader = _PackedFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns an iterable, that when traversed will return Readers for messages. @@ -2922,7 +2922,7 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultiplePackedMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit) return reader def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns an iterable, that when traversed will return Readers for messages. @@ -3369,21 +3369,23 @@ cdef class _StreamFdMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_message_to_fd` and :class:`_MessageBuilder`, but in one class:: f = open('out.txt') - message = _StreamFdMessageReader(f.fileno()) + message = _StreamFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name :Parameters: - fd (`int`) - A file descriptor """ - def __init__(self, int fd, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): cdef schema_cpp.ReaderOptions opts + self._parent = file + if traversal_limit_in_words is not None: opts.traversalLimitInWords = traversal_limit_in_words if nesting_limit is not None: opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.StreamFdMessageReader(fd, opts) + self.thisptr = new schema_cpp.StreamFdMessageReader(file.fileno(), opts) def __dealloc__(self): del self.thisptr @@ -3395,7 +3397,7 @@ cdef class _PackedMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name @@ -3452,7 +3454,7 @@ cdef class _InputMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name @@ -3484,21 +3486,23 @@ cdef class _PackedFdMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name :Parameters: - fd (`int`) - A file descriptor """ - def __init__(self, int fd, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): cdef schema_cpp.ReaderOptions opts + self._parent = file + if traversal_limit_in_words is not None: opts.traversalLimitInWords = traversal_limit_in_words if nesting_limit is not None: opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.PackedFdMessageReader(fd, opts) + self.thisptr = new schema_cpp.PackedFdMessageReader(file.fileno(), opts) def __dealloc__(self): del self.thisptr @@ -3508,14 +3512,15 @@ cdef class _MultipleMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream - cdef public object traversal_limit_in_words, nesting_limit, schema + cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, int fd, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None): + self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - self.stream = new schema_cpp.FdInputStream(fd) + self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): @@ -3539,14 +3544,15 @@ cdef class _MultiplePackedMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream - cdef public object traversal_limit_in_words, nesting_limit, schema + cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, int fd, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None): + self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - self.stream = new schema_cpp.FdInputStream(fd) + self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): @@ -3730,7 +3736,7 @@ def _write_message_to_fd(int fd, _MessageBuilder message): _write_message_to_fd(f.fileno(), message) ... f = open('out.txt') - _StreamFdMessageReader(f.fileno()) + _StreamFdMessageReader(f) :type fd: int :param fd: A file descriptor @@ -3756,7 +3762,7 @@ def _write_packed_message_to_fd(int fd, _MessageBuilder message): _write_packed_message_to_fd(f.fileno(), message) ... f = open('out.txt') - _PackedFdMessageReader(f.fileno()) + _PackedFdMessageReader(f) :type fd: int :param fd: A file descriptor diff --git a/test/test_regression.py b/test/test_regression.py index 388bcc5..d21c2f4 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -39,7 +39,7 @@ def test_addressbook_message_classes(addressbook): def printAddressBook(fd): - message = capnp._PackedFdMessageReader(f.fileno()) + message = capnp._PackedFdMessageReader(f) addressBook = message.get_root(addressbook.AddressBook) people = addressBook.people From 2516e3e4f1fcc6be1060310daf43100c28faa21f Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Dec 2015 16:31:26 -0800 Subject: [PATCH 022/126] Change read_multiple and read_multiple_packed to copy by default Add a `skip_copy` paramter for people that know what they're doing and need higher performance. Fixes #87 --- capnp/lib/capnp.pyx | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 59fd519..d5c6d1d 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2879,7 +2879,7 @@ class _StructModule(object): :rtype: :class:`_DynamicStructReader`""" reader = _StreamFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) - def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None): + def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): """Returns an iterable, that when traversed will return Readers for messages. :type file: file @@ -2891,8 +2891,11 @@ class _StructModule(object): :type nesting_limit: int :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + :type skip_copy: bool + :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is never read completely. Skip this only if you know what you're doing. + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultipleMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultipleMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) return reader def read_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object read from file. @@ -2909,7 +2912,7 @@ class _StructModule(object): :rtype: :class:`_DynamicStructReader`""" reader = _PackedFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) - def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): + def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): """Returns an iterable, that when traversed will return Readers for messages. :type file: file @@ -2921,8 +2924,11 @@ class _StructModule(object): :type nesting_limit: int :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + :type skip_copy: bool + :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is never read completely. Skip this only if you know what you're doing. + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultiplePackedMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultiplePackedMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) return reader def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns an iterable, that when traversed will return Readers for messages. @@ -3511,14 +3517,16 @@ cdef class _PackedFdMessageReader(_MessageReader): cdef class _MultipleMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef cbool skip_copy cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit + self.skip_copy = skip_copy self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) @@ -3530,7 +3538,10 @@ cdef class _MultipleMessageReader: def __next__(self): try: reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root(self.schema) + ret = reader.get_root(self.schema) + if not self.skip_copy: + ret = ret.as_builder().as_reader() + return ret except KjException as e: if 'EOF' in str(e): raise StopIteration @@ -3543,14 +3554,16 @@ cdef class _MultipleMessageReader: cdef class _MultiplePackedMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef cbool skip_copy cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit + self.skip_copy = skip_copy self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) @@ -3562,7 +3575,10 @@ cdef class _MultiplePackedMessageReader: def __next__(self): try: reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root(self.schema) + ret = reader.get_root(self.schema) + if not self.skip_copy: + ret = ret.as_builder().as_reader() + return ret except KjException as e: if 'EOF' in str(e): raise StopIteration From b7a9d05e5e3478ff9193347c8d1cbd84c2771aba Mon Sep 17 00:00:00 2001 From: Benjamin Piwowarski Date: Fri, 5 Feb 2016 17:15:26 +0100 Subject: [PATCH 023/126] Support mmap objects for reading with from_bytes --- capnp/lib/capnp.pyx | 34 ++++++++++++++++++++++++++++------ test/test_serialization.py | 15 +++++++++++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index d5c6d1d..8c531a8 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -29,6 +29,7 @@ import threading as _threading import socket as _socket import random as _random import collections as _collections +import mmap as _mmap _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR @@ -277,6 +278,8 @@ ctypedef fused PromiseTypes: cdef extern from "Python.h": cdef int PyObject_AsReadBuffer(object, void** b, Py_ssize_t* c) cdef int PyObject_AsWriteBuffer(object, void** b, Py_ssize_t* c) + cdef int PyObject_GetBuffer(object, Py_buffer *view, int flags) + cdef void PyBuffer_Release(Py_buffer *view) # Templated classes are weird in cython. I couldn't put it in a pxd header for some reason cdef extern from "capnp/list.h" namespace " ::capnp": @@ -3682,6 +3685,19 @@ cdef class _AlignedBuffer: if self.allocated: free(self.buf) + +@cython.internal +cdef class _BufferView: + cdef Py_buffer view + cdef char * buf + + def __init__(self, other): + PyObject_GetBuffer(other, &self.view, 0) + self.buf = self.view.buf + + def __dealloc__(self): + PyBuffer_Release(&self.view) + @cython.internal cdef class _FlatArrayMessageReader(_MessageReader): cdef object _object_to_pin @@ -3698,13 +3714,19 @@ cdef class _FlatArrayMessageReader(_MessageReader): if sz % 8 != 0: raise ValueError("input length must be a multiple of eight bytes") - cdef char * ptr = buf - if (ptr) % 8 != 0: - aligned = _AlignedBuffer(buf) - ptr = aligned.buf - self._object_to_pin = aligned + cdef char * ptr + if type(buf) == _mmap.mmap: + view = _BufferView(buf) + ptr = view.view.buf + self._object_to_pin = view else: - self._object_to_pin = buf + ptr = buf + if (ptr) % 8 != 0: + aligned = _AlignedBuffer(buf) + ptr = aligned.buf + self._object_to_pin = aligned + else: + self._object_to_pin = buf self.thisptr = new schema_cpp.FlatArrayMessageReader(schema_cpp.WordArrayPtr(ptr, sz//8)) diff --git a/test/test_serialization.py b/test/test_serialization.py index a1893e7..87c77cf 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -5,6 +5,7 @@ import platform import test_regression import tempfile import pickle +import mmap this_dir = os.path.dirname(__file__) @@ -40,6 +41,20 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) +def test_roundtrip_bytes_mmap(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + with tempfile.TemporaryFile() as f: + msg.write(f) + length = f.tell() + + f.seek(0) + memory = mmap.mmap(f.fileno(), length) + + msg = all_types.TestAllTypes.from_bytes(memory) + test_regression.check_all_types(msg) + def test_roundtrip_bytes_packed(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From 5d69eda752b54ceeb06819fac3dc5f7b692c8d4e Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 17 Feb 2016 17:23:04 -0800 Subject: [PATCH 024/126] Add --libcapnp-url to allow installing arbitrary libcapnp versions --- README.md | 5 +++++ buildutils/bundle.py | 20 ++++++++++++++++---- setup.py | 10 +++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d4e9f9c..7daf998 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,11 @@ Note: for OSX, if using clang from Xcode 5, you may need to set `CFLAGS` like so CFLAGS='-stdlib=libc++' pip install pycapnp +If you wish to install using the latest upstream C++ Cap'n Proto: + + pip install --install-option "--libcapnp-url" --install-option "https://github.com/sandstorm-io/capnproto/archive/master.tar.gz" --install-option "--force-bundled-libcapnp" . + + ## Python Versions Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 9271036..7a6b446 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -73,19 +73,32 @@ def fetch_archive(savedir, url, fname, force=False): # libcapnp #----------------------------------------------------------------------------- -def fetch_libcapnp(savedir): +def fetch_libcapnp(savedir, url=None): """download and extract libcapnp""" + is_preconfigured = False + if url is None: + url = libcapnp_url + is_preconfigured = True dest = pjoin(savedir, 'capnproto-c++') if os.path.exists(dest): info("already have %s" % dest) return - fname = fetch_archive(savedir, libcapnp_url, libcapnp) + fname = fetch_archive(savedir, url, libcapnp) tf = tarfile.open(fname) with_version = pjoin(savedir, tf.firstmember.path) tf.extractall(savedir) tf.close() # remove version suffix: - shutil.move(with_version, dest) + if is_preconfigured: + shutil.move(with_version, dest) + else: + cpp_dir = os.path.join(with_version, 'c++') + conf = Popen(['autoreconf', '-i'], cwd=cpp_dir) + returncode = conf.wait() + if returncode != 0: + raise RuntimeError('Autoreconf failed. Make sure autotools are installed on your system.') + shutil.move(cpp_dir, dest) + def stage_platform_hpp(capnproot): """stage platform.hpp into libcapnp sources @@ -162,4 +175,3 @@ def copy_and_patch_libcapnp(capnp, libcapnp): out,err = p.communicate() if p.returncode: fatal("Could not patch bundled libcapnp install_name: %s"%err, p.returncode) - diff --git a/setup.py b/setup.py index 512a464..6211e04 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,14 @@ force_cython = "--force-cython" in sys.argv if force_cython: sys.argv.remove("--force-cython") use_cython = True +libcapnp_url = None +try: + libcapnp_url_index = sys.argv.index("--libcapnp-url") + libcapnp_url = sys.argv[libcapnp_url_index + 1] + sys.argv.remove("--libcapnp-url") + sys.argv.remove(libcapnp_url) +except: + pass if use_cython: from Cython.Distutils import build_ext as build_ext_c @@ -107,7 +115,7 @@ class build_libcapnp_ext(build_ext_c): build_dir = os.path.join(_this_dir, "build") if not os.path.exists(build_dir): os.mkdir(build_dir) - fetch_libcapnp(bundle_dir) + fetch_libcapnp(bundle_dir, libcapnp_url) build_libcapnp(bundle_dir, build_dir) From aabd61301c46419827ef6c47b1b1a0ec5ef7acd1 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Mar 2016 11:04:58 -0800 Subject: [PATCH 025/126] Fix travis builds Stop running `make check` for capnproto. It has some networking test problem with travis. --- buildutils/setup_travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh index e25ae80..7a89a3e 100755 --- a/buildutils/setup_travis.sh +++ b/buildutils/setup_travis.sh @@ -11,5 +11,5 @@ sudo update-alternatives --quiet --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 if ! [ -z "${BUILD_CAPNP}" ]; then - wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz && tar xzvf capnproto-c++-${CAPNP_VERSION}.tar.gz && cd capnproto-c++-${CAPNP_VERSION} && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. + wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz && tar xzvf capnproto-c++-${CAPNP_VERSION}.tar.gz && cd capnproto-c++-${CAPNP_VERSION} && ./configure && make -j6 && sudo make install && sudo ldconfig && cd .. fi From 2be308db2733fad08d48f87a835c38d3ea5288ec Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Mar 2016 12:42:45 -0800 Subject: [PATCH 026/126] Throw an exception on invalid BufferView instantiation Also adjust tests to skip mmap under Python 2. --- capnp/lib/capnp.pyx | 8 ++++++-- test/test_serialization.py | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 8c531a8..290c4cf 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -15,6 +15,8 @@ from libc.stdlib cimport malloc, free from libc.string cimport memcpy from cython.operator cimport dereference as deref from cpython.exc cimport PyErr_Clear +from cpython cimport Py_buffer +from cpython.buffer cimport PyBUF_SIMPLE from types import ModuleType as _ModuleType import os as _os @@ -3692,7 +3694,9 @@ cdef class _BufferView: cdef char * buf def __init__(self, other): - PyObject_GetBuffer(other, &self.view, 0) + cdef int ret = PyObject_GetBuffer(other, &self.view, PyBUF_SIMPLE) + if ret < 0: + raise ValueError("Invalid buffer passed to BufferView") self.buf = self.view.buf def __dealloc__(self): @@ -3717,7 +3721,7 @@ cdef class _FlatArrayMessageReader(_MessageReader): cdef char * ptr if type(buf) == _mmap.mmap: view = _BufferView(buf) - ptr = view.view.buf + ptr = view.buf self._object_to_pin = view else: ptr = buf diff --git a/test/test_serialization.py b/test/test_serialization.py index 87c77cf..00c08cc 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -6,6 +6,7 @@ import test_regression import tempfile import pickle import mmap +import sys this_dir = os.path.dirname(__file__) @@ -41,6 +42,7 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) +@pytest.mark.skipif(sys.version_info.major < 3, reason="mmap doesn't implement the buffer interface under python 2.") def test_roundtrip_bytes_mmap(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From c625981eb67649dbfbcb99da6dd3ed32eda62826 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Mar 2016 13:12:01 -0800 Subject: [PATCH 027/126] Check version info the old way because of python 2.6 --- test/test_serialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_serialization.py b/test/test_serialization.py index 00c08cc..47c45e8 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -42,7 +42,7 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) -@pytest.mark.skipif(sys.version_info.major < 3, reason="mmap doesn't implement the buffer interface under python 2.") +@pytest.mark.skipif(sys.version_info[0] < 3, reason="mmap doesn't implement the buffer interface under python 2.") def test_roundtrip_bytes_mmap(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From 8c3c57aa325758fc44916a2dab93c9c52802369f Mon Sep 17 00:00:00 2001 From: Mike Lundy Date: Thu, 3 Mar 2016 14:29:32 -0800 Subject: [PATCH 028/126] Make sure to encode to utf-8, not the default encoding This allows text fields to take unicode strings under python 2. --- capnp/lib/capnp.pyx | 4 ++-- capnp/templates/module.pyx | 4 ++-- test/all-types.binary | Bin 2816 -> 2816 bytes test/all-types.packed | Bin 831 -> 831 bytes test/all-types.txt | 2 +- test/test_regression.py | 21 +++++++++++++++++++-- 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 290c4cf..55b3964 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -679,7 +679,7 @@ cdef _setBytes(_DynamicSetterClasses thisptr, field, value): thisptr.set(field, temp) cdef _setBaseString(_DynamicSetterClasses thisptr, field, value): - encoded_value = value.encode() + encoded_value = value.encode('utf-8') cdef capnp.StringPtr temp_string = capnp.StringPtr(encoded_value, len(encoded_value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.set(field, temp) @@ -690,7 +690,7 @@ cdef _setBytesField(DynamicStruct_Builder thisptr, _StructSchemaField field, val thisptr.setByField(field.thisptr, temp) cdef _setBaseStringField(DynamicStruct_Builder thisptr, _StructSchemaField field, value): - encoded_value = value.encode() + encoded_value = value.encode('utf-8') cdef capnp.StringPtr temp_string = capnp.StringPtr(encoded_value, len(encoded_value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.setByField(field.thisptr, temp) diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index df45582..9e51a7e 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -90,7 +90,7 @@ cpdef _set_{{field.name}}(self, value): if type(value) is bytes: temp_string = StringPtr(value, len(value)) else: - encoded_value = value.encode() + encoded_value = value.encode('utf-8') temp_string = StringPtr(encoded_value, len(encoded_value)) self.thisptr_child.set{{field.c_name}}(temp_string) {% elif 'data' == field['type'] -%} @@ -99,7 +99,7 @@ cpdef _set_{{field.name}}(self, value): if type(value) is bytes: temp_string = StringPtr(value, len(value)) else: - encoded_value = value.encode() + encoded_value = value.encode('utf-8') temp_string = StringPtr(encoded_value, len(encoded_value)) self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](temp_string.begin(), temp_string.size())) {% else -%} diff --git a/test/all-types.binary b/test/all-types.binary index ea39763774b2ed570407a3384a8865fbeaa79213..3381caad76714027a9f0768938dc79cd0314d8d4 100644 GIT binary patch delta 26 fcmZn=YY^LTfRX#rjAjN9C@ig*tjJ`#h=~&ba}Ecb delta 26 fcmZn=YY^LTfRQ^Xv5El%3QH>{D>9ibV&Vh Date: Thu, 3 Mar 2016 16:32:01 -0800 Subject: [PATCH 029/126] Include the traceback in exceptions This makes debugging into the cython side of the code much easier. --- capnp/lib/capnp.pyx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 55b3964..ab65392 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1025,7 +1025,7 @@ cdef class _DynamicStructReader: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _get_by_field(self, _StructSchemaField field): return to_python_reader(self.thisptr.getByField(field.thisptr), self._parent) @@ -1225,7 +1225,7 @@ cdef class _DynamicStructBuilder: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _set(self, field, value): _setDynamicField(self.thisptr, field, value, self._parent) @@ -1237,7 +1237,7 @@ cdef class _DynamicStructBuilder: try: self._set(field, value) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _has(self, field): return self.thisptr.has(field) @@ -1456,7 +1456,7 @@ cdef class _DynamicStructPipeline: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] property schema: """A property that returns the _StructSchema object matching this reader""" @@ -1936,7 +1936,7 @@ cdef class _RemotePromise: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] property schema: """A property that returns the _StructSchema object matching this reader""" @@ -2039,7 +2039,7 @@ cdef class _DynamicCapabilityServer: try: return getattr(self.server, field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cdef class _DynamicCapabilityClient: cdef C_DynamicCapability.Client thisptr @@ -2121,7 +2121,7 @@ cdef class _DynamicCapabilityClient: raise AttributeError('Method named %s not found' % name) return _partial(self._send, name) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef upcast(self, schema) except +reraise_kj_exception: cdef _InterfaceSchema s From 7164f04a7cf98ebe0f7ffb3814fe42f19f18ff76 Mon Sep 17 00:00:00 2001 From: Mike Lundy Date: Thu, 3 Mar 2016 15:03:33 -0800 Subject: [PATCH 030/126] Eliminate outdated function --- capnp/templates/module.pyx | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index 9e51a7e..943e802 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -133,40 +133,6 @@ cdef _from_list(_DynamicListBuilder msg, list d): msg._set(count, val) count += 1 -cdef DynamicValue.Reader to_dynamic_value(value): - cdef DynamicValue.Reader temp - cdef StringPtr temp_string - value_type = type(value) - - if value_type is int or value_type is long: - if value < 0: - temp = DynamicValue.Reader(value) - else: - temp = DynamicValue.Reader(value) - elif value_type is float: - temp = DynamicValue.Reader(value) - elif value_type is bool: - temp = DynamicValue.Reader(value) - elif value_type is bytes: - temp_string = StringPtr(value, len(value)) - temp = DynamicValue.Reader(temp_string) - elif isinstance(value, basestring): - encoded_value = value.encode() - temp_string = StringPtr(encoded_value, len(encoded_value)) - temp = DynamicValue.Reader(temp_string) - elif value is None: - temp = DynamicValue.Reader(VOID) - elif value_type is _DynamicStructBuilder: - temp = _extract_dynamic_struct_builder(value) - elif value_type is _DynamicStructReader: - temp = _extract_dynamic_struct_reader(value) - elif value_type is _DynamicEnum: - temp = _extract_dynamic_enum(value) - else: - raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value)))) - - return temp - cdef extern from "{{file.filename}}.h": {%- for node in code.nodes %} From 31ecf0a95e1ccb9f387e6760062657338bf0ba1f Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 4 Mar 2016 13:13:22 -0800 Subject: [PATCH 031/126] Include the changelog in the manifest Fixes #91 --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index fc2d9af..1ea2323 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.md +include CHANGELOG.md include requirements.txt include buildutils/* From b45152dba2756bacc3b5686d6eabf4b87ee98f00 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 8 Mar 2016 11:29:38 -0800 Subject: [PATCH 032/126] Skip failing PyPy test due to travis's outdated version --- test/test_serialization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index 47c45e8..d1b840c 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -57,6 +57,7 @@ def test_roundtrip_bytes_mmap(all_types): msg = all_types.TestAllTypes.from_bytes(memory) test_regression.check_all_types(msg) +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test.") def test_roundtrip_bytes_packed(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From 589400f53a0c471668e3bf7cdb4e797154b559b1 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 27 May 2016 13:52:46 -0700 Subject: [PATCH 033/126] Fix build problem with Cython v0.24 Fixes #97 --- capnp/includes/schema_cpp.pxd | 5 +++++ capnp/lib/capnp.pyx | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index e615017..3716648 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -628,6 +628,11 @@ cdef extern from "capnp/schema.capnp.h" namespace " ::capnp::schema": void setId(UInt64) Value getValue() void setValue(Value) + cdef cppclass ListNestedNodeReader"capnp::List::Reader": + ListNestedNodeReader() + ListNestedNodeReader(ListNestedNodeReader) + Node.NestedNode.Reader operator[](uint) + uint size() cdef extern from "capnp/message.h" namespace " ::capnp": cdef cppclass ReaderOptions: diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index ab65392..6d1eb8c 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -559,9 +559,9 @@ cdef class _DynamicListBuilder: return '' % strListBuilder(self.thisptr).cStr() cdef class _List_NestedNode_Reader: - cdef List[C_Node.NestedNode].Reader thisptr + cdef C_Node.NestedNode.Reader.ListNestedNodeReader thisptr cdef _init(self, List[C_Node.NestedNode].Reader other): - self.thisptr = other + self.thisptr = other return self def __getitem__(self, index): From 239286d4ac7a6b14694c6e74580838709ea5de51 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 27 May 2016 17:21:24 -0700 Subject: [PATCH 034/126] Bump version to v0.5.8 and update CHANGELOG --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad13689..59936c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## v0.5.8 (2016-05-27) +- Fix build problem with Cython v0.24 +- Include the changelog in the manifest (should fix install problems if pandoc is present) +- Include the traceback in exceptions +- Make sure to encode to utf-8, not the default encoding (thanks to @novas0x2a) +- Add --libcapnp-url option in installer to allow installing arbitrary libcapnp versions +- Support mmap objects for reading with from_bytes (thanks to @bpiwowar) +- Change read_multiple and read_multiple_packed to copy by default +- Fix mistakenly discarding the file parameter on reads +- Add reraise_kj_exception to the prettyPrint functions. (thanks to @kdienes) +- Fix KjException init (missing wrapper). (thanks to @E8-Storage) +- Add `result_type` to InterfaceMethodSchema + + ## v0.5.7 (2015-06-16) - Update bundled libcapnp to v0.5.2 - Add warnings for using old restorer methods. You should use `bootstrap` instead diff --git a/setup.py b/setup.py index 6211e04..de30104 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 7 +MICRO = 8 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From e4c128de611a0402fef96a693ae52e0a60225475 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 17 Jun 2016 13:50:26 -0700 Subject: [PATCH 035/126] Test large reads --- test/test_large_read.capnp | 13 ++++++++++++ test/test_large_read.py | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 test/test_large_read.capnp create mode 100644 test/test_large_read.py diff --git a/test/test_large_read.capnp b/test/test_large_read.capnp new file mode 100644 index 0000000..45675ea --- /dev/null +++ b/test/test_large_read.capnp @@ -0,0 +1,13 @@ +@0x86dbb3b256f5d2af; + +struct Row { + values @0 :List(Int32); +} + +struct MultiArray { + rows @0 :List(Row); +} + +struct Msg { + data @0 :List(UInt8); +} diff --git a/test/test_large_read.py b/test/test_large_read.py new file mode 100644 index 0000000..c925d1f --- /dev/null +++ b/test/test_large_read.py @@ -0,0 +1,42 @@ +import pytest +import capnp +import os +import tempfile +import sys + +this_dir = os.path.dirname(__file__) + + +@pytest.fixture +def test_capnp(): + return capnp.load(os.path.join(this_dir, 'test_large_read.capnp')) + + +def test_large_read(test_capnp): + f = tempfile.TemporaryFile() + + array = test_capnp.MultiArray.new_message() + + row = array.init('rows', 1)[0] + values = row.init('values', 10000) + for i in range(len(values)): + values[i] = i + + array.write_packed(f) + f.seek(0) + + array = test_capnp.MultiArray.read_packed(f) + del f + assert array.rows[0].values[9000] == 9000 + +def test_large_read_multiple(test_capnp): + f = tempfile.TemporaryFile() + msg1 = test_capnp.Msg.new_message() + msg1.data = [0x41] * 8192 + msg1.write(f) + msg2 = test_capnp.Msg.new_message() + msg2.write(f) + f.seek(0) + + for m in test_capnp.Msg.read_multiple(f): + pass From 5ffe7eb2f6cf7539f03be715613359efbf053afb Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 17 Jun 2016 13:50:42 -0700 Subject: [PATCH 036/126] Fix response objects not referencing parents correctly Fixes #103 --- capnp/lib/capnp.pyx | 4 ++-- test/test_response.capnp | 13 +++++++++++++ test/test_response.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 test/test_response.capnp create mode 100644 test/test_response.py diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 6d1eb8c..a052146 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1019,7 +1019,7 @@ cdef class _DynamicStructReader: return self cpdef _get(self, field): - return to_python_reader(self.thisptr.get(field), self._parent) + return to_python_reader(self.thisptr.get(field), self) def __getattr__(self, field): try: @@ -1028,7 +1028,7 @@ cdef class _DynamicStructReader: raise e._to_python(), None, _sys.exc_info()[2] cpdef _get_by_field(self, _StructSchemaField field): - return to_python_reader(self.thisptr.getByField(field.thisptr), self._parent) + return to_python_reader(self.thisptr.getByField(field.thisptr), self) cpdef _has(self, field): return self.thisptr.has(field) diff --git a/test/test_response.capnp b/test/test_response.capnp new file mode 100644 index 0000000..268bc08 --- /dev/null +++ b/test/test_response.capnp @@ -0,0 +1,13 @@ +@0x84249be5c3bff005; + +interface Foo { + foo @0 () -> (val :UInt32); +} + +struct Bar { + foo @0 :Foo; +} + +interface Baz { + grault @0 () -> (bar: Bar); +} diff --git a/test/test_response.py b/test/test_response.py new file mode 100644 index 0000000..296031c --- /dev/null +++ b/test/test_response.py @@ -0,0 +1,40 @@ +import pytest +import capnp +import os +import time + +import test_response_capnp + +class FooServer(test_response_capnp.Foo.Server): + def __init__(self, val=1): + self.val = val + + def foo(self, **kwargs): + return 1 + +class BazServer(test_response_capnp.Baz.Server): + def __init__(self, val=1): + self.val = val + + def grault(self, **kwargs): + return {"foo": FooServer()} + +def test_response_reference(): + baz = test_response_capnp.Baz._new_client(BazServer()) + + bar = baz.grault().wait().bar + + foo = bar.foo + # This used to cause an exception about invalid pointers because the response got garbage collected + foo.foo().wait() + +def test_response_reference2(): + baz = test_response_capnp.Baz._new_client(BazServer()) + + bar = baz.grault().wait().bar + + # This always worked since it saved the intermediate response object + response = baz.grault().wait() + bar = response.bar + foo = bar.foo + foo.foo().wait() From 032713f8b9120afc31642469560f38a49e539bf2 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 17 Jun 2016 13:53:48 -0700 Subject: [PATCH 037/126] Add asserts to test_response.py --- test/test_response.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_response.py b/test/test_response.py index 296031c..c9bff3d 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -26,7 +26,7 @@ def test_response_reference(): foo = bar.foo # This used to cause an exception about invalid pointers because the response got garbage collected - foo.foo().wait() + assert foo.foo().wait().val == 1 def test_response_reference2(): baz = test_response_capnp.Baz._new_client(BazServer()) @@ -37,4 +37,4 @@ def test_response_reference2(): response = baz.grault().wait() bar = response.bar foo = bar.foo - foo.foo().wait() + assert foo.foo().wait().val == 1 From 09705b6d1f85a10b22ecf9dc0d0788323f65ab8b Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Thu, 23 Jun 2016 14:15:31 -0700 Subject: [PATCH 038/126] Add support for segment (de)serialization. --- capnp/includes/schema_cpp.pxd | 16 +++++++++++ capnp/lib/capnp.pxd | 1 + capnp/lib/capnp.pyx | 51 +++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index 3716648..d9b691c 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -667,6 +667,8 @@ cdef extern from "capnp/message.h" namespace " ::capnp": DynamicStruct_Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema) void setRootDynamicStruct'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader) + ConstWordArrayArrayPtr getSegmentsForOutput'getSegmentsForOutput'() + AnyPointer.Builder getRootAnyPointer'getRoot< ::capnp::AnyPointer>'() DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema) @@ -691,6 +693,10 @@ cdef extern from "capnp/message.h" namespace " ::capnp": MallocMessageBuilder() MallocMessageBuilder(int) + cdef cppclass SegmentArrayMessageReader(MessageReader): + SegmentArrayMessageReader(ConstWordArrayArrayPtr array) except +reraise_kj_exception + SegmentArrayMessageReader(ConstWordArrayArrayPtr array, ReaderOptions) except +reraise_kj_exception + cdef cppclass FlatMessageBuilder(MessageBuilder): FlatMessageBuilder(WordArrayPtr array) FlatMessageBuilder(WordArrayPtr array, ReaderOptions) @@ -714,6 +720,16 @@ cdef extern from "kj/common.h" namespace " ::kj": ByteArrayPtr(byte *, size_t size) size_t size() byte& operator[](size_t index) + cdef cppclass ConstWordArrayPtr " ::kj::ArrayPtr< const ::capnp::word>": + ConstWordArrayPtr() + ConstWordArrayPtr(word *, size_t size) + size_t size() + const word* begin() + cdef cppclass ConstWordArrayArrayPtr " ::kj::ArrayPtr< const ::kj::ArrayPtr< const ::capnp::word>>": + ConstWordArrayArrayPtr() + ConstWordArrayArrayPtr(ConstWordArrayPtr*, size_t size) + size_t size() + ConstWordArrayPtr& operator[](size_t index) cdef extern from "kj/array.h" namespace " ::kj": # Cython can't handle Array[word] as a function argument diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 1673365..6ac296a 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -53,6 +53,7 @@ cdef class _DynamicStructBuilder: cdef _check_write(self) cpdef to_bytes(_DynamicStructBuilder self) except +reraise_kj_exception + cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception cpdef to_bytes_packed(_DynamicStructBuilder self) except +reraise_kj_exception diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index a052146..8ee1c3d 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1192,6 +1192,12 @@ cdef class _DynamicStructBuilder: self._is_written = True return ret + cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception: + self._check_write() + cdef _MessageBuilder builder = self._parent + segments = builder.get_segments_for_output() + return segments + cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception: cdef _MessageBuilder builder = self._parent array = helpers.messageToPackedBytes(deref(builder.thisptr), word_count) @@ -2989,6 +2995,9 @@ class _StructModule(object): else: message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) + def from_segments(self, segments): + message = _SegmentArrayMessageReader(segments) + return message.get_root(self.schema) def from_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object in buf. @@ -3285,6 +3294,18 @@ cdef class _MessageBuilder: self.thisptr.setRootDynamicStruct((<_DynamicStructReader>value).thisptr) return self.get_root(value.schema) + cpdef get_segments_for_output(self) except +reraise_kj_exception: + segments = self.thisptr.getSegmentsForOutput() + res = [] + cdef const char* ptr + cdef bytes segment_bytes + for i in range(0, segments.size()): + segment = segments[i] + ptr = segment.begin() + segment_bytes = ptr[:8*segment.size()] + res.append(segment_bytes) + return res + cpdef new_orphan(self, schema) except +reraise_kj_exception: """A method for instantiating Cap'n Proto orphans @@ -3738,6 +3759,36 @@ cdef class _FlatArrayMessageReader(_MessageReader): del self.thisptr +@cython.internal +cdef class _SegmentArrayMessageReader(_MessageReader): + + cdef object _objects_to_pin + + def __init__(self, segments): + # takes a Python array of bytes and constructs a ConstWordArrayArrayPtr + num_segments = len(segments) + cdef char* ptr + cdef schema_cpp.ConstWordArrayPtr seg_ptr + cdef schema_cpp.ConstWordArrayPtr* seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) + self._objects_to_pin = [] + for i in range(0, num_segments): + segment = bytes(segments[i]) + ptr = segment + if (ptr) % 8 != 0: + aligned = _AlignedBuffer(segment) + ptr = aligned.buf + self._objects_to_pin.append(aligned) + else: + self._objects_to_pin.append(segment) + seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, len(segment)//8) + seg_ptrs[i] = seg_ptr + self.thisptr = new schema_cpp.SegmentArrayMessageReader( + schema_cpp.ConstWordArrayArrayPtr(seg_ptrs, num_segments)) + + def __dealloc__(self): + del self.thisptr + + @cython.internal cdef class _FlatMessageBuilder(_MessageBuilder): cdef object _object_to_pin From 73d6c5dc107c87d5b2b26460fc4486a5789f82b8 Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Thu, 23 Jun 2016 14:15:39 -0700 Subject: [PATCH 039/126] Add segment (de)serialization round-trip test. --- test/test_serialization.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index d1b840c..5694088 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -42,6 +42,13 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) +def test_roundtrip_segments(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + segments = msg.to_segments() + msg = all_types.TestAllTypes.from_segments(segments) + test_regression.check_all_types(msg) + @pytest.mark.skipif(sys.version_info[0] < 3, reason="mmap doesn't implement the buffer interface under python 2.") def test_roundtrip_bytes_mmap(all_types): msg = all_types.TestAllTypes.new_message() From 42ed1e819fbd28f296e4f8b8a15e1c781c7ed8fb Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Thu, 23 Jun 2016 14:15:46 -0700 Subject: [PATCH 040/126] Update documentation to cover segments. --- docs/quickstart.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index aa89b89..b19ad55 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -304,6 +304,27 @@ There are also packed versions:: alice2 = addressbook_capnp.Person.from_bytes_packed(alice.to_bytes_packed()) + +Byte Segments +~~~~~~~~~~~~~ + +Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way). + +For compatibility on the Python side, use the ``to_segments()`` and ``from_segments()`` functions:: + + segments = alice.to_segments() + +This returns a list of segments, each a byte buffer. Each segment can be, e.g., turned into a ZeroMQ message frame. The list of segments can also be turned back into an object:: + + alice = addressbook_capnp.Person.from_segments(segments) + +For more information, please refer to the following links: + +- `Advice on minimizing copies from Cap'n Proto `_ (from the author of Cap'n Proto) +- `Advice on using Cap'n Proto over ZeroMQ `_ (from the author of Cap'n Proto) +- `Discussion about sending and reassembling Cap'n Proto message segments in C++ `_ (from the Cap'n Proto mailing list; includes sample code) + + RPC ---------- From 70e4e2d930a594733b148b901a8576635dc0c330 Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 13:25:27 -0700 Subject: [PATCH 041/126] Add ReaderOptions support to segment (de)serialization. --- capnp/lib/capnp.pyx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 8ee1c3d..bc8c99b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2995,8 +2995,8 @@ class _StructModule(object): else: message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) - def from_segments(self, segments): - message = _SegmentArrayMessageReader(segments) + def from_segments(self, segments, traversal_limit_in_words = None, nesting_limit = None): + message = _SegmentArrayMessageReader(segments, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) def from_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object in buf. @@ -3764,8 +3764,13 @@ cdef class _SegmentArrayMessageReader(_MessageReader): cdef object _objects_to_pin - def __init__(self, segments): - # takes a Python array of bytes and constructs a ConstWordArrayArrayPtr + def __init__(self, segments, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts + if traversal_limit_in_words is not None: + opts.traversalLimitInWords = traversal_limit_in_words + if nesting_limit is not None: + opts.nestingLimit = nesting_limit + # take a Python array of bytes and constructs a ConstWordArrayArrayPtr num_segments = len(segments) cdef char* ptr cdef schema_cpp.ConstWordArrayPtr seg_ptr @@ -3783,7 +3788,8 @@ cdef class _SegmentArrayMessageReader(_MessageReader): seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, len(segment)//8) seg_ptrs[i] = seg_ptr self.thisptr = new schema_cpp.SegmentArrayMessageReader( - schema_cpp.ConstWordArrayArrayPtr(seg_ptrs, num_segments)) + schema_cpp.ConstWordArrayArrayPtr(seg_ptrs, num_segments), + opts) def __dealloc__(self): del self.thisptr From fb8a3d8ac2622ee13e10c502eb00b23219dfb4aa Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 13:27:21 -0700 Subject: [PATCH 042/126] Add docstrings to (de)serialization functions. --- capnp/lib/capnp.pyx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index bc8c99b..263515d 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1193,6 +1193,14 @@ cdef class _DynamicStructBuilder: return ret cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception: + """Returns the struct's containing message as a Python list of Python bytes objects. + + This avoids making copies. + + NB: This is not currently supported on PyPy. + + :rtype: list + """ self._check_write() cdef _MessageBuilder builder = self._parent segments = builder.get_segments_for_output() @@ -2996,6 +3004,14 @@ class _StructModule(object): message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) def from_segments(self, segments, traversal_limit_in_words = None, nesting_limit = None): + """Returns a Reader for a list of segment bytes. + + This avoids making copies. + + NB: This is not currently supported on PyPy. + + :rtype: list + """ message = _SegmentArrayMessageReader(segments, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) def from_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): From 5a63dacf146cafe8614596b79ae7b870bcd7fcef Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 13:27:39 -0700 Subject: [PATCH 043/126] Disable segment serialization test on PyPy (for now). --- test/test_serialization.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index 5694088..ebead7e 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -42,6 +42,7 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="TODO: Investigate why this works on CPython but fails on PyPy.") def test_roundtrip_segments(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From f703a7f099ed24c783a24fc889a41c8af27734f0 Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 13:27:54 -0700 Subject: [PATCH 044/126] Update segment (de)serialization documentation. --- docs/quickstart.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b19ad55..3b7a1a8 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -308,6 +308,8 @@ There are also packed versions:: Byte Segments ~~~~~~~~~~~~~ +.. note:: This feature is not supported in PyPy at the moment, pending investigation. + Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way). For compatibility on the Python side, use the ``to_segments()`` and ``from_segments()`` functions:: From e8a8d26260cba0fb69670bc474e8db68990259b7 Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 14:03:20 -0700 Subject: [PATCH 045/126] Fix memory leak. --- capnp/lib/capnp.pyx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 263515d..54344b2 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3779,6 +3779,7 @@ cdef class _FlatArrayMessageReader(_MessageReader): cdef class _SegmentArrayMessageReader(_MessageReader): cdef object _objects_to_pin + cdef schema_cpp.ConstWordArrayPtr* _seg_ptrs def __init__(self, segments, traversal_limit_in_words = None, nesting_limit = None): cdef schema_cpp.ReaderOptions opts @@ -3790,7 +3791,7 @@ cdef class _SegmentArrayMessageReader(_MessageReader): num_segments = len(segments) cdef char* ptr cdef schema_cpp.ConstWordArrayPtr seg_ptr - cdef schema_cpp.ConstWordArrayPtr* seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) + self._seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) self._objects_to_pin = [] for i in range(0, num_segments): segment = bytes(segments[i]) @@ -3802,12 +3803,13 @@ cdef class _SegmentArrayMessageReader(_MessageReader): else: self._objects_to_pin.append(segment) seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, len(segment)//8) - seg_ptrs[i] = seg_ptr + self._seg_ptrs[i] = seg_ptr self.thisptr = new schema_cpp.SegmentArrayMessageReader( - schema_cpp.ConstWordArrayArrayPtr(seg_ptrs, num_segments), + schema_cpp.ConstWordArrayArrayPtr(self._seg_ptrs, num_segments), opts) def __dealloc__(self): + free(self._seg_ptrs) del self.thisptr From 41c418aa1cbc2a7d130176ab08326f3f488238ac Mon Sep 17 00:00:00 2001 From: Constantine Vetoshev Date: Fri, 24 Jun 2016 16:43:57 -0700 Subject: [PATCH 046/126] Use PyObject_AsReadBuffer instead of bytes(). --- capnp/lib/capnp.pyx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 54344b2..5d44e8b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3789,20 +3789,20 @@ cdef class _SegmentArrayMessageReader(_MessageReader): opts.nestingLimit = nesting_limit # take a Python array of bytes and constructs a ConstWordArrayArrayPtr num_segments = len(segments) - cdef char* ptr + cdef const void* ptr + cdef Py_ssize_t segment_size cdef schema_cpp.ConstWordArrayPtr seg_ptr self._seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) self._objects_to_pin = [] for i in range(0, num_segments): - segment = bytes(segments[i]) - ptr = segment + PyObject_AsReadBuffer(segments[i], &ptr, &segment_size) if (ptr) % 8 != 0: - aligned = _AlignedBuffer(segment) + aligned = _AlignedBuffer(segments[i]) ptr = aligned.buf self._objects_to_pin.append(aligned) else: - self._objects_to_pin.append(segment) - seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, len(segment)//8) + self._objects_to_pin.append(segments[i]) + seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, segment_size//8) self._seg_ptrs[i] = seg_ptr self.thisptr = new schema_cpp.SegmentArrayMessageReader( schema_cpp.ConstWordArrayArrayPtr(self._seg_ptrs, num_segments), From da233c46ad7806efb2e4bc21b79d978b51515475 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 7 Jul 2016 12:11:42 -0700 Subject: [PATCH 047/126] Make the event loop be lazy initialized Fixes #101 --- capnp/lib/capnp.pyx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 5d44e8b..57d675c 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1638,6 +1638,7 @@ _THREAD_LOCAL_EVENT_LOOPS = [] cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): 'Optimization for not having to deal with threadlocal event loops unless we need to' + global C_DEFAULT_EVENT_LOOP if C_DEFAULT_EVENT_LOOP is not None: return C_DEFAULT_EVENT_LOOP elif _C_DEFAULT_EVENT_LOOP_LOCAL is not None: @@ -1647,8 +1648,9 @@ cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): else: _C_DEFAULT_EVENT_LOOP_LOCAL.loop = _EventLoop() return _C_DEFAULT_EVENT_LOOP_LOCAL.loop - - raise KjException("You don't have any EventLoops running. Please make sure to add one") + else: + C_DEFAULT_EVENT_LOOP = _EventLoop() + return C_DEFAULT_EVENT_LOOP cdef class _Timer: cdef capnp.Timer * thisptr From 3efb3f41f08748b23c1ed97d44f3c32724ddfa29 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 7 Jul 2016 12:15:14 -0700 Subject: [PATCH 048/126] Bump version to v0.5.9 and update CHANGELOG --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59936c9..f224895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.5.9 (2016-07-07) +- Make the event loop be lazy initialized +- Add support for segment (de)serialization (thanks to @gcv). See to_segments/from_segments methods. +- Fix response objects not referencing parents correctly +- Add test for large reads + ## v0.5.8 (2016-05-27) - Fix build problem with Cython v0.24 - Include the changelog in the manifest (should fix install problems if pandoc is present) diff --git a/setup.py b/setup.py index de30104..7881bb4 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 8 +MICRO = 9 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 4488194f59f8785da02336820cee5b389c552a2b Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 29 Jul 2016 11:57:05 -0700 Subject: [PATCH 049/126] Make event loop be lazy initialized for real --- capnp/lib/capnp.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 57d675c..915968a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1631,7 +1631,7 @@ cdef class _EventLoop: cdef Own[AsyncIoStream] wrapSocketFd(self, int fd): return deref(deref(self.thisptr).lowLevelProvider).wrapSocketFd(fd) -cdef _EventLoop C_DEFAULT_EVENT_LOOP = _EventLoop() +cdef _EventLoop C_DEFAULT_EVENT_LOOP _C_DEFAULT_EVENT_LOOP_LOCAL = None _THREAD_LOCAL_EVENT_LOOPS = [] From d85200cee34653b6df8b2c8484a6f541f66a8ddd Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 29 Jul 2016 11:59:02 -0700 Subject: [PATCH 050/126] Remove recursive loop in KjException.type Fixes #108 --- capnp/lib/capnp.pyx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 915968a..72b8244 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -205,7 +205,7 @@ class KjException(Exception): Type = _make_enum('Type', **{x : x for x in _Type.reverse_mapping.values()}) - def __init__(self, message=None, nature=None, durability=None, wrapper=None): + def __init__(self, message=None, nature=None, durability=None, wrapper=None, type=None): if wrapper is not None: self.wrapper = wrapper self.message = str(wrapper) @@ -214,6 +214,7 @@ class KjException(Exception): self.message = message self.nature = nature self.durability = durability + self._type = type @property def file(self): @@ -226,7 +227,7 @@ class KjException(Exception): if self.wrapper is not None: return self.wrapper.type else: - return self.type + return self._type @property def description(self): if self.wrapper is not None: From 97824b957b6afb8487e2ffe575af1d5431a02dc5 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 9 Aug 2016 12:26:18 -0700 Subject: [PATCH 051/126] Add `clear_write_flag` method to builders Fixes #111 --- capnp/lib/capnp.pxd | 2 +- capnp/lib/capnp.pyx | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 6ac296a..8cbf3b7 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -46,7 +46,7 @@ cdef class _DynamicStructBuilder: cdef DynamicStruct_Builder thisptr cdef public object _parent cdef public bint is_root - cdef bint _is_written + cdef public bint _is_written cdef object _schema cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 72b8244..f4ec511 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1140,7 +1140,7 @@ cdef class _DynamicStructBuilder: if not self.is_root: raise KjException("You can only call write() on the message's root struct.") if self._is_written: - _warnings.warn("This message has already been written once. Be very careful that you're not setting Text/Struct/List fields more than once, since that will cause memory leaks (both in memory and in the serialized data). You can disable this warning by setting the `_is_written` field of this object to False after every write.") + _warnings.warn("This message has already been written once. Be very careful that you're not setting Text/Struct/List fields more than once, since that will cause memory leaks (both in memory and in the serialized data). You can disable this warning by calling the `clear_write_flag` method of this object after every write.") def write(self, file): """Writes the struct's containing message to the given file object in unpacked binary format. @@ -1437,6 +1437,13 @@ cdef class _DynamicStructBuilder: size = self.thisptr.totalSize() return _MessageSize(size.wordCount, size.capCount) + def clear_write_flag(self): + """A method used to clear the _is_written flag. + + This allows you to write the struct more than once without seeing any warnings. + """ + self._is_written = False + def __reduce_ex__(self, proto): return _struct_reducer, (self.schema.node.id, self.to_bytes()) From 82a53013bd11cf8306c9e9a010cee269a076f93d Mon Sep 17 00:00:00 2001 From: Matt Mullins Date: Sat, 20 Aug 2016 15:00:05 -0700 Subject: [PATCH 052/126] customize_compiler is recommended by distutils doc On Unix platforms, environment variables such as CFLAGS, LDFLAGS are not honored if this is not called. --- buildutils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/misc.py b/buildutils/misc.py index 72d01b7..5721ac1 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -35,7 +35,7 @@ def get_compiler(compiler, **compiler_attrs): """get and customize a compiler""" if compiler is None or isinstance(compiler, str): cc = ccompiler.new_compiler(compiler=compiler) - # customize_compiler(cc) + customize_compiler(cc) if cc.compiler_type == 'mingw32': customize_mingw(cc) else: From 0a941a9e06d2a60ae569ed6b75e25468feeb6c04 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 31 Aug 2016 12:47:40 -0700 Subject: [PATCH 053/126] Switch to using setuptools --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7881bb4..75d40be 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from __future__ import print_function use_cython = False -from distutils.core import setup +from setuptools import setup import os import sys from buildutils import test_build, fetch_libcapnp, build_libcapnp, info From 468375d5667ffc3c51c9f6aed4bc083822a2f470 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 31 Aug 2016 12:48:05 -0700 Subject: [PATCH 054/126] Fix benchamrks --- benchmark/bin/run_all.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/benchmark/bin/run_all.py b/benchmark/bin/run_all.py index da6d9a3..60421a0 100755 --- a/benchmark/bin/run_all.py +++ b/benchmark/bin/run_all.py @@ -6,10 +6,13 @@ import sys import os import json import argparse +import time + +_this_dir = os.path.dirname(__file__) def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l capnproto -l protobuf", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp']) + parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l pyproto -l pyproto_cpp", action='append', default=['pycapnp']) parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true') parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true') parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0) @@ -26,26 +29,24 @@ def run_one(prefix, name, mode, iters, faster, compression): if compression != 'none': res_type += '_' + compression - command = ["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)] + command = [os.path.join(_this_dir, prefix+"-"+name), mode, reuse, compression, str(iters)] + start = time.time() + print('running: ' + ' '.join(command), file=sys.stderr) p = Popen(command, stdout=PIPE, stderr=PIPE) - res = p.communicate()[1] + res = p.wait() + end = time.time() data = {} if p.returncode != 0: - sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + res + '\n') + sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + p.stderr.read() + '\n') sys.stderr.flush() - else: - res = res.strip() - - for line in res.split('\n'): - vals = line.split() - data[vals[0]] = float(vals[1]) data['type'] = res_type data['mode'] = mode data['name'] = name data['iters'] = iters + data['time'] = end - start return data From 7ad51120e65536df984a72708da2812ac4e648a3 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 28 Nov 2016 13:38:50 -0800 Subject: [PATCH 055/126] Bump to v0.5.10 and update changelog --- CHANGELOG.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f224895..8a06354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.5.10 (2016-11-28) +- Fix bug that prevented event loop from actually being lazy initialized +- Fix possible recursive loop in KjException +- Add `clear_write_flag` method to builder classes + ## v0.5.9 (2016-07-07) - Make the event loop be lazy initialized - Add support for segment (de)serialization (thanks to @gcv). See to_segments/from_segments methods. diff --git a/setup.py b/setup.py index 75d40be..c720940 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 9 +MICRO = 10 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 93ea35e55829948f5654fd3c879453ec26070a7c Mon Sep 17 00:00:00 2001 From: Madeleine Thompson Date: Mon, 5 Dec 2016 22:27:40 -0500 Subject: [PATCH 056/126] make enums hashable --- capnp/lib/capnp.pyx | 4 ++++ test/test_regression.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index f4ec511..284bd7c 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -943,6 +943,10 @@ cdef class _DynamicEnum: elif op == 5: # >= return left >= right + def __hash__(_DynamicEnum self): + return hash(self._as_str()) + + cdef class _DynamicEnumField: cdef _init(self, proto): self.thisptr = proto diff --git a/test/test_regression.py b/test/test_regression.py index 78f3fec..93a8a26 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -415,6 +415,12 @@ def check_all_types(reader): assert subSubReader.structField.textField == "really nested" assert subReader.enumField == "baz" + # Check that enums are hashable and can be used as keys in dicts + # interchangably with their string version. + assert hash(subReader.enumField) == hash('baz') + assert {subReader.enumField: 17}.get(subReader.enumField) == 17 + assert {subReader.enumField: 17}.get('baz') == 17 + assert {'baz': 17}.get(subReader.enumField) == 17 check_list(subReader.voidList, [None, None, None]) check_list(subReader.boolList, [False, True, False, True, True]) From 2ee80532a06929af588705050ef4907d20cc4131 Mon Sep 17 00:00:00 2001 From: Ben Nizette Date: Mon, 9 Jan 2017 13:07:08 +1100 Subject: [PATCH 057/126] Rework logic to determine whether to use bundled or system libcapnp. The current logic unconditionally tries to build /and run/ a small executable linked against libcapnp. The 'run' step is ostensibly to get the current library version number, however this number isn't actually used anywhere. More importantly, this breaks cross-compiling completely as in that mode you can't run an executable that's just been built. One option would be to fix the build scripts for cross-compiling but this is relatively complex. This patch implements a simple change that basically boils down to '--force-system-libcapnp' actually forcing that, skipping the build and run tests. The assumption is that if you're cross-compiling then you're most likely to be building the library yourself anyway and if you're specifying that flag, then you know what you're doing. --- setup.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 75d40be..b4ffb7f 100644 --- a/setup.py +++ b/setup.py @@ -98,17 +98,21 @@ class build_libcapnp_ext(build_ext_c): build_ext_c.build_extension(self, ext) def run(self): - build_failed = False - try: - test_build() - except CompileError: - build_failed = True + if force_bundled_libcapnp: + need_build = True + elif force_system_libcapnp: + need_build = False + else: + # Try to autodetect presence of library. Requires compile/run + # step so only works for host (non-cross) compliation + try: + test_build() + need_build = False + except CompileError: + need_build = True - if build_failed and force_system_libcapnp: - raise RuntimeError("libcapnp C++ library not detected and --force-system-libcapnp was used") - if build_failed or force_bundled_libcapnp: - if build_failed: - info("*WARNING* no libcapnp detected. Will download and build it from source now. If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. Downloading and building libcapnp may take a while.") + if need_build: + info("*WARNING* no libcapnp detected or rebuild forced. Will download and build it from source now. If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. Downloading and building libcapnp may take a while.") bundle_dir = os.path.join(_this_dir, "bundled") if not os.path.exists(bundle_dir): os.mkdir(bundle_dir) From f2c7f61b7d4b419ca0edd3e79463db6c6c60ebea Mon Sep 17 00:00:00 2001 From: Florian Friesdorf Date: Thu, 19 Jan 2017 19:07:12 +0100 Subject: [PATCH 058/126] include class attributes in __dir__ This enables tab completion for methods as well as fieldnames. --- capnp/lib/capnp.pyx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 284bd7c..7397b1e 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1081,7 +1081,7 @@ cdef class _DynamicStructReader: return self._schema def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def __str__(self): return printStructReader(self.thisptr).flatten().cStr() @@ -1413,7 +1413,7 @@ cdef class _DynamicStructBuilder: return self._schema def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def __str__(self): return printStructBuilder(self.thisptr).flatten().cStr() @@ -1490,7 +1490,7 @@ cdef class _DynamicStructPipeline: return _StructSchema()._init(self.thisptr.getSchema()) def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) # def __str__(self): # return printStructReader(self.thisptr).flatten().cStr() @@ -1972,7 +1972,7 @@ cdef class _RemotePromise: return _StructSchema()._init(self.thisptr.getSchema()) def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def to_dict(self, verbose=False, ordered=False): return _to_dict(self, verbose, ordered) @@ -2176,7 +2176,7 @@ cdef class _DynamicCapabilityClient: return self._cached_schema def __dir__(self): - return list(self.schema.method_names_inherited) + return list(set(self.schema.method_names_inherited) + tuple(dir(self.__class__))) cdef class _CapabilityClient: cdef C_Capability.Client * thisptr From 0382beb26a02292d5e6bd721f0a6608dfa20830d Mon Sep 17 00:00:00 2001 From: Alex Silverstein Date: Sun, 22 Jan 2017 01:57:01 +0000 Subject: [PATCH 059/126] Let TwoParty Clients and Servers take ReaderOptions --- capnp/includes/capnp_cpp.pxd | 6 +-- capnp/lib/capnp.pyx | 76 ++++++++++++------------------------ test/test_rpc.py | 18 +++++++++ 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index 7816d20..e4f5675 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -4,7 +4,7 @@ cdef extern from "capnp/helpers/checkCompiler.h": pass -from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader +from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyRestorer, PyEventPort, ErrorHandler from capnp.includes.types cimport * @@ -45,7 +45,7 @@ cdef extern from "kj/exception.h" namespace " ::kj": cdef extern from "kj/memory.h" namespace " ::kj": cdef cppclass Own[T]: T& operator*() - Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side) + Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side, ReaderOptions) Own[PromiseFulfillerPair] copyPromiseFulfillerPair" ::kj::heap< ::kj::PromiseFulfillerPair >"(PromiseFulfillerPair&) Own[PyRefCounter] makePyRefCounter" ::kj::heap< PyRefCounter >"(PyObject *) @@ -341,7 +341,7 @@ cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp": cdef Side SERVER" ::capnp::rpc::twoparty::Side::SERVER" cdef cppclass TwoPartyVatNetwork: - TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side) + TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side, ReaderOptions) VoidPromise onDisconnect() VoidPromise onDrained() RpcSystem makeRpcServer(TwoPartyVatNetwork&, PyRestorer&) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 284bd7c..d73c15b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -264,6 +264,14 @@ cdef public object get_exception_info(object exc_type, object exc_obj, object ex except: return (b'', 0, b"Couldn't determine python exception") +cdef schema_cpp.ReaderOptions make_reader_opts(traversal_limit_in_words, nesting_limit) with gil: + cdef schema_cpp.ReaderOptions opts + if traversal_limit_in_words is not None: + opts.traversalLimitInWords = traversal_limit_in_words + if nesting_limit is not None: + opts.nestingLimit = nesting_limit + return opts + ctypedef fused _DynamicStructReaderOrBuilder: _DynamicStructReader _DynamicStructBuilder @@ -2217,9 +2225,9 @@ cdef class _TwoPartyVatNetwork: cdef Own[C_TwoPartyVatNetwork] thisptr cdef _AsyncIoStream stream - cdef _init(self, _AsyncIoStream stream, Side side): + cdef _init(self, _AsyncIoStream stream, Side side, schema_cpp.ReaderOptions opts): self.stream = stream - self.thisptr = makeTwoPartyVatNetwork(deref(stream.thisptr), side) + self.thisptr = makeTwoPartyVatNetwork(deref(stream.thisptr), side, opts) return self cpdef on_disconnect(self) except +reraise_kj_exception: @@ -2244,13 +2252,15 @@ cdef class TwoPartyClient: cdef public _Restorer _restorer cdef public _AsyncIoStream _stream - def __init__(self, socket, restorer=None): + def __init__(self, socket, restorer=None, traversal_limit_in_words=None, nesting_limit=None): if isinstance(socket, basestring): socket = self._connect(socket) + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) + self._orig_stream = socket self._stream = _FdAsyncIoStream(socket.fileno()) - self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT) + self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT, opts) if restorer is None: self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) self._restorer = None @@ -2340,11 +2350,13 @@ cdef class TwoPartyServer: cdef capnp.TaskSet * _task_set cdef capnp.ErrorHandler _error_handler - def __init__(self, socket, restorer=None, server_socket=None, bootstrap=None): + def __init__(self, socket, restorer=None, server_socket=None, bootstrap=None, + traversal_limit_in_words=None, nesting_limit=None): if not restorer and not bootstrap: raise KjException("You must provide either a bootstrap interface or a restorer (deperecated) to a server constructor.") cdef _InterfaceSchema schema + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._restorer = None self._bootstrap = None @@ -2355,7 +2367,7 @@ cdef class TwoPartyServer: self._stream = _FdAsyncIoStream(socket.fileno()) self._server_socket = server_socket self._port = 0 - self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER) + self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER, opts) if bootstrap: self._bootstrap = bootstrap @@ -3438,15 +3450,9 @@ cdef class _StreamFdMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = file - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.StreamFdMessageReader(file.fileno(), opts) def __dealloc__(self): @@ -3469,15 +3475,9 @@ cdef class _PackedMessageReader(_MessageReader): pass cdef _init(self, schema_cpp.BufferedInputStream & stream, traversal_limit_in_words = None, nesting_limit = None, parent = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = parent - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.PackedMessageReader(stream, opts) return self @@ -3489,15 +3489,10 @@ cdef class _PackedMessageReaderBytes(_MessageReader): cdef schema_cpp.ArrayInputStream * stream def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = buf - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - cdef const void *ptr cdef Py_ssize_t sz PyObject_AsReadBuffer(buf, &ptr, &sz) @@ -3526,15 +3521,9 @@ cdef class _InputMessageReader(_MessageReader): pass cdef _init(self, schema_cpp.BufferedInputStream & stream, traversal_limit_in_words = None, nesting_limit = None, parent = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = parent - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.InputStreamMessageReader(stream, opts) return self @@ -3555,15 +3544,9 @@ cdef class _PackedFdMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = file - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.PackedFdMessageReader(file.fileno(), opts) def __dealloc__(self): @@ -3757,14 +3740,9 @@ cdef class _BufferView: cdef class _FlatArrayMessageReader(_MessageReader): cdef object _object_to_pin def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) cdef _AlignedBuffer aligned - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - sz = len(buf) if sz % 8 != 0: raise ValueError("input length must be a multiple of eight bytes") @@ -3796,11 +3774,7 @@ cdef class _SegmentArrayMessageReader(_MessageReader): cdef schema_cpp.ConstWordArrayPtr* _seg_ptrs def __init__(self, segments, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) # take a Python array of bytes and constructs a ConstWordArrayArrayPtr num_segments = len(segments) cdef const void* ptr diff --git a/test/test_rpc.py b/test/test_rpc.py index d288d04..c51797e 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -43,6 +43,24 @@ def test_simple_rpc(): assert response.x == '125' +def test_simple_rpc_with_options(): + read, write = socket.socketpair(socket.AF_UNIX) + + restorer = SimpleRestorer() + server = capnp.TwoPartyServer(write, restorer) + # This traversal limit is too low to receive the response in, so we expect + # an exception during the call. + client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) + + ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') + cap = client.restore(ref) + cap = cap.cast_as(test_capability_capnp.TestInterface) + + remote = cap.foo(i=5) + with pytest.raises(capnp.KjException): + response = remote.wait() + + def test_simple_rpc_restore_func(): read, write = socket.socketpair(socket.AF_UNIX) From f47d21d1bbc2f523afea3c95f326b892586391c4 Mon Sep 17 00:00:00 2001 From: asilversempirical Date: Tue, 7 Feb 2017 08:51:31 -0500 Subject: [PATCH 060/126] Update binary package docs Fixes https://github.com/jparyani/pycapnp/issues/134 --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7daf998..e80586c 100644 --- a/README.md +++ b/README.md @@ -38,15 +38,13 @@ This project uses [git-flow](http://jeffkreeftmeijer.com/2010/why-arent-you-usin ### Binary Packages -In order to build binary packages from this source code, you must specify the `--disable-cython` option: - Building a dumb binary distribution: - python setup.py bdist_dumb --disable-cython + python setup.py bdist_dumb Building a Python wheel distributiion: - python setup.py bdist_wheel --disable-cython + python setup.py bdist_wheel If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with: From 1bfc20fefb4c2915e030939e25ff4d35113bba33 Mon Sep 17 00:00:00 2001 From: Florian Friesdorf Date: Thu, 9 Feb 2017 02:10:50 +0100 Subject: [PATCH 061/126] support initializing DynamicListBuilder from tuple --- capnp/lib/capnp.pxd | 1 + capnp/lib/capnp.pyx | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 8cbf3b7..dc0ed4e 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -122,5 +122,6 @@ cdef to_python_reader(C_DynamicValue.Reader self, object parent) cdef to_python_builder(C_DynamicValue.Builder self, object parent) cdef _to_dict(msg, bint verbose, bint ordered) cdef _from_list(_DynamicListBuilder msg, list d) +cdef _from_tuple(_DynamicListBuilder msg, tuple d) cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent) cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 284bd7c..53756ca 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -719,6 +719,9 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): elif value_type is list: builder = to_python_builder(thisptr.init(field, len(value)), parent) _from_list(builder, value) + elif value_type is tuple: + builder = to_python_builder(thisptr.init(field, len(value)), parent) + _from_tuple(builder, value) elif value_type is dict: if _DynamicSetterClasses is DynamicStruct_Builder: builder = to_python_builder(thisptr.get(field), parent) @@ -904,6 +907,11 @@ cdef _from_list(_DynamicListBuilder msg, list d): msg._set(i, x) +cdef _from_tuple(_DynamicListBuilder msg, tuple d): + for i, x in enumerate(d): + msg._set(i, x) + + cdef class _DynamicEnum: cdef _init(self, capnp.DynamicEnum other, object parent): self.thisptr = other From 34385afbbd525c104d9b56f299455f35c3741549 Mon Sep 17 00:00:00 2001 From: Christian Plesner Hansen Date: Wed, 15 Feb 2017 12:02:41 +0100 Subject: [PATCH 062/126] Pass options correctly through from_bytes to FlatArrayMessageReader --- capnp/lib/capnp.pyx | 4 +++- test/test_serialization.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 284bd7c..ae27c4b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3783,7 +3783,9 @@ cdef class _FlatArrayMessageReader(_MessageReader): else: self._object_to_pin = buf - self.thisptr = new schema_cpp.FlatArrayMessageReader(schema_cpp.WordArrayPtr(ptr, sz//8)) + self.thisptr = new schema_cpp.FlatArrayMessageReader( + schema_cpp.WordArrayPtr(ptr, sz//8), + opts) def __dealloc__(self): del self.thisptr diff --git a/test/test_serialization.py b/test/test_serialization.py index ebead7e..66a1d88 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -168,3 +168,26 @@ def test_pickle(all_types): msg2 = pickle.loads(data) test_regression.check_all_types(msg2) + +def test_from_bytes_traversal_limit(all_types): + size = 1024 + bld = all_types.TestAllTypes.new_message() + bld.init("structList", size) + data = bld.to_bytes() + + msg = all_types.TestAllTypes.from_bytes(data, + traversal_limit_in_words=2**62) + for i in range(0, size): + assert msg.structList[i].uInt8Field == 0 + + +def test_from_bytes_packed_traversal_limit(all_types): + size = 1024 + bld = all_types.TestAllTypes.new_message() + bld.init("structList", size) + data = bld.to_bytes_packed() + + msg = all_types.TestAllTypes.from_bytes_packed(data, + traversal_limit_in_words=2**62) + for i in range(0, size): + assert msg.structList[i].uInt8Field == 0 From e4db5e6d6634545b2f1b4384d4c682036d8d11cf Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 9 Apr 2017 17:17:10 -0700 Subject: [PATCH 063/126] Add exception raising tests for `traversal_limit_in_words` --- test/test_serialization.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index 66a1d88..392bbd2 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -175,6 +175,11 @@ def test_from_bytes_traversal_limit(all_types): bld.init("structList", size) data = bld.to_bytes() + msg = all_types.TestAllTypes.from_bytes(data) + with pytest.raises(capnp.KjException): + for i in range(0, size): + msg.structList[i].uInt8Field == 0 + msg = all_types.TestAllTypes.from_bytes(data, traversal_limit_in_words=2**62) for i in range(0, size): @@ -187,6 +192,11 @@ def test_from_bytes_packed_traversal_limit(all_types): bld.init("structList", size) data = bld.to_bytes_packed() + msg = all_types.TestAllTypes.from_bytes_packed(data) + with pytest.raises(capnp.KjException): + for i in range(0, size): + msg.structList[i].uInt8Field == 0 + msg = all_types.TestAllTypes.from_bytes_packed(data, traversal_limit_in_words=2**62) for i in range(0, size): From 51c4bb1bb603650eb2bd6d8f3b8ca08a78342e32 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 9 Apr 2017 20:57:33 -0700 Subject: [PATCH 064/126] Add tuple setting test --- test/test_struct.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/test_struct.py b/test/test_struct.py index 14aaca2..953a8be 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -115,6 +115,14 @@ def test_builder_set_from_list(all_types): assert list(msg.int32List) == [0, 1, 2] +def test_builder_set_from_tuple(all_types): + msg = all_types.TestAllTypes.new_message() + + msg.int32List = (0, 1, 2) + + assert list(msg.int32List) == [0, 1, 2] + + def test_null_str(all_types): msg = all_types.TestAllTypes.new_message() From 44c2b8dfe6c339f879de4f2eb4edcebcd78d8100 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Mon, 10 Apr 2017 20:37:16 -0700 Subject: [PATCH 065/126] Bump version to v0.5.11 and update CHANGELOG --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a06354..c37efaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.5.11 (2017-04-10) +- Make enums hashable (thanks to @madeleine-empirical) +- Rework logic on when to build bundled libcapnp. Fixes cross-compilation (thanks to @benizl) +- Add traversal_limit_in_words and nesting_limit to RPC classes (thanks to @asilversempirical) +- Include class attributes in __dir__. This allows for code completion of class methods (thanks to @chaoflow ) +- Allow setting lists with python tuples (thanks to @chaoflow) +- Fix traversal_limit_in_words and nesting_limit being ignored by `from_bytes` (thanks to @plesner) + ## v0.5.10 (2016-11-28) - Fix bug that prevented event loop from actually being lazy initialized - Fix possible recursive loop in KjException diff --git a/setup.py b/setup.py index 25cbaca..495a228 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 10 +MICRO = 11 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 341b536bafe971f57fb3d8f61b4ed50ab2d22f9a Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 18 Apr 2017 21:42:50 -0700 Subject: [PATCH 066/126] Bump bundled capnp version to v0.5.3.1 --- buildutils/bundle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 7a6b446..c0afba1 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,8 +35,8 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,5,2) -libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) +bundled_version = (0,5,3,1) +libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp HERE = os.path.dirname(__file__) From 98b5ed837bdecdd8c57eb680872b516edf3d6362 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Tue, 18 Apr 2017 21:44:30 -0700 Subject: [PATCH 067/126] Bump version to v0.5.12 and update CHANGELOG --- CHANGELOG.md | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37efaf..d879dec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## v0.5.12 (2017-04-18) +- Bump bundled capnp version to v0.5.3.1 + ## v0.5.11 (2017-04-10) - Make enums hashable (thanks to @madeleine-empirical) - Rework logic on when to build bundled libcapnp. Fixes cross-compilation (thanks to @benizl) diff --git a/setup.py b/setup.py index 495a228..cd6daaa 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 11 +MICRO = 12 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From fbc17674750c1ffd02db1a4d9f971e72d66eafb1 Mon Sep 17 00:00:00 2001 From: Ben Moran Date: Thu, 8 Jun 2017 12:01:11 +0100 Subject: [PATCH 068/126] Expose Duration units and add a Nanoseconds function to let us build against 0.6.0 --- capnp/includes/capnp_cpp.pxd | 12 +++++++++++- capnp/lib/capnp.pyx | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index e4f5675..9e4f437 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -101,7 +101,14 @@ ctypedef Promise[PyArray] PyPromiseArray cdef extern from "kj/time.h" namespace " ::kj": cdef cppclass Duration: - Duration(int64_t) + Duration operator*(int64_t) + Duration NANOSECONDS + Duration MICROSECONDS + Duration MILLISECONDS + Duration SECONDS + Duration MINUTES + Duration HOURS + Duration DAYS # cdef cppclass TimePoint: # TimePoint(Duration) cdef cppclass Timer: @@ -109,6 +116,9 @@ cdef extern from "kj/time.h" namespace " ::kj": # VoidPromise atTime(TimePoint time) VoidPromise afterDelay(Duration delay) +cdef inline Duration Nanoseconds(int64_t nanos): + return NANOSECONDS * nanos + cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass AsyncIoStream: pass diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index f25eb6c..aa23137 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1688,7 +1688,7 @@ cdef class _Timer: return self cpdef after_delay(self, time) except +reraise_kj_exception: - return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Duration(time))) + return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Nanoseconds(time))) def getTimer(): return _Timer()._init(helpers.getTimer(C_DEFAULT_EVENT_LOOP_GETTER().thisptr)) From 1221ec75a95b3e018de317c4e2c89e42c699435e Mon Sep 17 00:00:00 2001 From: Ben Moran Date: Thu, 8 Jun 2017 12:19:52 +0100 Subject: [PATCH 069/126] Update bundled capnproto to 0.6.0 --- buildutils/bundle.py | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index c0afba1..7f7e483 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,8 +35,8 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,5,3,1) -libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version) +bundled_version = (0,6,0) +libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp HERE = os.path.dirname(__file__) diff --git a/setup.py b/setup.py index cd6daaa..d7928e5 100644 --- a/setup.py +++ b/setup.py @@ -13,8 +13,8 @@ from distutils.extension import Extension _this_dir = os.path.dirname(__file__) MAJOR = 0 -MINOR = 5 -MICRO = 12 +MINOR = 6 +MICRO = 0 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 4435c60872b1a1a591a537ffe495b01d1ff926c3 Mon Sep 17 00:00:00 2001 From: Trevor Highland Date: Wed, 26 Jul 2017 20:17:52 -0500 Subject: [PATCH 070/126] Issue 145: decrement exception object prior to returning. --- capnp/helpers/capabilityHelper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/capnp/helpers/capabilityHelper.h b/capnp/helpers/capabilityHelper.h index 72ee96a..d6f7ff0 100644 --- a/capnp/helpers/capabilityHelper.h +++ b/capnp/helpers/capabilityHelper.h @@ -66,6 +66,7 @@ void reraise_kj_exception() { catch (kj::Exception& exn) { auto obj = wrap_kj_exception_for_reraise(exn); PyErr_SetObject((PyObject*)obj->ob_type, obj); + Py_DECREF(obj); } catch (const std::exception& exn) { PyErr_SetString(PyExc_RuntimeError, exn.what()); From b3ab9ab2a6eeb493ee2fe1d61a09a933828145c7 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 27 Jul 2017 19:59:54 -0700 Subject: [PATCH 071/126] Update changelog for v0.6.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d879dec..aa19a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.6.0 (2017-07-27) +- Update bundled capnp version to v0.6.0 and fix related problems (thanks to @benmoran) +- Fix memleak with KjException (thanks to @tsh56) + ## v0.5.12 (2017-04-18) - Bump bundled capnp version to v0.5.3.1 From 0242ba2d33ee65c777d8ad4f2ffd77eab7c70c16 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 27 Jul 2017 20:11:49 -0700 Subject: [PATCH 072/126] Bump to v0.6.1 and update changelog --- CHANGELOG.md | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa19a7a..20f51af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## v0.6.1 (2017-07-27) +- Fixed upload to PyPi (forgot to cythonize) + ## v0.6.0 (2017-07-27) - Update bundled capnp version to v0.6.0 and fix related problems (thanks to @benmoran) - Fix memleak with KjException (thanks to @tsh56) diff --git a/setup.py b/setup.py index d7928e5..81ebac7 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 6 -MICRO = 0 +MICRO = 1 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From 6a5f697f729dbb818fe3423790b126848c0cb7c2 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 28 Nov 2017 00:13:54 +0000 Subject: [PATCH 073/126] Support generic buffers in from_bytes() (Also throw a TypeError if it's not a bytes object or a buffer) --- capnp/lib/capnp.pyx | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index aa23137..7f0b0fc 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -15,7 +15,7 @@ from libc.stdlib cimport malloc, free from libc.string cimport memcpy from cython.operator cimport dereference as deref from cpython.exc cimport PyErr_Clear -from cpython cimport Py_buffer +from cpython cimport Py_buffer, PyObject_CheckBuffer from cpython.buffer cimport PyBUF_SIMPLE from types import ModuleType as _ModuleType @@ -3756,11 +3756,7 @@ cdef class _FlatArrayMessageReader(_MessageReader): raise ValueError("input length must be a multiple of eight bytes") cdef char * ptr - if type(buf) == _mmap.mmap: - view = _BufferView(buf) - ptr = view.buf - self._object_to_pin = view - else: + if isinstance(buf, bytes): ptr = buf if (ptr) % 8 != 0: aligned = _AlignedBuffer(buf) @@ -3768,6 +3764,12 @@ cdef class _FlatArrayMessageReader(_MessageReader): self._object_to_pin = aligned else: self._object_to_pin = buf + elif PyObject_CheckBuffer(buf): + view = _BufferView(buf) + ptr = view.buf + self._object_to_pin = view + else: + raise TypeError('expected buffer-like object in FlatArrayMessageReader') self.thisptr = new schema_cpp.FlatArrayMessageReader( schema_cpp.WordArrayPtr(ptr, sz//8), From 8fd6cc6f668a4f39c06fd94ee9fc9f6fc735021c Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 28 Nov 2017 00:15:05 +0000 Subject: [PATCH 074/126] Add a test for generic from_bytes() --- test/test_serialization.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index 392bbd2..a3ff183 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -65,6 +65,20 @@ def test_roundtrip_bytes_mmap(all_types): msg = all_types.TestAllTypes.from_bytes(memory) test_regression.check_all_types(msg) +@pytest.mark.skipif(sys.version_info[0] < 3, reason="memoryview is a builtin on Python 3") +def test_roundtrip_bytes_buffer(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + b = msg.to_bytes() + v = memoryview(b) + msg = all_types.TestAllTypes.from_bytes(v) + test_regression.check_all_types(msg) + +def test_roundtrip_bytes_fail(all_types): + with pytest.raises(TypeError): + all_types.TestAllTypes.from_bytes(42) + @pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test.") def test_roundtrip_bytes_packed(all_types): msg = all_types.TestAllTypes.new_message() From 18e87a17191d8c9468bfd8e3d10f3ed7f51c5021 Mon Sep 17 00:00:00 2001 From: Ivan Smirnov Date: Tue, 28 Nov 2017 00:19:15 +0000 Subject: [PATCH 075/126] (Remove now-redundant import) --- capnp/lib/capnp.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 7f0b0fc..864993a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -31,7 +31,6 @@ import threading as _threading import socket as _socket import random as _random import collections as _collections -import mmap as _mmap _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR From add8af4bc901df93713c7e9fa2948e6cb655fc34 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 30 Nov 2017 22:08:01 -0800 Subject: [PATCH 076/126] Bump version to v0.6.2 and update changelog --- CHANGELOG.md | 3 +++ setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20f51af..d55ce8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## v0.6.2 (2017-11-30) +- Add support for buffers/memoryviews in `from_bytes` (thanks to @aldanor) + ## v0.6.1 (2017-07-27) - Fixed upload to PyPi (forgot to cythonize) diff --git a/setup.py b/setup.py index 81ebac7..fbbfa8b 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 6 -MICRO = 1 +MICRO = 2 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From a73395e7bb100c5bcf2abea3d60b4a515d63c6d6 Mon Sep 17 00:00:00 2001 From: Yuval Katsnelson Date: Wed, 6 Dec 2017 10:48:38 +0200 Subject: [PATCH 077/126] Fixed Python object leak in RemotePromise --- capnp/lib/capnp.pyx | 3 --- 1 file changed, 3 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 864993a..441f67a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1958,9 +1958,6 @@ cdef class _RemotePromise: if args_length - defaults_length != 1: raise KjException('Function passed to `then` call must take exactly one argument') - Py_INCREF(func) - Py_INCREF(error_func) - cdef Promise new_promise = Promise()._init(helpers.then(deref(self.thisptr), func, error_func), self) return Promise()._init(new_promise.thisptr.attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), new_promise) From 74618de3efab0dd44e12156c3613f751cb57aa7d Mon Sep 17 00:00:00 2001 From: Yuval Katsnelson Date: Wed, 6 Dec 2017 10:49:09 +0200 Subject: [PATCH 078/126] Update bundled capnp to 0.6.1 --- buildutils/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 7f7e483..b0e7f0e 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,7 +35,7 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,6,0) +bundled_version = (0,6,1) libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp From 19e1b189caa786c7f572e679d6bb94aadfbdb5e0 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 14 Jan 2018 11:57:08 -0800 Subject: [PATCH 079/126] Bump version to v0.6.3 and update changelog --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d55ce8f..46bccb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.6.3 (2018-01-14) +- Bump bundled capnp version to v0.6.1 (thanks to @E8Yuval) +- Fix a memleak in RemotePromise (thanks to @E8Yuval) + ## v0.6.2 (2017-11-30) - Add support for buffers/memoryviews in `from_bytes` (thanks to @aldanor) diff --git a/setup.py b/setup.py index fbbfa8b..a3996a7 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 6 -MICRO = 2 +MICRO = 3 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From eb626939156a33454d6d3bcb604fc87f51352cf1 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Tue, 6 Nov 2018 21:11:59 -0500 Subject: [PATCH 080/126] Exposing SchemaParser in Cython header --- capnp/lib/capnp.pxd | 12 ++++++++++++ capnp/lib/capnp.pyx | 8 -------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index dc0ed4e..0882bb7 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -12,6 +12,18 @@ cdef class _StructSchemaField: cdef object _parent cdef _init(self, C_StructSchema.Field other, parent=?) +cdef class _StringArrayPtr: + cdef StringPtr * thisptr + cdef object parent + cdef size_t size + cdef ArrayPtr[StringPtr] asArrayPtr(self) except +reraise_kj_exception + +cdef class SchemaParser: + cdef C_SchemaParser * thisptr + cdef public dict modules_by_id + cdef list _all_imports + cdef _StringArrayPtr _last_import_array + cpdef _parse_disk_file(self, displayName, diskPath, imports) except +reraise_kj_exception cdef class _DynamicOrphan: cdef C_DynamicOrphan thisptr diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 441f67a..6e62207 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3101,10 +3101,6 @@ class _EnumModule(object): setattr(self, name, val) cdef class _StringArrayPtr: - cdef StringPtr * thisptr - cdef object parent - cdef size_t size - def __cinit__(self, size_t size, parent): self.size = size self.thisptr = malloc(sizeof(StringPtr) * size) @@ -3122,10 +3118,6 @@ cdef class SchemaParser: Do not use this class unless you're sure you know what you're doing. Use the convenience method :func:`load` instead. """ - cdef C_SchemaParser * thisptr - cdef public dict modules_by_id - cdef list _all_imports - cdef _StringArrayPtr _last_import_array def __cinit__(self): self.thisptr = new C_SchemaParser() From 3afcbb391817380a84e9b2a8f585d56766e5a3f5 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Sun, 18 Nov 2018 10:01:10 -0500 Subject: [PATCH 081/126] Replacing end-of-life Python versions with live versions --- .travis.yml | 5 +++-- README.md | 2 +- setup.py | 7 ++++--- tox.ini | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5b28cdc..99ed79f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,11 @@ sudo: required language: python python: - - 2.6 - 2.7 - - 3.3 - 3.4 + - 3.5 + - 3.6 + - 3.7 - pypy env: diff --git a/README.md b/README.md index e80586c..30f8d28 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you wish to install using the latest upstream C++ Cap'n Proto: ## Python Versions -Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. +Python 2.7, Python 3.4+, and PyPy 2.1+ are supported. One oddity to note is that `Text` type fields will be treated as byte strings under Python 2, and unicode strings under Python 3. `Data` fields will always be treated as byte strings. diff --git a/setup.py b/setup.py index a3996a7..83ed9be 100644 --- a/setup.py +++ b/setup.py @@ -171,11 +171,12 @@ setup( 'Programming Language :: C++', 'Programming Language :: Cython', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Communications'], ) diff --git a/tox.ini b/tox.ini index e41316b..6f64bb5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py32,py33,py34 +envlist = py27,py34,py35,py36,py37 [testenv] deps= From 770896132b43c7f6d0abdc8c8429b1a82f2829e3 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Sun, 18 Nov 2018 10:35:57 -0500 Subject: [PATCH 082/126] Adding patch to Travis CI for Python 3.7 support --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 99ed79f..e0b10ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,13 @@ python: - 3.7 - pypy +# Travis CI support for Python 3.7 currently requires patch (https://github.com/travis-ci/travis-ci/issues/9815) +matrix: + include: + - python: 3.7 + dist: xenial + sudo: true + env: - BUILD_CAPNP= - BUILD_CAPNP=true From 7052cdbaba113771f28aa1805899c1a49585b6c2 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Sun, 18 Nov 2018 10:52:58 -0500 Subject: [PATCH 083/126] Removing Python 3.7 from normal Travis Python list --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e0b10ef..2b3b303 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ python: - 3.4 - 3.5 - 3.6 - - 3.7 - pypy # Travis CI support for Python 3.7 currently requires patch (https://github.com/travis-ci/travis-ci/issues/9815) From afe72a15b1441c1df4eb64a65242621cd8e29976 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Sun, 18 Nov 2018 15:39:33 -0500 Subject: [PATCH 084/126] Setting distro to Xenial globally in attempt to get Python 3.7 CI --- .travis.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b3b303..bcd76be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ +# Travis CI support for Python 3.7 requires Xenial (https://github.com/travis-ci/travis-ci/issues/9815) +dist: xenial + # Use older, non-container infrastructure to allow sudo sudo: required @@ -8,15 +11,9 @@ python: - 3.4 - 3.5 - 3.6 + - 3.7 - pypy -# Travis CI support for Python 3.7 currently requires patch (https://github.com/travis-ci/travis-ci/issues/9815) -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true - env: - BUILD_CAPNP= - BUILD_CAPNP=true From 2b26665f5613258e8ca63f09c9c76c2d6bf8c122 Mon Sep 17 00:00:00 2001 From: Colin Jermain Date: Sun, 18 Nov 2018 16:27:09 -0500 Subject: [PATCH 085/126] Removing Python 3.7 from this PR --- .travis.yml | 4 ---- setup.py | 1 - tox.ini | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcd76be..a852039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,3 @@ -# Travis CI support for Python 3.7 requires Xenial (https://github.com/travis-ci/travis-ci/issues/9815) -dist: xenial - # Use older, non-container infrastructure to allow sudo sudo: required @@ -11,7 +8,6 @@ python: - 3.4 - 3.5 - 3.6 - - 3.7 - pypy env: diff --git a/setup.py b/setup.py index 83ed9be..c7c7b79 100644 --- a/setup.py +++ b/setup.py @@ -176,7 +176,6 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Communications'], ) diff --git a/tox.ini b/tox.ini index 6f64bb5..75af016 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37 +envlist = py27,py34,py35,py36 [testenv] deps= From dbd23721a914711e0ec310ec012257b534dcbfd9 Mon Sep 17 00:00:00 2001 From: Trevor Highland Date: Mon, 28 Jan 2019 17:04:34 +0000 Subject: [PATCH 086/126] Support long messages in read_multiple_bytes. --- capnp/includes/schema_cpp.pxd | 1 + capnp/lib/capnp.pyx | 73 ++++++++++++++++++++++++++++------- test/test_large_read.py | 18 +++++++++ 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index d9b691c..9cec13d 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -786,6 +786,7 @@ cdef extern from "capnp/serialize.h" namespace " ::capnp": cdef cppclass FlatArrayMessageReader(MessageReader): FlatArrayMessageReader(WordArrayPtr array) except +reraise_kj_exception FlatArrayMessageReader(WordArrayPtr array, ReaderOptions) except +reraise_kj_exception + const word* getEnd() const void writeMessageToFd(int, MessageBuilder&) except +reraise_kj_exception diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 6e62207..65461b0 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3624,31 +3624,43 @@ cdef class _MultiplePackedMessageReader: return self cdef class _MultipleBytesMessageReader: - cdef schema_cpp.ArrayInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - - cdef public object traversal_limit_in_words, nesting_limit, schema, buf + cdef Py_ssize_t offset, sz + cdef const char *ptr + cdef object _object_to_pin + cdef public object traversal_limit_in_words, nesting_limit, schema def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.offset = 0 self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) + self.sz = len(buf) + if isinstance(buf, bytes): + self.ptr = buf + if (self.ptr) % 8 != 0: + aligned = _AlignedBuffer(buf) + self.ptr = aligned.buf + self._object_to_pin = aligned + else: + self._object_to_pin = buf + self.ptr = buf + elif PyObject_CheckBuffer(buf): + view = _BufferView(buf) + self.ptr = view.buf + self._object_to_pin = view + else: + raise TypeError('expected buffer-like object in FlatArrayMessageReader') - self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - del self.buffered_stream - del self.stream def __next__(self): + cdef _FlatArrayMessageReaderAligned reader + if self.offset == self.sz: + raise StopIteration try: - reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + reader = _FlatArrayMessageReaderAligned() + reader._init(self._object_to_pin, self.ptr + self.offset, self.sz - self.offset, self.traversal_limit_in_words, self.nesting_limit) + self.offset += reader.msg_size return reader.get_root(self.schema) except KjException as e: if 'EOF' in str(e): @@ -3732,6 +3744,37 @@ cdef class _BufferView: def __dealloc__(self): PyBuffer_Release(&self.view) +@cython.internal +cdef class _FlatArrayMessageReaderAligned(_MessageReader): + """ + Creates a reader based on a contiguous block of memory + + For performance consideration it's assumed that the provided buffer is already aligned. This + allows us to align a set of adjacent messages with a single align operation. + """ + cdef object _object_to_pin + cdef Py_ssize_t msg_size + def __init__(self): + self.msg_size = 0 + + + cdef _init(self, buf, const char *ptr, Py_ssize_t sz, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) + cdef schema_cpp.FlatArrayMessageReader * flat_reader + + self._object_to_pin = buf + + flat_reader = new schema_cpp.FlatArrayMessageReader( + schema_cpp.WordArrayPtr(ptr, sz//8), + opts) + self.thisptr = flat_reader + self.msg_size = flat_reader.getEnd() - ptr + return self + + def __dealloc__(self): + del self.thisptr + + @cython.internal cdef class _FlatArrayMessageReader(_MessageReader): cdef object _object_to_pin diff --git a/test/test_large_read.py b/test/test_large_read.py index c925d1f..353fc72 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -40,3 +40,21 @@ def test_large_read_multiple(test_capnp): for m in test_capnp.Msg.read_multiple(f): pass + + +def test_large_read_multiple_bytes(test_capnp): + msg1 = test_capnp.Msg.new_message() + msg1.data = [0x41] * 8192 + m1 = msg1.to_bytes() + msg2 = test_capnp.Msg.new_message() + m2 = msg2.to_bytes() + + data = m1 + m2 + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + + for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): + pass + + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass From ed6e39ded07a7dd8aebf0b98e206ded8e6a1a0b7 Mon Sep 17 00:00:00 2001 From: Trevor Highland Date: Mon, 28 Jan 2019 17:39:10 +0000 Subject: [PATCH 087/126] Update test to support 3.5 --- test/test_large_read.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/test_large_read.py b/test/test_large_read.py index 353fc72..82bb925 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -53,8 +53,5 @@ def test_large_read_multiple_bytes(test_capnp): for m in test_capnp.Msg.read_multiple_bytes(data): pass - for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): - pass - for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): pass From 2a66c2f6bf4604a1d5f6281d67475a1d12306393 Mon Sep 17 00:00:00 2001 From: Trevor Highland Date: Mon, 28 Jan 2019 18:09:17 +0000 Subject: [PATCH 088/126] Skip tests based on python version. --- test/test_large_read.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/test_large_read.py b/test/test_large_read.py index 82bb925..bb1129e 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -1,4 +1,5 @@ import pytest +import platform import capnp import os import tempfile @@ -41,17 +42,28 @@ def test_large_read_multiple(test_capnp): for m in test_capnp.Msg.read_multiple(f): pass - -def test_large_read_multiple_bytes(test_capnp): +def get_two_adjacent_messages(test_capnp): msg1 = test_capnp.Msg.new_message() msg1.data = [0x41] * 8192 m1 = msg1.to_bytes() msg2 = test_capnp.Msg.new_message() m2 = msg2.to_bytes() - data = m1 + m2 + return m1 + m2 + +def test_large_read_multiple_bytes(test_capnp): + data = get_two_adjacent_messages(test_capnp) for m in test_capnp.Msg.read_multiple_bytes(data): pass +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="PyPy memoryview support is limited") +def test_large_read_mutltiple_bytes_memoryview(test_capnp): + data = get_two_adjacent_messages(test_capnp) for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): pass + +@pytest.mark.skipif(sys.version_info[0] == 3, reason="Legacy buffer support only for python 2.7") +def test_large_read_mutltiple_bytes_buffer(test_capnp): + data = get_two_adjacent_messages(test_capnp) + for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): + pass From da8f8869fc231d3a624e7893632af75dda1356d6 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 31 Jan 2019 01:02:11 -0800 Subject: [PATCH 089/126] Add some edge case tests for `read_multiple_bytes` --- test/test_large_read.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/test_large_read.py b/test/test_large_read.py index bb1129e..e912008 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -56,14 +56,44 @@ def test_large_read_multiple_bytes(test_capnp): for m in test_capnp.Msg.read_multiple_bytes(data): pass + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp)[:-1] + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp) + b' ' + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + @pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="PyPy memoryview support is limited") def test_large_read_mutltiple_bytes_memoryview(test_capnp): data = get_two_adjacent_messages(test_capnp) for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): pass + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp)[:-1] + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp) + b' ' + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass + @pytest.mark.skipif(sys.version_info[0] == 3, reason="Legacy buffer support only for python 2.7") def test_large_read_mutltiple_bytes_buffer(test_capnp): data = get_two_adjacent_messages(test_capnp) for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp)[:-1] + for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp) + b' ' + for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): + pass From e4d7e21a660d2202f4bce2fb39a2ed101c1b53d6 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 31 Jan 2019 01:03:21 -0800 Subject: [PATCH 090/126] Add error if trying to run `python setup.py` sdist without pandoc --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index c7c7b79..c1ddfdc 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,8 @@ try: changelog = '\nChangelog\n=============\n' + changelog long_description += changelog except (IOError, ImportError): + if sys.argv[2] == 'sdist': + raise long_description = '' # Clean command, invoked with `python setup.py clean` From a4f740fb0fa7f9c6f5f27907e07c65fc331ef371 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 31 Jan 2019 01:10:15 -0800 Subject: [PATCH 091/126] Bump version to v0.6.4 and update changelog --- CHANGELOG.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46bccb5..de44447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.6.4 (2019-1-31) +- Fix bugs in `read_multiple_bytes` (thanks to @tsh56) +- Remove end-of-life Python versions 2.6, 3.2, and 3.3. Add CI tests for 3.6 +- Expose SchemaParser in Cython header + ## v0.6.3 (2018-01-14) - Bump bundled capnp version to v0.6.1 (thanks to @E8Yuval) - Fix a memleak in RemotePromise (thanks to @E8Yuval) diff --git a/setup.py b/setup.py index c1ddfdc..afb5d01 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 6 -MICRO = 3 +MICRO = 4 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From bdb7d101e929fc3c7e5f43470dd6774e11a3493f Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 31 Jan 2019 01:20:29 -0800 Subject: [PATCH 092/126] Add DEPLOY.md with instructions for deploying to PyPI --- DEPLOY.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 DEPLOY.md diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..277e5f4 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,38 @@ +# Deployment instructions for PyPi + +This file is meant for maintainers of pycapnp, and documents the process for uploading to PyPI. + +## Pre-requisites + +``` +pip install pypandoc cython +``` + +## Run tests + +I typically sanity check by running the tests once again locally, but as long as Travis is green, you're probably fine. + +## Add a commit that bumps the version + +Bump the version in setup.py, and add descriptions of all the changes to CHANGELOG.md (see 19e1b189caa786c7f572e679d6bb94aadfbdb5e0 for an example commit). + +## Run the build and upload + +Run the following command to clean up old artifacts, run the build, and then upload the result to PyPI + +``` +rm -rf bundled/ capnp/version.py capnp/lib/capnp.{h,cpp} build; python setup.py build && python setup.py sdist upload -r PyPI +``` + +## Test the PyPI release + +I manually test the PyPI release after it's been uploaded. I have a few virtualenvs that I manually run the following command in (run this from the pycapnp directory since it runs the tests at the end): +``` +yes | pip uninstall pycapnp; pip install pycapnp && py.test test +``` + +I usually test the following configurations: +- Python 2.7 with and without cython installed +- Python 3.6 with and without cython installed + +This step could probably benefit greatly from some automation. Perhaps even Travis could handle it, but I'm not sure how best to trigger Travis from a PyPI release. From 3ab199b4b96c821c7806e20501948d155c2456de Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Thu, 31 Jan 2019 01:30:07 -0800 Subject: [PATCH 093/126] Fix minor typo in Changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de44447..ef14edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.6.4 (2019-1-31) +## v0.6.4 (2019-01-31) - Fix bugs in `read_multiple_bytes` (thanks to @tsh56) - Remove end-of-life Python versions 2.6, 3.2, and 3.3. Add CI tests for 3.6 - Expose SchemaParser in Cython header From cb3f190b955bdb1bfb6e0ac0b2f9306a5c79f7b5 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 1 Feb 2019 15:11:53 -0800 Subject: [PATCH 094/126] Add deploy notes about tags --- DEPLOY.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/DEPLOY.md b/DEPLOY.md index 277e5f4..69eae40 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -36,3 +36,9 @@ I usually test the following configurations: - Python 3.6 with and without cython installed This step could probably benefit greatly from some automation. Perhaps even Travis could handle it, but I'm not sure how best to trigger Travis from a PyPI release. + +## Tag the github release + +Tag the release on the develop branch (not the master branch). Sadly, I've stopped using git-flow, and at this point it might be worth moving back to using just master, but that would take some amount of work and I worry that it would break open PRs. Definitely worth considering if development picks back up. + +Version numbers roughly follow semver, although I try to loosely follow upstream Cap'n Proto C++ versions as well. So when pycapnp officially starts using v0.7.0 of the C++ library, pycapnp's version should be bumped to v0.7.0 as well. From ab267eccbdb7df1b657005cd90725b73aa975093 Mon Sep 17 00:00:00 2001 From: Andrey Cizov Date: Sat, 13 Jul 2019 15:06:18 +0100 Subject: [PATCH 095/126] fix travis build failure --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index afb5d01..42051ac 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ try: changelog = '\nChangelog\n=============\n' + changelog long_description += changelog except (IOError, ImportError): - if sys.argv[2] == 'sdist': + if len(sys.argv) and sys.argv[-1] == 'sdist': raise long_description = '' From de22f7eb97744230e98efbc2567c0404c2ffcf36 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Mon, 16 Sep 2019 21:33:56 -0700 Subject: [PATCH 096/126] Fixing compilation errors with capnproto-7.0.0 --- buildutils/detect.py | 2 +- capnp/includes/capnp_cpp.pxd | 2 +- capnp/includes/schema_cpp.pxd | 2 +- capnp/lib/capnp.pyx | 2 +- capnp/templates/module.pyx | 2 +- setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/buildutils/detect.py b/buildutils/detect.py index 8fd628d..7810380 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -58,7 +58,7 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): else: lpreargs = ['-m64'] extra = compiler_attrs.get('extra_compile_args', []) - extra += ['--std=c++11'] + extra += ['--std=c++14'] objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra) cc.link_executable(objs, efile, extra_preargs=lpreargs) diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index 9e4f437..f43956e 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -1,6 +1,6 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 +# distutils: extra_compile_args = --std=c++14 cdef extern from "capnp/helpers/checkCompiler.h": pass diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index 9cec13d..c0ba23f 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -1,6 +1,6 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 +# distutils: extra_compile_args = --std=c++14 from libc.stdint cimport * from capnp_cpp cimport DynamicOrphan diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 65461b0..071317f 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1,6 +1,6 @@ # capnp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 +# distutils: extra_compile_args = --std=c++14 # distutils: libraries = capnpc capnp-rpc capnp kj-async kj # distutils: include_dirs = . # cython: c_string_type = str diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index 943e802..1dcf7f5 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -1,6 +1,6 @@ # addressbook_fast.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 +# distutils: extra_compile_args = --std=c++14 # distutils: include_dirs = {{include_dir}} # distutils: libraries = capnpc capnp capnp-rpc # distutils: sources = {{file.filename}}.cpp diff --git a/setup.py b/setup.py index 42051ac..f04d2de 100644 --- a/setup.py +++ b/setup.py @@ -138,7 +138,7 @@ else: extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"], include_dirs=["."], language='c++', - extra_compile_args=['--std=c++11'], + extra_compile_args=['--std=c++14'], libraries=['capnpc', 'capnp-rpc', 'capnp', 'kj-async', 'kj'])] setup( From 423e4f1f8ce9672215319fe42f10900f7ee4291e Mon Sep 17 00:00:00 2001 From: Andrey Cizov Date: Sat, 13 Jul 2019 16:44:42 +0100 Subject: [PATCH 097/126] remove warning during compilation --- capnp/lib/capnp.pxd | 2 ++ capnp/lib/capnp.pyx | 1 + setup.py | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 0882bb7..e8dffa5 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -1,3 +1,5 @@ +# cython: language_level = 2 + from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 071317f..6830ba6 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -6,6 +6,7 @@ # cython: c_string_type = str # cython: c_string_encoding = default # cython: embedsignature = True +# cython: language_level = 2 cimport cython diff --git a/setup.py b/setup.py index f04d2de..db60d82 100644 --- a/setup.py +++ b/setup.py @@ -44,8 +44,8 @@ write_version_py() # Try to convert README using pandoc try: import pypandoc - long_description = pypandoc.convert('README.md', 'rst') - changelog = pypandoc.convert('CHANGELOG.md', 'rst') + long_description = pypandoc.convert_file('README.md', 'rst') + changelog = pypandoc.convert_file('CHANGELOG.md', 'rst') changelog = '\nChangelog\n=============\n' + changelog long_description += changelog except (IOError, ImportError): From e8662a1dc9f291ae9d2ae90862e57d49994b0a64 Mon Sep 17 00:00:00 2001 From: Andrey Cizov Date: Sat, 13 Jul 2019 16:12:59 +0100 Subject: [PATCH 098/126] remove deprecation warnings --- capnp/lib/capnp.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 6830ba6..c396402 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1814,7 +1814,7 @@ cdef class Promise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: @@ -1877,7 +1877,7 @@ cdef class _VoidPromise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: @@ -1950,7 +1950,7 @@ cdef class _RemotePromise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: From 33725b9e7f74d786fce9595da895eb28840e8e99 Mon Sep 17 00:00:00 2001 From: Andrey Cizov Date: Sat, 13 Jul 2019 16:45:12 +0100 Subject: [PATCH 099/126] ignore files created during setup.py install --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index ad81cce..ff5b24b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,9 @@ capnp/*.cpp capnp/version.py MANIFEST docs/_build + +capnp/lib/capnp.cpp +capnp/lib/capnp.h +bundled/ +example +*.iml From e73b63ddd1d6e0398d2ab944b3a6fb3ec7b493de Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Wed, 11 Sep 2019 23:38:39 -0700 Subject: [PATCH 100/126] Removed deprecated functions - PyObject_AsReadBuffer - PyObject_AsWriteBuffer --- capnp/lib/capnp.pyx | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index c396402..d11c1e0 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -16,8 +16,8 @@ from libc.stdlib cimport malloc, free from libc.string cimport memcpy from cython.operator cimport dereference as deref from cpython.exc cimport PyErr_Clear -from cpython cimport Py_buffer, PyObject_CheckBuffer -from cpython.buffer cimport PyBUF_SIMPLE +from cpython cimport array, Py_buffer, PyObject_CheckBuffer +from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_WRITABLE from types import ModuleType as _ModuleType import os as _os @@ -32,6 +32,7 @@ import threading as _threading import socket as _socket import random as _random import collections as _collections +import array _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR @@ -287,8 +288,6 @@ ctypedef fused PromiseTypes: PromiseFulfillerPair cdef extern from "Python.h": - cdef int PyObject_AsReadBuffer(object, void** b, Py_ssize_t* c) - cdef int PyObject_AsWriteBuffer(object, void** b, Py_ssize_t* c) cdef int PyObject_GetBuffer(object, Py_buffer *view, int flags) cdef void PyBuffer_Release(Py_buffer *view) @@ -3484,23 +3483,24 @@ cdef class _PackedMessageReader(_MessageReader): cdef class _PackedMessageReaderBytes(_MessageReader): cdef schema_cpp.ArrayInputStream * stream + cdef Py_buffer view def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = buf - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) + if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) self.thisptr = new schema_cpp.PackedMessageReader(deref(self.stream), opts) def __dealloc__(self): del self.thisptr del self.stream + PyBuffer_Release(&self.view) cdef class _InputMessageReader(_MessageReader): """Read a Cap'n Proto message from a file descriptor in a packed manner @@ -3675,6 +3675,7 @@ cdef class _MultipleBytesMessageReader: cdef class _MultipleBytesPackedMessageReader: cdef schema_cpp.ArrayInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef Py_buffer view cdef public object traversal_limit_in_words, nesting_limit, schema, buf @@ -3683,15 +3684,15 @@ cdef class _MultipleBytesPackedMessageReader: self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) + if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): + PyBuffer_Release(&self.view) del self.buffered_stream del self.stream @@ -3712,23 +3713,24 @@ cdef class _MultipleBytesPackedMessageReader: cdef class _AlignedBuffer: cdef char * buf cdef bint allocated + cdef Py_buffer view # other should also have a length that's a multiple of 8 def __init__(self, other): - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(other, &ptr, &sz) + if PyObject_GetBuffer(other, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") other_len = len(other) # malloc is defined as being word aligned # we don't care about adding NULL terminating character self.buf = malloc(other_len) - memcpy(self.buf, ptr, other_len) + memcpy(self.buf, self.view.buf, other_len) self.allocated = True def __dealloc__(self): if self.allocated: free(self.buf) + PyBuffer_Release(&self.view) @cython.internal @@ -3816,25 +3818,27 @@ cdef class _SegmentArrayMessageReader(_MessageReader): cdef object _objects_to_pin cdef schema_cpp.ConstWordArrayPtr* _seg_ptrs + cdef Py_buffer* views def __init__(self, segments, traversal_limit_in_words = None, nesting_limit = None): cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) # take a Python array of bytes and constructs a ConstWordArrayArrayPtr num_segments = len(segments) - cdef const void* ptr - cdef Py_ssize_t segment_size cdef schema_cpp.ConstWordArrayPtr seg_ptr self._seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) + self.views = malloc(num_segments * sizeof(Py_buffer)) self._objects_to_pin = [] for i in range(0, num_segments): - PyObject_AsReadBuffer(segments[i], &ptr, &segment_size) - if (ptr) % 8 != 0: + if PyObject_GetBuffer(segments[i], &self.views[i], PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") + + if (self.views[i].buf) % 8 != 0: aligned = _AlignedBuffer(segments[i]) - ptr = aligned.buf + self.views[i].buf = aligned.buf self._objects_to_pin.append(aligned) else: self._objects_to_pin.append(segments[i]) - seg_ptr = schema_cpp.ConstWordArrayPtr(ptr, segment_size//8) + seg_ptr = schema_cpp.ConstWordArrayPtr(self.views[i].buf, self.views[i].len//8) self._seg_ptrs[i] = seg_ptr self.thisptr = new schema_cpp.SegmentArrayMessageReader( schema_cpp.ConstWordArrayArrayPtr(self._seg_ptrs, num_segments), @@ -3842,20 +3846,24 @@ cdef class _SegmentArrayMessageReader(_MessageReader): def __dealloc__(self): free(self._seg_ptrs) + free(self.views) del self.thisptr @cython.internal cdef class _FlatMessageBuilder(_MessageBuilder): cdef object _object_to_pin + cdef Py_buffer view def __init__(self, buf): - cdef void *ptr - cdef Py_ssize_t sz - PyObject_AsWriteBuffer(buf, &ptr, &sz) - if sz % 8 != 0: + if PyObject_GetBuffer(buf, &self.view, PyBUF_WRITABLE) != 0: + raise KjException("expected variable length string object") + if self.view.len % 8 != 0: raise KjException("input length must be a multiple of eight bytes") self._object_to_pin = buf - self.thisptr = new schema_cpp.FlatMessageBuilder(schema_cpp.WordArrayPtr(ptr, sz//8)) + self.thisptr = new schema_cpp.FlatMessageBuilder(schema_cpp.WordArrayPtr(self.view.buf, self.view.len//8)) + + def __dealloc__(self): + PyBuffer_Release(&self.view) def _message_to_packed_bytes(_MessageBuilder message): r, w = _os.pipe() From 8915ef79f14d72171bec7dc8dbbc9a8ab6b402b3 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Mon, 16 Sep 2019 21:34:29 -0700 Subject: [PATCH 101/126] TwoWayPipe and basic asyncio support Note: I've tried not to break any behaviour of the previously working APIs Python API Changes / Additions - capnp/lib/capnp.pyx * class _RemotePromise + [Added] cpdef _wait(self) = Exception raising code that used to be inside of wait(self) + [Modified] def wait(self) = Same functionality as before + [Added] async def a_wait(self) = Cannot use await as that's a reserved keyword = Uses pollRemote and asyncio.sleep(0) to make call asynchronous * class _TwoPartyVatNetwork + [Added] cdef _init_pipe(self, _TwoWayPipe pipe, Side side, schema_cpp.ReaderOptions opts) = Instanciates a TwoPartyVatNetwork using a TwoWayPipe (instead of using a file handle or connection as before) * class TwoPartyClient + [Modified] def __init__(self, socket=None, restorer=None, traversal_limit_in_words=None, nesting_limit=None) = Changes the socket parameter to be optional = If socket is not specified, default to using a TwoWayPipe + [Added] async def read(self, bufsize) = awaitable function that blocks until data has been read = bufsize defines the maximum amount of data to be read back (e.g. 4096 bytes) = Reads data from TwoWayPipe + [Added] def write(self, data) = Write data to TwoWayPipe = Not awaitable as the write interface of the TwoWayPipe doesn't have poll functionality * class TwoPartyServer + [Modified] def __init__(self, socket=None, restorer=None, server_socket=None, bootstrap=None, traversal_limit_in_words=None, nesting_limit=None) = Changes the socket parameter to be optional = If socket is not specified, default to using a TwoWayPipe = Simplified code by removing an else (self._connect) + [Added] async def read(self, bufsize) = awaitable function that blocks until data has been read = bufsize defines the maximum amount of data to be read back (e.g. 4096 bytes) = Reads data from TwoWayPipe + [Added] def write(self, data) = Write data to TwoWayPipe = Not awaitable as the write interface of the TwoWayPipe doesn't have poll functionality + [Added] async def poll_forever(self) = asyncio equivalent of run_forever() * class _TwoWayPipe + Wrapper class for TwoWayPipe Other Additions - capnp/helpers/asyncHelper.h * pollWaitScope + Pumps the kj event handler + Used for the TwoWayServer * pollRemote + Polls a remote promise + i.e. a capnp RPC call - capnp/helpers/asyncIoHelper.h * AsyncIoStreamReadHelper + I wasn't able to figure out Promise[size_t] using Cython so this was the next best thing I could think of doing + Was needed to handle read polling from a read promise = Polling is used for asyncio as kj waits need a wrapper to be compatible - capnp/lib/capnp.pyx * makeTwoWayPipe + Wrapper for kj newTwoWayPipe function * poll_once + Single pump of the kj event handler (used with pollWaitScope) TwoWayClient Usage - TwoWayPipe - See examples/async_client.py TwoWayServer Usage - TwoWayPipe - See examples/async_server.py capnp/helpers/asyncIoHelper.h Misc Changes - Fixed thread_server.py and thread_client.py to use bootstrap instead of ez_restore - async_client.py and async_server.py examples * Uses the same thread.capnp as thread_client.py and thread_server.py * They are compatible, so you can mix and match client and server for compatibility testing * async_client.py and async_server.py require
: formatting (unlike autodetection from thread_client.py and thread_server.py) --- capnp/helpers/asyncHelper.h | 10 +++ capnp/helpers/asyncIoHelper.h | 53 +++++++++++ capnp/helpers/helpers.pxd | 6 +- capnp/helpers/non_circular.pxd | 9 ++ capnp/includes/capnp_cpp.pxd | 15 +++- capnp/lib/capnp.pxd | 2 +- capnp/lib/capnp.pyx | 160 +++++++++++++++++++++++++++------ examples/async_client.py | 92 +++++++++++++++++++ examples/async_server.py | 87 ++++++++++++++++++ examples/thread_client.py | 4 +- examples/thread_server.py | 10 +-- 11 files changed, 407 insertions(+), 41 deletions(-) create mode 100644 capnp/helpers/asyncIoHelper.h create mode 100755 examples/async_client.py create mode 100755 examples/async_server.py diff --git a/capnp/helpers/asyncHelper.h b/capnp/helpers/asyncHelper.h index 8b4e1f1..4d74e43 100644 --- a/capnp/helpers/asyncHelper.h +++ b/capnp/helpers/asyncHelper.h @@ -39,6 +39,11 @@ void waitNeverDone(kj::WaitScope & scope) { kj::NEVER_DONE.wait(scope); } +void pollWaitScope(kj::WaitScope & scope) { + GILRelease gil; + scope.poll(); +} + kj::Timer * getTimer(kj::AsyncIoContext * context) { return &context->lowLevelProvider->getTimer(); } @@ -57,3 +62,8 @@ capnp::Response< ::capnp::DynamicStruct> * waitRemote(capnp::RemotePromise< ::ca GILRelease gil; return new capnp::Response< ::capnp::DynamicStruct>(promise->wait(scope)); } + +bool pollRemote(capnp::RemotePromise< ::capnp::DynamicStruct> * promise, kj::WaitScope & scope) { + GILRelease gil; + return promise->poll(scope); +} diff --git a/capnp/helpers/asyncIoHelper.h b/capnp/helpers/asyncIoHelper.h new file mode 100644 index 0000000..c848ebb --- /dev/null +++ b/capnp/helpers/asyncIoHelper.h @@ -0,0 +1,53 @@ +#pragma once + +#include "kj/async.h" +#include "kj/async-io.h" + +class AsyncIoStreamReadHelper { +public: + AsyncIoStreamReadHelper(kj::AsyncIoStream * _stream, kj::WaitScope * _scope, size_t bufsize) { + io_stream = _stream; + wait_scope = _scope; + ready = false; + buffer_read_size = 0; + buffer = new unsigned char[bufsize]; + promise = io_stream->read(buffer, 1, bufsize); + } + + ~AsyncIoStreamReadHelper() { + delete[] buffer; + } + + bool poll() { + bool result = promise.poll(*wait_scope); + if (result) { + ready = true; + buffer_read_size = promise.wait(*wait_scope); + } + return result; + } + + size_t read_size() { + if (!ready) { + return 0; + } + return buffer_read_size; + } + + void * read_buffer() { + if (!ready) { + return 0; + } + return buffer; + } + +private: + kj::AsyncIoStream * io_stream; + kj::WaitScope * wait_scope; + kj::Promise promise = nullptr; + + unsigned char *buffer; + size_t buffer_read_size; + + bool ready; +}; diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index 0d9884e..6e506f8 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -2,10 +2,12 @@ from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, P from capnp.includes.schema_cpp cimport ByteArray -from non_circular cimport reraise_kj_exception +from non_circular cimport reraise_kj_exception, AsyncIoStreamReadHelper from cpython.ref cimport PyObject +from libcpp cimport bool + cdef extern from "capnp/helpers/fixMaybe.h": EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception @@ -41,7 +43,9 @@ cdef extern from "capnp/helpers/serialize.h": cdef extern from "capnp/helpers/asyncHelper.h": void waitNeverDone(WaitScope&) + void pollWaitScope(WaitScope&) Response * waitRemote(RemotePromise *, WaitScope&) + bool pollRemote(RemotePromise *, WaitScope&) PyObject * waitPyPromise(PyPromise *, WaitScope&) void waitVoidPromise(VoidPromise *, WaitScope&) Timer * getTimer(AsyncIoContext *) except +reraise_kj_exception diff --git a/capnp/helpers/non_circular.pxd b/capnp/helpers/non_circular.pxd index 58771dc..b0a9ffd 100644 --- a/capnp/helpers/non_circular.pxd +++ b/capnp/helpers/non_circular.pxd @@ -1,4 +1,6 @@ from cpython.ref cimport PyObject +from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope +from libcpp cimport bool cdef extern from "capnp/helpers/capabilityHelper.h": cppclass PythonInterfaceDynamicImpl: @@ -18,3 +20,10 @@ cdef extern from "capnp/helpers/rpcHelper.h": cdef extern from "capnp/helpers/asyncHelper.h": cdef cppclass PyEventPort: PyEventPort(PyObject *) + +cdef extern from "capnp/helpers/asyncIoHelper.h": + cdef cppclass AsyncIoStreamReadHelper: + AsyncIoStreamReadHelper(AsyncIoStream *, WaitScope *, size_t) + bool poll() + size_t read_size() + void* read_buffer() diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index f43956e..450a05e 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -4,6 +4,7 @@ cdef extern from "capnp/helpers/checkCompiler.h": pass +from libcpp cimport bool from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyRestorer, PyEventPort, ErrorHandler from capnp.includes.types cimport * @@ -45,6 +46,7 @@ cdef extern from "kj/exception.h" namespace " ::kj": cdef extern from "kj/memory.h" namespace " ::kj": cdef cppclass Own[T]: T& operator*() + T* get() Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side, ReaderOptions) Own[PromiseFulfillerPair] copyPromiseFulfillerPair" ::kj::heap< ::kj::PromiseFulfillerPair >"(PromiseFulfillerPair&) Own[PyRefCounter] makePyRefCounter" ::kj::heap< PyRefCounter >"(PyObject *) @@ -55,6 +57,7 @@ cdef extern from "kj/async.h" namespace " ::kj": Promise(Promise) Promise(T) T wait(WaitScope) + bool poll(WaitScope) # ForkedPromise fork() # Promise exclusiveJoin(Promise&& other) # Promise[T] eagerlyEvaluate() @@ -121,16 +124,21 @@ cdef inline Duration Nanoseconds(int64_t nanos): cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass AsyncIoStream: - pass + Promise[size_t] read(void*, size_t, size_t) + Promise[void] write(const void*, size_t) + cdef cppclass LowLevelAsyncIoProvider: # Own[AsyncInputStream] wrapInputFd(int) # Own[AsyncOutputStream] wrapOutputFd(int) Own[AsyncIoStream] wrapSocketFd(int) Timer& getTimer() except +reraise_kj_exception + cdef cppclass AsyncIoProvider: - pass + TwoWayPipe newTwoWayPipe() + cdef cppclass WaitScope: pass + cdef cppclass AsyncIoContext: AsyncIoContext(AsyncIoContext&) Own[LowLevelAsyncIoProvider] lowLevelProvider @@ -140,6 +148,9 @@ cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass TaskSet: TaskSet(ErrorHandler &) + cdef cppclass TwoWayPipe: + Own[AsyncIoStream] ends[2] + AsyncIoContext setupAsyncIo() cdef extern from "capnp/schema.capnp.h" namespace " ::capnp": diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index e8dffa5..4d9c7e1 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -2,7 +2,7 @@ from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp -from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder +from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder, TwoWayPipe from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode from capnp.includes.types cimport * from capnp.helpers.non_circular cimport reraise_kj_exception diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index d11c1e0..e5d420e 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -11,6 +11,8 @@ cimport cython from capnp.helpers.helpers cimport makeRpcClientWithRestorer +from capnp.helpers.helpers cimport AsyncIoStreamReadHelper +from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope from libc.stdlib cimport malloc, free from libc.string cimport memcpy @@ -33,6 +35,7 @@ import socket as _socket import random as _random import collections as _collections import array +import asyncio _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR @@ -1655,6 +1658,9 @@ cdef class _EventLoop: del self.thisptr self.thisptr = NULL + cdef TwoWayPipe makeTwoWayPipe(self): + return deref(deref(self.thisptr).provider).newTwoWayPipe() + cdef Own[AsyncIoStream] wrapSocketFd(self, int fd): return deref(deref(self.thisptr).lowLevelProvider).wrapSocketFd(fd) @@ -1738,6 +1744,10 @@ def wait_forever(): cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() helpers.waitNeverDone(deref(loop.thisptr).waitScope) +def poll_once(): + cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() + helpers.pollWaitScope(deref(loop.thisptr).waitScope) + cdef class _CallContext: cdef CallContext * thisptr @@ -1929,11 +1939,24 @@ cdef class _RemotePromise: def __dealloc__(self): del self.thisptr - cpdef wait(self) except +reraise_kj_exception: + cpdef _wait(self) except +reraise_kj_exception: + return _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent) + + def wait(self): if self.is_consumed: raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - ret = _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent) + ret = self._wait() + self.is_consumed = True + return ret + + async def a_wait(self): + if self.is_consumed: + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') + + while not helpers.pollRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope): + await asyncio.sleep(0) + ret = self._wait() self.is_consumed = True return ret @@ -2234,6 +2257,10 @@ cdef class _TwoPartyVatNetwork: self.thisptr = makeTwoPartyVatNetwork(deref(stream.thisptr), side, opts) return self + cdef _init_pipe(self, _TwoWayPipe pipe, Side side, schema_cpp.ReaderOptions opts): + self.thisptr = makeTwoPartyVatNetwork(deref(pipe._pipe.ends[0]), side, opts) + return self + cpdef on_disconnect(self) except +reraise_kj_exception: return _VoidPromise()._init(deref(self.thisptr).onDisconnect(), self) @@ -2255,16 +2282,23 @@ cdef class TwoPartyClient: cdef public object _orig_stream cdef public _Restorer _restorer cdef public _AsyncIoStream _stream + cdef public _TwoWayPipe _pipe - def __init__(self, socket, restorer=None, traversal_limit_in_words=None, nesting_limit=None): + def __init__(self, socket=None, restorer=None, traversal_limit_in_words=None, nesting_limit=None): if isinstance(socket, basestring): socket = self._connect(socket) cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._orig_stream = socket - self._stream = _FdAsyncIoStream(socket.fileno()) - self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT, opts) + if self._orig_stream: + self._stream = _FdAsyncIoStream(socket.fileno()) + self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT, opts) + else: + # Initialize TwoWayPipe, to use pipe() acquire other end of the pipe using read() and write() methods + self._pipe = _TwoWayPipe() + self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.CLIENT, opts) + if restorer is None: self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) self._restorer = None @@ -2274,10 +2308,35 @@ cdef class TwoPartyClient: self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self._network.thisptr), deref(self._restorer.thisptr))) Py_INCREF(self._restorer) - Py_INCREF(self._orig_stream) - Py_INCREF(self._stream) + if self._orig_stream: + Py_INCREF(self._orig_stream) + Py_INCREF(self._stream) + else: + Py_INCREF(self._pipe) Py_INCREF(self._network) # TODO:MEMORY: attach this to onDrained, also figure out what's leaking + async def read(self, bufsize): + cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper( + self._pipe._pipe.ends[1].get(), + &self._pipe._event_loop.thisptr.waitScope, + bufsize + ) + while not reader.poll(): + await asyncio.sleep(0) + + cdef array.array read_buffer = array.array('b', []) + array.resize(read_buffer, reader.read_size()) + memcpy(read_buffer.data.as_voidptr, reader.read_buffer(), reader.read_size()) + del reader + return read_buffer + + def write(self, data): + cdef array.array write_buffer = array.array('b', data) + deref(self._pipe._pipe.ends[1]).write( + write_buffer.data.as_voidptr, + len(data) + ).wait(self._pipe._event_loop.thisptr.waitScope) + def __dealloc__(self): del self.thisptr @@ -2349,12 +2408,13 @@ cdef class TwoPartyServer: cdef public object _orig_stream, _server_socket, _disconnect_promise cdef public _Restorer _restorer cdef public _AsyncIoStream _stream + cdef public _TwoWayPipe _pipe cdef object _port cdef public object port_promise, _bootstrap cdef capnp.TaskSet * _task_set cdef capnp.ErrorHandler _error_handler - def __init__(self, socket, restorer=None, server_socket=None, bootstrap=None, + def __init__(self, socket=None, restorer=None, server_socket=None, bootstrap=None, traversal_limit_in_words=None, nesting_limit=None): if not restorer and not bootstrap: raise KjException("You must provide either a bootstrap interface or a restorer (deperecated) to a server constructor.") @@ -2366,28 +2426,58 @@ cdef class TwoPartyServer: if isinstance(socket, basestring): self._connect(socket, restorer, bootstrap) - else: - self._orig_stream = socket + return + + self._orig_stream = socket + if self._orig_stream: self._stream = _FdAsyncIoStream(socket.fileno()) - self._server_socket = server_socket - self._port = 0 self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER, opts) + else: + # Initialize TwoWayPipe, to use pipe() acquire other end of the pipe using read() and write() methods + self._pipe = _TwoWayPipe() + self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.SERVER, opts) - if bootstrap: - self._bootstrap = bootstrap - schema = bootstrap.schema - self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) - elif restorer: - _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) - self._restorer = _convert_restorer(restorer) - self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr))) + self._server_socket = server_socket + self._port = 0 - Py_INCREF(self._orig_stream) - Py_INCREF(self._stream) - Py_INCREF(self._restorer) - Py_INCREF(self._bootstrap) - Py_INCREF(self._network) - self._disconnect_promise = self.on_disconnect().then(self._decref) + if bootstrap: + self._bootstrap = bootstrap + schema = bootstrap.schema + self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) + elif restorer: + _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) + self._restorer = _convert_restorer(restorer) + self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr))) + + Py_INCREF(self._restorer) + Py_INCREF(self._orig_stream) + Py_INCREF(self._stream) + Py_INCREF(self._pipe) + Py_INCREF(self._bootstrap) + Py_INCREF(self._network) + self._disconnect_promise = self.on_disconnect().then(self._decref) + + async def read(self, bufsize): + cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper( + self._pipe._pipe.ends[1].get(), + &self._pipe._event_loop.thisptr.waitScope, + bufsize + ) + while not reader.poll(): + await asyncio.sleep(0) + + cdef array.array read_buffer = array.array('b', []) + array.resize(read_buffer, reader.read_size()) + memcpy(read_buffer.data.as_voidptr, reader.read_buffer(), reader.read_size()) + del reader + return read_buffer + + async def write(self, data): + cdef array.array write_buffer = array.array('b', data) + deref(self._pipe._pipe.ends[1]).write( + write_buffer.data.as_voidptr, + len(data) + ).wait(self._pipe._event_loop.thisptr.waitScope) cpdef _connect(self, host_string, restorer, bootstrap): cdef _InterfaceSchema schema @@ -2406,6 +2496,7 @@ cdef class TwoPartyServer: def _decref(self): Py_DECREF(self._bootstrap) Py_DECREF(self._restorer) + Py_INCREF(self._pipe) Py_DECREF(self._orig_stream) Py_DECREF(self._stream) Py_DECREF(self._network) @@ -2417,6 +2508,11 @@ cdef class TwoPartyServer: cpdef on_disconnect(self) except +reraise_kj_exception: return _VoidPromise()._init(deref(self._network.thisptr).onDisconnect()) + async def poll_forever(self): + while True: + poll_once() + await asyncio.sleep(0) + cpdef run_forever(self): if self.port_promise is None: raise KjException("You must pass a string as the socket parameter in __init__ to use this function") @@ -2437,6 +2533,18 @@ cdef class TwoPartyServer: cdef class _AsyncIoStream: cdef Own[AsyncIoStream] thisptr +cdef class _TwoWayPipe: + cdef _EventLoop _event_loop + cdef TwoWayPipe _pipe + + def __init__(self): + self._init() + + cpdef _init(self) except +reraise_kj_exception: + self._event_loop = C_DEFAULT_EVENT_LOOP_GETTER() + # Create two way pipe using AsyncIoContext + self._pipe = self._event_loop.makeTwoWayPipe() + cdef class _FdAsyncIoStream(_AsyncIoStream): cdef _EventLoop _event_loop diff --git a/examples/async_client.py b/examples/async_client.py new file mode 100755 index 0000000..df70f27 --- /dev/null +++ b/examples/async_client.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import asyncio +import argparse +import threading +import time +import capnp +import socket + +import thread_capnp + +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + +if __name__ == '__main__': + asyncio.run(main(parse_args().host)) diff --git a/examples/async_server.py b/examples/async_server.py new file mode 100755 index 0000000..1c6d81e --- /dev/null +++ b/examples/async_server.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import capnp + +import thread_capnp +import asyncio +import socket + + +class ExampleImpl(thread_capnp.Example.Server): + + "Implementation of the Example threading Cap'n Proto interface." + + def subscribeStatus(self, subscriber, **kwargs): + return capnp.getTimer().after_delay(10**9) \ + .then(lambda: subscriber.status(True)) \ + .then(lambda _: self.subscribeStatus(subscriber)) + + def longRunning(self, **kwargs): + return capnp.getTimer().after_delay(3 * 10**9) + + +async def myreader(server, reader): + while True: + data = await reader.read(4096) + # Close connection if 0 bytes read + if len(data) == 0: + server.close() + await server.write(data) + + +async def mywriter(server, writer): + while True: + data = await server.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=ExampleImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ) + except: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + family=socket.AF_INET6 + ) + + async with server: + await server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/thread_client.py b/examples/thread_client.py index 7201dd0..459550b 100755 --- a/examples/thread_client.py +++ b/examples/thread_client.py @@ -31,7 +31,7 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): def start_status_thread(host): client = capnp.TwoPartyClient(host) - cap = client.ez_restore('example').cast_as(thread_capnp.Example) + cap = client.bootstrap().cast_as(thread_capnp.Example) subscriber = StatusSubscriber() promise = cap.subscribeStatus(subscriber) @@ -40,7 +40,7 @@ def start_status_thread(host): def main(host): client = capnp.TwoPartyClient(host) - cap = client.ez_restore('example').cast_as(thread_capnp.Example) + cap = client.bootstrap().cast_as(thread_capnp.Example) status_thread = threading.Thread(target=start_status_thread, args=(host,)) status_thread.daemon = True diff --git a/examples/thread_server.py b/examples/thread_server.py index 04b1e2d..a61a767 100755 --- a/examples/thread_server.py +++ b/examples/thread_server.py @@ -31,18 +31,10 @@ given address/port ADDRESS may be '*' to bind to all local addresses.\ return parser.parse_args() -impl = ExampleImpl() - - -def restore(ref): - assert ref.as_text() == 'example' - return impl - - def main(): address = parse_args().address - server = capnp.TwoPartyServer(address, restore) + server = capnp.TwoPartyServer(address, bootstrap=ExampleImpl()) server.run_forever() if __name__ == '__main__': From 1a127bec6f18072b1e1e9af84fd35fea3a0fcf2f Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 17 Sep 2019 00:52:48 -0700 Subject: [PATCH 102/126] Adding pure python SSL test using asyncio - Uses thread.capnp - Follows same format as thread_client.py/thread_server.py and async_client.py/async_server.py - Including a basic self signed certificate for testing convenience - Python 3.7 has a bug cleaning up SSL when using asyncio.run https://bugs.python.org/issue36709 Have a slightly more verbose workaround to do proper cleanup --- examples/async_ssl_client.py | 103 +++++++++++++++++++++++++++++++++++ examples/async_ssl_server.py | 94 ++++++++++++++++++++++++++++++++ examples/selfsigned.cert | 17 ++++++ examples/selfsigned.key | 28 ++++++++++ 4 files changed, 242 insertions(+) create mode 100755 examples/async_ssl_client.py create mode 100755 examples/async_ssl_server.py create mode 100644 examples/selfsigned.cert create mode 100644 examples/selfsigned.key diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py new file mode 100755 index 0000000..6534219 --- /dev/null +++ b/examples/async_ssl_client.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import asyncio +import argparse +import threading +import time +import capnp +import socket +import ssl + +import thread_capnp + +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile='selfsigned.cert') + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + ) + except: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + +if __name__ == '__main__': + # Using asyncio.run hits an asyncio ssl bug + # https://bugs.python.org/issue36709 + #asyncio.run(main(parse_args().host), loop=loop, debug=True) + loop = asyncio.get_event_loop() + loop.run_until_complete(main(parse_args().host)) diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py new file mode 100755 index 0000000..2fcad3b --- /dev/null +++ b/examples/async_ssl_server.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import argparse +import capnp + +import thread_capnp +import asyncio +import socket +import ssl + + +class ExampleImpl(thread_capnp.Example.Server): + + "Implementation of the Example threading Cap'n Proto interface." + + def subscribeStatus(self, subscriber, **kwargs): + return capnp.getTimer().after_delay(10**9) \ + .then(lambda: subscriber.status(True)) \ + .then(lambda _: self.subscribeStatus(subscriber)) + + def longRunning(self, **kwargs): + return capnp.getTimer().after_delay(3 * 10**9) + + +async def myreader(server, reader): + while True: + data = await reader.read(4096) + # Close connection if 0 bytes read + if len(data) == 0: + server.close() + await server.write(data) + + +async def mywriter(server, writer): + while True: + data = await server.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=ExampleImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ctx.load_cert_chain('selfsigned.cert', 'selfsigned.key') + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ssl=ctx, + ) + except: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + ssl=ctx, + family=socket.AF_INET6, + ) + + async with server: + await server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/selfsigned.cert b/examples/selfsigned.cert new file mode 100644 index 0000000..399026c --- /dev/null +++ b/examples/selfsigned.cert @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQDR+CRWUUUdKDANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMTkwOTE3MDczODI2WhcNNDcwMjAyMDczODI2WjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg ++X+utjujNcq/zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXs +czvUTWDoPhvvbxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9 +UNKzf4VQVoa0n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapL +GO2kiPzp36388n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjS +Fcml6J06CITpYeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEON +vWZiM7Ul+9mOPejUMPwlAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAICkFOw0Dd1U +r60rgVpUiHMoFNuBP3ikZfBQ+KXOtfTIYbxbi+iKvvPlB1NFA7Qy3FqqN4sUncvp +QQLxdm+KClM+hvAogng/SyEJJW169vuQbqn5s/1iKCOtGFkI18thCr3rwsI6vTaR +0TmTPtjSQKl5PqcS8kQJTED+CnQhqOAv7C68Bpg+x2dSD9VCq81cPeDbfnK6gico +29qJYUm4RCXMicrzvEwNObx06TQKJb/pWjpl1NAmpFvcz+2MYPL/QTfH/cS5lhgx +KCe4/kDO0HCueOi2MqBFaO2B0kZxanMoZ2KZe2b/Bp1CTJzXCXNYWQj6QbLzZxaD +fOp0J8wAo1U= +-----END CERTIFICATE----- diff --git a/examples/selfsigned.key b/examples/selfsigned.key new file mode 100644 index 0000000..296d799 --- /dev/null +++ b/examples/selfsigned.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCg+X+utjujNcq/ +zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXsczvUTWDoPhvv +bxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9UNKzf4VQVoa0 +n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapLGO2kiPzp3638 +8n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjSFcml6J06CITp +YeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEONvWZiM7Ul+9mO +PejUMPwlAgMBAAECggEAD6Vwlai8zzZRSKc7Vf98LI3dDRkRVS3XLf/uSluNlo7e +o9Iyz8fOypA8GT2NwGKNyvfAXvhObQRsbq9bvXhvhJLRKde2m5x7vovZ3mztOj63 +f/kwHSjC5hksgjxC8NFtGBadBDlm2dIvMasxk7bbr4tn36orbr0NPGMTm0C/Md8B +bSUzuc/mT+6KWfW9g4svqebSbKvC7tGuAu3/RfL1cmbuuvtJJA+EPfRjgCtOiFSk +8NLE0KLUYySf6M3MAHMeSwQhVr1xyYUkiOqQoxMC7CpaplNqaB2rrOe2nEjLpXgx +80WLFbB22HNEkBSgX5zz4FmrLchwaI79f3PiGQ2DWQKBgQDSIsl1t13eR8ubsGU5 +Z097U8/eylyXJC+ZU1/TbWgaPNHlMf26EncSJwlmq7pNJ0Rk7p/edPMWKu9tOgQE +iG0QeKxvxcbc1LzVmfLKXY8hW4DOhiBTiRNNb+YWuYmKYnHXM8hrKthNXBFfRrwb +Pb+mid9FcK4GcIDkWbgXEahmwwKBgQDEG9sB7Ee4fDiQi1oVsPkMwxnxsiNpSRRG +9CmG/xIvL81vSaONPmg1f5q5/Wqkd6s3QmMs4rE5U8+kgRvuw/MnN/UxQBH5/LAn +T3hbo7qlIONOOpgQswg7wIQnKP31spNU2FthA1ACazRBsIyH+hPmgeqvtvFhL4Qy +6q5tfhFy9wKBgGYZtu82aCqPkdOU0qogk1Ll9zNV+cUKNQJ3qzDMkO9mq8mED7cw +L6CnTP8Q45WHRckQ1Ka/Bjm4JNtafAdDzlJZf9dTLnuv9gyHH5vJ97iKgDxYmS5d +hP50J0TVY4nUqWGZ7IB9sdlsqZg0g0NtLkiZ5t0TkcrZMRdCrJqw3rUHAoGBAJNk +wEmEti8Rpk31fsK43aba6KABHJ5gX84oayHcimVOz1/qf/ODyT0UaE2MC2AL1XLW +AcZVp5AHzxO8OitNuW5rn2zh0+EJK7iQAU0XFQxRWKaOYYaDmReXzXvFUoMdMaDe +cGfM3pDC1Gbe8/CrY9OnJ6XjoS5DUWAXhPwkeabnAoGBAMmI9xcyfcPjO6XKKymZ +z+6bwvFaey9Acy5vkxLrEcRqH8pRR09CltWzR4vhpJwHABWPHVGqdtvlLw1/lUrj +xr4RXvYXK28cK9Tdak0c3+HPSsozxrsX6AQzcG27ymo5s0KYaVuxWh98iZU7Eb52 +YfCx3eOSIHPH0ay7KBVAQocG +-----END PRIVATE KEY----- From db4e567c6bbbc8b66b12655dbe5206a3a709391e Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Wed, 18 Sep 2019 19:01:46 -0700 Subject: [PATCH 103/126] Upgrading to capnproto-0.7.0 - Needed for asyncio support --- buildutils/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index b0e7f0e..18d0cdc 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,7 +35,7 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,6,1) +bundled_version = (0,7,0) libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp From 1f0200af9ceb20f7c7fe56d260baecdb33383f8d Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Thu, 26 Sep 2019 21:54:12 -0700 Subject: [PATCH 104/126] Unlocking python package version requirements --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8712fde..99f2720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -jinja2 >= 2.7.3 -cython == 0.21.2 -setuptools >= 0.8 +jinja2 +cython +setuptools pytest tox From b3021e4f6bab0952790ea5b83901cafca83cbba4 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Thu, 26 Sep 2019 22:18:28 -0700 Subject: [PATCH 105/126] Fixing flake8 warnings and errors flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude benchmark Excluding the benchmark directory (due to protobuf generated files) Also removing some Python2 specific code --- buildutils/__init__.py | 1 + buildutils/build.py | 45 ++++++------ buildutils/bundle.py | 43 ++++++----- buildutils/config.py | 28 +++---- buildutils/constants.py | 5 +- buildutils/detect.py | 24 +++--- buildutils/misc.py | 11 +-- buildutils/msg.py | 14 ++-- buildutils/patch.py | 11 +-- capnp/__init__.py | 20 ++++- capnp/_gen.py | 94 ++++++++++++------------ docs/conf.py | 126 ++++++++++++++++---------------- examples/addressbook.py | 3 +- examples/async_client.py | 92 ++++++++++++----------- examples/async_server.py | 3 +- examples/async_ssl_client.py | 116 +++++++++++++++-------------- examples/async_ssl_server.py | 2 +- examples/calculator_client.py | 2 +- examples/calculator_server.py | 3 +- examples/thread_client.py | 41 ++++++----- examples/thread_server.py | 1 + scripts/capnp-json.py | 7 +- scripts/capnp_test_pycapnp.py | 8 +- setup.py | 66 ++++++++++++----- test/test_capability.py | 8 +- test/test_capability_context.py | 12 +-- test/test_capability_old.py | 12 +-- test/test_large_read.py | 27 ++----- test/test_load.py | 14 ++-- test/test_object.py | 2 +- test/test_regression.py | 22 +++--- test/test_response.py | 5 -- test/test_rpc.py | 13 ++-- test/test_rpc_calculator.py | 16 ++-- test/test_serialization.py | 15 +++- test/test_struct.py | 23 +++--- test/test_threads.py | 53 +++++++++++--- 37 files changed, 536 insertions(+), 452 deletions(-) diff --git a/buildutils/__init__.py b/buildutils/__init__.py index 1da698d..65bb10d 100644 --- a/buildutils/__init__.py +++ b/buildutils/__init__.py @@ -2,6 +2,7 @@ Largely adapted from h5py """ +# flake8: noqa F401 F403 from .msg import * from .config import * diff --git a/buildutils/build.py b/buildutils/build.py index 82ffaab..5797c46 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -5,26 +5,29 @@ import os import tempfile def build_libcapnp(bundle_dir, build_dir, verbose=False): - bundle_dir = os.path.abspath(bundle_dir) - capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') - build_dir = os.path.abspath(build_dir) + ''' + Build capnproto + ''' + bundle_dir = os.path.abspath(bundle_dir) + capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') + build_dir = os.path.abspath(build_dir) - with tempfile.TemporaryFile() as f: - stdout = f - if verbose: - stdout = None - cxxflags = os.environ.get('CXXFLAGS', None) - os.environ['CXXFLAGS'] = (cxxflags or '') + ' -fPIC -O2 -DNDEBUG' - conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout) - returncode = conf.wait() - if returncode != 0: - raise RuntimeError('Configure failed') + with tempfile.TemporaryFile() as f: + stdout = f + if verbose: + stdout = None + cxxflags = os.environ.get('CXXFLAGS', None) + os.environ['CXXFLAGS'] = (cxxflags or '') + ' -fPIC -O2 -DNDEBUG' + conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout) + returncode = conf.wait() + if returncode != 0: + raise RuntimeError('Configure failed') - make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout) - returncode = make.wait() - if cxxflags is None: - del os.environ['CXXFLAGS'] - else: - os.environ['CXXFLAGS'] = cxxflags - if returncode != 0: - raise RuntimeError('Make failed') + make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout) + returncode = make.wait() + if cxxflags is None: + del os.environ['CXXFLAGS'] + else: + os.environ['CXXFLAGS'] = cxxflags + if returncode != 0: + raise RuntimeError('Make failed') diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 18d0cdc..817797b 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -1,12 +1,11 @@ """utilities for fetching build dependencies.""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # Distributed under the terms of the Modified BSD License. # # This bundling code is largely adapted from pyzmq-static's get.sh by # Brandon Craig-Rhodes, which is itself BSD licensed. -#----------------------------------------------------------------------------- # # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq # for original project. @@ -17,7 +16,6 @@ import shutil import stat import sys import tarfile -from glob import glob from subprocess import Popen, PIPE try: @@ -27,27 +25,28 @@ except ImportError: # py3 from urllib.request import urlopen -from .msg import fatal, debug, info, warn +from .msg import fatal, info, warn pjoin = os.path.join -#----------------------------------------------------------------------------- +# # Constants -#----------------------------------------------------------------------------- +# -bundled_version = (0,7,0) -libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) -libcapnp_url = "https://capnproto.org/" + libcapnp +bundled_version = (0, 7, 4) +libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) +libcapnp_url = "https://capnproto.org/" + libcapnp_name HERE = os.path.dirname(__file__) ROOT = os.path.dirname(HERE) -#----------------------------------------------------------------------------- +# # Utilities -#----------------------------------------------------------------------------- +# def untgz(archive): + """Remove .tar.gz""" return archive.replace('.tar.gz', '') def localpath(*args): @@ -69,9 +68,9 @@ def fetch_archive(savedir, url, fname, force=False): f.write(req.read()) return dest -#----------------------------------------------------------------------------- +# # libcapnp -#----------------------------------------------------------------------------- +# def fetch_libcapnp(savedir, url=None): """download and extract libcapnp""" @@ -83,7 +82,7 @@ def fetch_libcapnp(savedir, url=None): if os.path.exists(dest): info("already have %s" % dest) return - fname = fetch_archive(savedir, url, libcapnp) + fname = fetch_archive(savedir, url, libcapnp_name) tf = tarfile.open(fname) with_version = pjoin(savedir, tf.firstmember.path) tf.extractall(savedir) @@ -96,7 +95,7 @@ def fetch_libcapnp(savedir, url=None): conf = Popen(['autoreconf', '-i'], cwd=cpp_dir) returncode = conf.wait() if returncode != 0: - raise RuntimeError('Autoreconf failed. Make sure autotools are installed on your system.') + raise RuntimeError('Autoreconf failed. Make sure autotools are installed on your system.') shutil.move(cpp_dir, dest) @@ -120,7 +119,7 @@ def stage_platform_hpp(capnproot): p = Popen('./configure', cwd=capnproot, shell=True, stdout=PIPE, stderr=PIPE, ) - o,e = p.communicate() + _, e = p.communicate() if p.returncode: warn("failed to configure libcapnp:\n%s" % e) if sys.platform == 'darwin': @@ -146,14 +145,14 @@ def copy_and_patch_libcapnp(capnp, libcapnp): if sys.platform.startswith('win'): return # copy libcapnp into capnp for bdist - local = localpath('capnp',libcapnp) + local = localpath('capnp', libcapnp) if not capnp and not os.path.exists(local): fatal("Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` " "or copy libcapnp into capnp/ manually prior to running bdist.") try: # resolve real file through symlinks lib = os.path.realpath(pjoin(capnp, 'lib', libcapnp)) - print ("copying %s -> %s"%(lib, local)) + print ("copying %s -> %s" % (lib, local)) shutil.copy(lib, local) except Exception: if not os.path.exists(local): @@ -167,11 +166,11 @@ def copy_and_patch_libcapnp(capnp, libcapnp): mode = os.stat(local).st_mode os.chmod(local, mode | stat.S_IWUSR) # patch install_name on darwin, instead of using rpath - cmd = ['install_name_tool', '-id', '@loader_path/../%s'%libcapnp, local] + cmd = ['install_name_tool', '-id', '@loader_path/../%s' % libcapnp, local] try: - p = Popen(cmd, stdout=PIPE,stderr=PIPE) + p = Popen(cmd, stdout=PIPE, stderr=PIPE) except OSError: fatal("install_name_tool not found, cannot patch libcapnp for bundling.") - out,err = p.communicate() + _, err = p.communicate() if p.returncode: - fatal("Could not patch bundled libcapnp install_name: %s"%err, p.returncode) + fatal("Could not patch bundled libcapnp install_name: %s" % err, p.returncode) diff --git a/buildutils/config.py b/buildutils/config.py index c674655..6701259 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -1,5 +1,5 @@ """Config functions""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # # This file is part of pyzmq, copied and adapted from h5py. @@ -9,23 +9,24 @@ # # Distributed under the terms of the New BSD License. The full license is in # the file COPYING.BSD, distributed as part of this software. -#----------------------------------------------------------------------------- +# import sys import os import json +from .msg import debug, warn + try: from configparser import ConfigParser -except: +except Exception: from ConfigParser import ConfigParser pjoin = os.path.join -from .msg import debug, fatal, warn -#----------------------------------------------------------------------------- +# # Utility functions (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- +# def load_config(name, base='conf'): @@ -46,7 +47,7 @@ def save_config(name, data, base='conf'): """Save config dict to JSON""" if not os.path.exists(base): os.mkdir(base) - fname = pjoin(base, name+'.json') + fname = pjoin(base, name + '.json') with open(fname, 'w') as f: json.dump(data, f, indent=2) @@ -69,7 +70,7 @@ def get_eargs(): def cfg2dict(cfg): """turn a ConfigParser into a nested dict - + because ConfigParser objects are dumb. """ d = {} @@ -120,7 +121,7 @@ def config_from_prefix(prefix): def merge(into, d): """merge two containers - + into is updated, d has priority """ if isinstance(into, dict): @@ -130,10 +131,9 @@ def merge(into, d): else: into[key] = merge(into[key], d[key]) return into - elif isinstance(into, list): + if isinstance(into, list): return into + d - else: - return d + return d def discover_settings(conf_base=None): """ Discover custom settings for ZMQ path""" @@ -147,11 +147,11 @@ def discover_settings(conf_base=None): } if sys.platform.startswith('win'): settings['have_sys_un_h'] = False - + if conf_base: # lowest priority merge(settings, load_config('config', conf_base)) merge(settings, get_cfg_args()) merge(settings, get_eargs()) - + return settings diff --git a/buildutils/constants.py b/buildutils/constants.py index e98c650..fa39722 100644 --- a/buildutils/constants.py +++ b/buildutils/constants.py @@ -23,7 +23,7 @@ pjoin = os.path.join root = os.path.abspath(pjoin(os.path.dirname(__file__), os.path.pardir)) sys.path.insert(0, pjoin(root, 'zmq', 'utils')) -from constant_names import all_names, no_prefix +from constant_names import all_names, no_prefix # noqa: E402 ifndef_t = """#ifndef {0} #define {0} (_PYZMQ_UNDEFINED) @@ -38,7 +38,7 @@ def cython_enums(): lines.append('enum: ZMQ_{0} "{0}"'.format(name)) else: lines.append('enum: ZMQ_{0}'.format(name)) - + return dict(ZMQ_ENUMS='\n '.join(lines)) def ifndefs(): @@ -79,5 +79,6 @@ def render_constants(): generate_file("constants.pxi", constants_pyx, pjoin(root, 'zmq', 'backend', 'cython')) generate_file("zmq_constants.h", ifndefs, pjoin(root, 'zmq', 'utils')) + if __name__ == '__main__': render_constants() diff --git a/buildutils/detect.py b/buildutils/detect.py index 7810380..19459b1 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -1,5 +1,5 @@ """Detect zmq version""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # # This file is part of pyzmq, copied and adapted from h5py. @@ -9,7 +9,7 @@ # # Distributed under the terms of the New BSD License. The full license is in # the file COPYING.BSD, distributed as part of this software. -#----------------------------------------------------------------------------- +# # # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq # for original project. @@ -29,20 +29,20 @@ from .patch import patch_lib_paths pjoin = os.path.join -#----------------------------------------------------------------------------- +# # Utility functions (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- +# def test_compilation(cfile, compiler=None, **compiler_attrs): """Test simple compilation with given settings""" cc = get_compiler(compiler, **compiler_attrs) - efile, ext = os.path.splitext(cfile) + efile, _ = os.path.splitext(cfile) cpreargs = lpreargs = [] if sys.platform == 'darwin': # use appropriate arch for compiler - if platform.architecture()[0]=='32bit': + if platform.architecture()[0] == '32bit': if platform.processor() == 'powerpc': cpu = 'ppc' else: @@ -53,7 +53,7 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): # allow for missing UB arch, since it will still work: lpreargs = ['-undefined', 'dynamic_lookup'] if sys.platform == 'sunos5': - if platform.architecture()[0]=='32bit': + if platform.architecture()[0] == '32bit': lpreargs = ['-m32'] else: lpreargs = ['-m64'] @@ -65,6 +65,7 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): return efile def compile_and_run(basedir, src, compiler=None, **compiler_attrs): + """Compile and run""" if not os.path.exists(basedir): os.makedirs(basedir) cfile = pjoin(basedir, os.path.basename(src)) @@ -130,11 +131,11 @@ def detect_version(basedir, compiler=None, **compiler_attrs): rc, so, se = get_output_error([efile]) if rc: - msg = "Error running version detection script:\n%s\n%s" % (so,se) + msg = "Error running version detection script:\n%s\n%s" % (so, se) logging.error(msg) raise IOError(msg) - handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))} + handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))} props = {} for line in (x for x in so.split('\n') if x): @@ -161,8 +162,9 @@ def test_build(): return detected -def erase_dir(dir): +def erase_dir(path): + """Erase directory""" try: - shutil.rmtree(dir) + shutil.rmtree(path) except Exception: pass diff --git a/buildutils/misc.py b/buildutils/misc.py index 5721ac1..fabb52d 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -4,7 +4,6 @@ # Distributed under the terms of the Modified BSD License. import os -import sys import logging from distutils import ccompiler from distutils.sysconfig import customize_compiler @@ -14,13 +13,8 @@ from subprocess import Popen, PIPE pjoin = os.path.join -if sys.version_info[0] >= 3: - u = lambda x: x -else: - u = lambda x: x.decode('utf8', 'replace') - - def customize_mingw(cc): + """customize mingw""" # strip -mno-cygwin from mingw32 (Python Issue #12641) for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]: if '-mno-cygwin' in cmd: @@ -55,11 +49,10 @@ def get_output_error(cmd): try: result = Popen(cmd, stdout=PIPE, stderr=PIPE) except IOError as e: - return -1, u(''), u('Failed to run %r: %r' % (cmd, e)) + return -1, '', 'Failed to run %r: %r' % (cmd, e) so, se = result.communicate() # unicode: so = so.decode('utf8', 'replace') se = se.decode('utf8', 'replace') return result.returncode, so, se - diff --git a/buildutils/msg.py b/buildutils/msg.py index 70cd716..63f2e51 100644 --- a/buildutils/msg.py +++ b/buildutils/msg.py @@ -9,9 +9,9 @@ import os import sys import logging -#----------------------------------------------------------------------------- +# # Logging (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- +# logger = logging.getLogger() @@ -22,18 +22,22 @@ else: logger.addHandler(logging.StreamHandler(sys.stderr)) def debug(msg): + """Debug""" logger.debug(msg) def info(msg): + """Info""" logger.info(msg) def fatal(msg, code=1): - logger.error("Fatal: " + msg) + """Fatal""" + logger.error("Fatal: %s", msg) exit(code) def warn(msg): - logger.error("Warning: " + msg) + """Warning""" + logger.error("Warning: %s", msg) def line(c='*', width=48): + """Horizontal rule""" print(c * (width // len(c))) - diff --git a/buildutils/patch.py b/buildutils/patch.py index 925b67b..dc58ac2 100644 --- a/buildutils/patch.py +++ b/buildutils/patch.py @@ -20,7 +20,7 @@ LIB_PAT = re.compile(r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), " def _get_libs(fname): rc, so, se = get_output_error(['otool', '-L', fname]) if rc: - logging.error("otool -L %s failed: %r" % (fname, se)) + logging.error("otool -L %s failed: %r", fname, se) return for line in so.splitlines()[1:]: m = LIB_PAT.match(line) @@ -33,6 +33,7 @@ def _find_library(lib, path): real_lib = os.path.join(d, lib) if os.path.exists(real_lib): return real_lib + return None def _install_name_change(fname, lib, real_lib): rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname]) @@ -41,15 +42,15 @@ def _install_name_change(fname, lib, real_lib): def patch_lib_paths(fname, library_dirs): """Load any weakly-defined libraries from their real location - + (only on OS X) - + - Find libraries with `otool -L` - Update with `install_name_tool -change` """ if sys.platform != 'darwin': return - + libs = _get_libs(fname) for lib in libs: if not lib.startswith(('@', '/')): @@ -58,4 +59,4 @@ def patch_lib_paths(fname, library_dirs): _install_name_change(fname, lib, real_lib) -__all__ = ['patch_lib_paths'] \ No newline at end of file +__all__ = ['patch_lib_paths'] diff --git a/capnp/__init__.py b/capnp/__init__.py index 2a04e92..7d36a9a 100644 --- a/capnp/__init__.py +++ b/capnp/__init__.py @@ -31,8 +31,26 @@ Example Usage:: for phone in person.phones: print(phone.type, ':', phone.number) """ +# flake8: noqa F401 F403 F405 from .version import version as __version__ from .lib.capnp import * -from .lib.capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicResizableListBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan, _DynamicResizableListBuilder, _MallocMessageBuilder, _PackedFdMessageReader, _StreamFdMessageReader, _write_message_to_fd, _write_packed_message_to_fd, _StructModule, _InterfaceModule, _DynamicCapabilityClient, _CapabilityClient, _EventLoop +from .lib.capnp import ( + _CapabilityClient, + _DynamicCapabilityClient, + _DynamicListBuilder, + _DynamicListReader, + _DynamicOrphan, + _DynamicResizableListBuilder, + _DynamicStructBuilder, + _DynamicStructReader, + _EventLoop, + _InterfaceModule, + _MallocMessageBuilder, + _PackedFdMessageReader, + _StreamFdMessageReader, + _StructModule, + _write_message_to_fd, + _write_packed_message_to_fd, +) add_import_hook() # enable import hook by default diff --git a/capnp/_gen.py b/capnp/_gen.py index 5cb3eee..52087c1 100644 --- a/capnp/_gen.py +++ b/capnp/_gen.py @@ -7,58 +7,58 @@ from jinja2 import Environment, PackageLoader import os def find_type(code, id): - for node in code['nodes']: - if node['id'] == id: - return node + for node in code['nodes']: + if node['id'] == id: + return node - return None + return None def main(): - env = Environment(loader=PackageLoader('capnp', 'templates')) - env.filters['format_name'] = lambda name: name[name.find(':')+1:] + env = Environment(loader=PackageLoader('capnp', 'templates')) + env.filters['format_name'] = lambda name: name[name.find(':') + 1:] - code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) - code=code.to_dict() - code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] - for node in code['nodes']: - displayName = node['displayName'] - parent, path = displayName.split(':') - node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) - node['module_name'] = path.replace('.', '_') - node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) - node['schema'] = '_{}_Schema'.format(node['module_name']) - is_union = False - for field in node['struct']['fields']: - if field['discriminantValue'] != 65535: - is_union = True - field['c_name'] = field['name'][0].upper() + field['name'][1:] - if 'slot' in field: - field['type'] = field['slot']['type'].keys()[0] - if not isinstance(field['slot']['type'][field['type']], dict): - continue - sub_type = field['slot']['type'][field['type']].get('typeId', None) - if sub_type: - field['sub_type'] = find_type(code, sub_type) - sub_type = field['slot']['type'][field['type']].get('elementType', None) - if sub_type: - field['sub_type'] = sub_type - else: - field['type'] = find_type(code, field['group']['typeId']) - node['is_union'] = is_union + code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) + code = code.to_dict() + code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] + for node in code['nodes']: + displayName = node['displayName'] + parent, path = displayName.split(':') + node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['module_name'] = path.replace('.', '_') + node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['schema'] = '_{}_Schema'.format(node['module_name']) + is_union = False + for field in node['struct']['fields']: + if field['discriminantValue'] != 65535: + is_union = True + field['c_name'] = field['name'][0].upper() + field['name'][1:] + if 'slot' in field: + field['type'] = field['slot']['type'].keys()[0] + if not isinstance(field['slot']['type'][field['type']], dict): + continue + sub_type = field['slot']['type'][field['type']].get('typeId', None) + if sub_type: + field['sub_type'] = find_type(code, sub_type) + sub_type = field['slot']['type'][field['type']].get('elementType', None) + if sub_type: + field['sub_type'] = sub_type + else: + field['type'] = find_type(code, field['group']['typeId']) + node['is_union'] = is_union - include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) - module = env.get_template('module.pyx') + include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) + module = env.get_template('module.pyx') - for f in code['requestedFiles']: - filename = f['filename'].replace('.', '_') + '_cython.pyx' + for f in code['requestedFiles']: + filename = f['filename'].replace('.', '_') + '_cython.pyx' - file_code = dict(code) - file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] - with open(filename, 'w') as out: - out.write(module.render(code=file_code, file=f, include_dir=include_dir)) + file_code = dict(code) + file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] + with open(filename, 'w') as out: + out.write(module.render(code=file_code, file=f, include_dir=include_dir)) - setup = env.get_template('setup.py.tmpl') - with open('setup_capnp.py', 'w') as out: - out.write(setup.render(code=code)) - print('You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.') - print() + setup = env.get_template('setup.py.tmpl') + with open('setup_capnp.py', 'w') as out: + out.write(setup.render(code=code)) + print('You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.') + print() diff --git a/docs/conf.py b/docs/conf.py index 08eed2d..948267a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,6 @@ +''' +Docs configuration +''' # -*- coding: utf-8 -*- # # capnp documentation build configuration file, created by @@ -11,17 +14,19 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os, string +import string +# import sys, os +import capnp # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. @@ -34,7 +39,7 @@ templates_path = ['_templates'] source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' @@ -48,47 +53,46 @@ copyright = u'2013, Author' # built documents. # # The short X.Y version. -import capnp vs = capnp.__version__ # The short X.Y version. -version = vs.rstrip(string.letters) +version = vs.rstrip(string.ascii_letters) # The full version, including alpha/beta/rc tags. release = vs # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -100,26 +104,26 @@ html_theme = 'nature' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -128,44 +132,44 @@ html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'capnpdoc' @@ -173,43 +177,43 @@ htmlhelp_basename = 'capnpdoc' # -- Options for LaTeX output -------------------------------------------------- -latex_elements = { # The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', +# 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', +# 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. -#'preamble': '', +# 'preamble': '', +latex_elements = { } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'capnp.tex', u'capnp Documentation', - u'Author', 'manual'), + ('index', 'capnp.tex', u'capnp Documentation', + u'Author', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -222,7 +226,7 @@ man_pages = [ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -231,19 +235,19 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'capnp', u'capnp Documentation', - u'Author', 'capnp', 'One line description of project.', - 'Miscellaneous'), + ('index', 'capnp', u'capnp Documentation', + u'Author', 'capnp', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # -- Options for Epub output --------------------------------------------------- @@ -256,36 +260,36 @@ epub_copyright = u'2013, Author' # The language of the text. It defaults to the language option # or en if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -#epub_exclude_files = [] +# epub_exclude_files = [] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/examples/addressbook.py b/examples/addressbook.py index c1ca793..b6ae64f 100755 --- a/examples/addressbook.py +++ b/examples/addressbook.py @@ -1,6 +1,5 @@ from __future__ import print_function -import os -import capnp +import capnp # noqa: F401 import addressbook_capnp diff --git a/examples/async_client.py b/examples/async_client.py index df70f27..6808faa 100755 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -4,7 +4,6 @@ from __future__ import print_function import asyncio import argparse -import threading import time import capnp import socket @@ -16,15 +15,14 @@ capnp.create_event_loop(threaded=True) def parse_args(): - parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ at the given address and does some RPCs') - parser.add_argument("host", help="HOST:PORT") + parser.add_argument("host", help="HOST:PORT") - return parser.parse_args() + return parser.parse_args() class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): - '''An implementation of the StatusSubscriber interface''' def status(self, value, **kwargs): @@ -32,61 +30,61 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): async def myreader(client, reader): - while True: - data = await reader.read(4096) - client.write(data) + while True: + data = await reader.read(4096) + client.write(data) async def mywriter(client, writer): - while True: - data = await client.read(4096) - writer.write(data.tobytes()) - await writer.drain() + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() async def background(cap): - subscriber = StatusSubscriber() - promise = cap.subscribeStatus(subscriber) - await promise.a_wait() + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() async def main(host): - host = host.split(':') - addr = host[0] - port = host[1] - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - reader, writer = await asyncio.open_connection( - addr, port, - ) - except: - print("Try IPv6") - reader, writer = await asyncio.open_connection( - addr, port, - family=socket.AF_INET6 - ) + host = host.split(':') + addr = host[0] + port = host[1] + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) - # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) - client = capnp.TwoPartyClient() - cap = client.bootstrap().cast_as(thread_capnp.Example) + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) - # Assemble reader and writer tasks, run in the background - coroutines = [myreader(client, reader), mywriter(client, writer)] - asyncio.gather(*coroutines, return_exceptions=True) + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) - # Start background task for subscriber - tasks = [background(cap)] - asyncio.gather(*tasks, return_exceptions=True) + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) - # Run blocking tasks - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) if __name__ == '__main__': asyncio.run(main(parse_args().host)) diff --git a/examples/async_server.py b/examples/async_server.py index 1c6d81e..5030ec3 100755 --- a/examples/async_server.py +++ b/examples/async_server.py @@ -72,7 +72,7 @@ async def main(): myserver, addr, port, ) - except: + except Exception: print("Try IPv6") server = await asyncio.start_server( myserver, @@ -83,5 +83,6 @@ async def main(): async with server: await server.serve_forever() + if __name__ == '__main__': asyncio.run(main()) diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py index 6534219..a95d03a 100755 --- a/examples/async_ssl_client.py +++ b/examples/async_ssl_client.py @@ -4,7 +4,6 @@ from __future__ import print_function import asyncio import argparse -import threading import time import capnp import socket @@ -17,87 +16,86 @@ capnp.create_event_loop(threaded=True) def parse_args(): - parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ at the given address and does some RPCs') - parser.add_argument("host", help="HOST:PORT") + parser.add_argument("host", help="HOST:PORT") - return parser.parse_args() + return parser.parse_args() class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + '''An implementation of the StatusSubscriber interface''' - '''An implementation of the StatusSubscriber interface''' - - def status(self, value, **kwargs): - print('status: {}'.format(time.time())) + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) async def myreader(client, reader): - while True: - data = await reader.read(4096) - client.write(data) + while True: + data = await reader.read(4096) + client.write(data) async def mywriter(client, writer): - while True: - data = await client.read(4096) - writer.write(data.tobytes()) - await writer.drain() + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() async def background(cap): - subscriber = StatusSubscriber() - promise = cap.subscribeStatus(subscriber) - await promise.a_wait() + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() async def main(host): - host = host.split(':') - addr = host[0] - port = host[1] + host = host.split(':') + addr = host[0] + port = host[1] - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile='selfsigned.cert') + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile='selfsigned.cert') - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - reader, writer = await asyncio.open_connection( - addr, port, - ssl=ctx, - ) - except: - print("Try IPv6") - reader, writer = await asyncio.open_connection( - addr, port, - ssl=ctx, - family=socket.AF_INET6 - ) + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + family=socket.AF_INET6 + ) - # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) - client = capnp.TwoPartyClient() - cap = client.bootstrap().cast_as(thread_capnp.Example) + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) - # Assemble reader and writer tasks, run in the background - coroutines = [myreader(client, reader), mywriter(client, writer)] - asyncio.gather(*coroutines, return_exceptions=True) + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) - # Start background task for subscriber - tasks = [background(cap)] - asyncio.gather(*tasks, return_exceptions=True) + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) - # Run blocking tasks - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) - await cap.longRunning().a_wait() - print('main: {}'.format(time.time())) + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) if __name__ == '__main__': - # Using asyncio.run hits an asyncio ssl bug - # https://bugs.python.org/issue36709 - #asyncio.run(main(parse_args().host), loop=loop, debug=True) - loop = asyncio.get_event_loop() - loop.run_until_complete(main(parse_args().host)) + # Using asyncio.run hits an asyncio ssl bug + # https://bugs.python.org/issue36709 + # asyncio.run(main(parse_args().host), loop=loop, debug=True) + loop = asyncio.get_event_loop() + loop.run_until_complete(main(parse_args().host)) diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py index 2fcad3b..ebbcf47 100755 --- a/examples/async_ssl_server.py +++ b/examples/async_ssl_server.py @@ -78,7 +78,7 @@ async def main(): addr, port, ssl=ctx, ) - except: + except Exception: print("Try IPv6") server = await asyncio.start_server( myserver, diff --git a/examples/calculator_client.py b/examples/calculator_client.py index de246c3..f6569e4 100755 --- a/examples/calculator_client.py +++ b/examples/calculator_client.py @@ -2,7 +2,6 @@ from __future__ import print_function import argparse -import socket import capnp import calculator_capnp @@ -302,5 +301,6 @@ def main(host): print("PASS") + if __name__ == '__main__': main(parse_args().host) diff --git a/examples/calculator_server.py b/examples/calculator_server.py index f072f58..32af680 100755 --- a/examples/calculator_server.py +++ b/examples/calculator_server.py @@ -2,8 +2,6 @@ from __future__ import print_function import argparse -import socket -import random import capnp import calculator_capnp @@ -136,5 +134,6 @@ def main(): server = capnp.TwoPartyServer(address, bootstrap=CalculatorImpl()) server.run_forever() + if __name__ == '__main__': main() diff --git a/examples/thread_client.py b/examples/thread_client.py index 459550b..fdd50ad 100755 --- a/examples/thread_client.py +++ b/examples/thread_client.py @@ -14,11 +14,11 @@ capnp.create_event_loop(threaded=True) def parse_args(): - parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ at the given address and does some RPCs') - parser.add_argument("host", help="HOST:PORT") + parser.add_argument("host", help="HOST:PORT") - return parser.parse_args() + return parser.parse_args() class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): @@ -30,29 +30,30 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): def start_status_thread(host): - client = capnp.TwoPartyClient(host) - cap = client.bootstrap().cast_as(thread_capnp.Example) + client = capnp.TwoPartyClient(host) + cap = client.bootstrap().cast_as(thread_capnp.Example) - subscriber = StatusSubscriber() - promise = cap.subscribeStatus(subscriber) - promise.wait() + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + promise.wait() def main(host): - client = capnp.TwoPartyClient(host) - cap = client.bootstrap().cast_as(thread_capnp.Example) + client = capnp.TwoPartyClient(host) + cap = client.bootstrap().cast_as(thread_capnp.Example) - status_thread = threading.Thread(target=start_status_thread, args=(host,)) - status_thread.daemon = True - status_thread.start() + status_thread = threading.Thread(target=start_status_thread, args=(host,)) + status_thread.daemon = True + status_thread.start() + + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) if __name__ == '__main__': main(parse_args().host) diff --git a/examples/thread_server.py b/examples/thread_server.py index a61a767..142bd1f 100755 --- a/examples/thread_server.py +++ b/examples/thread_server.py @@ -37,5 +37,6 @@ def main(): server = capnp.TwoPartyServer(address, bootstrap=ExampleImpl()) server.run_forever() + if __name__ == '__main__': main() diff --git a/scripts/capnp-json.py b/scripts/capnp-json.py index f41a05e..cc385d7 100755 --- a/scripts/capnp-json.py +++ b/scripts/capnp-json.py @@ -18,7 +18,7 @@ def encode(schema_file, struct_name, **kwargs): schema = capnp.load(schema_file) struct_schema = getattr(schema, struct_name) - + struct_dict = json.load(sys.stdin) struct = struct_schema.from_dict(struct_dict) @@ -29,7 +29,7 @@ def decode(schema_file, struct_name, defaults): struct_schema = getattr(schema, struct_name) struct = struct_schema.read(sys.stdin) - + json.dump(struct.to_dict(defaults), sys.stdout) def main(): @@ -41,4 +41,5 @@ def main(): globals()[command](**kwargs) # hacky way to get defined functions, and call function with name=command -main() \ No newline at end of file + +main() diff --git a/scripts/capnp_test_pycapnp.py b/scripts/capnp_test_pycapnp.py index 5a02600..7f6b046 100755 --- a/scripts/capnp_test_pycapnp.py +++ b/scripts/capnp_test_pycapnp.py @@ -1,12 +1,13 @@ #!/usr/bin/env python from __future__ import print_function -import capnp import os +import sys + +import capnp capnp.add_import_hook([os.getcwd(), "/usr/local/include/"]) # change this to be auto-detected? -import test_capnp +import test_capnp # noqa: E402 -import sys def decode(name): class_name = name[0].upper() + name[1:] @@ -18,6 +19,7 @@ def encode(name): message = getattr(test_capnp, class_name).from_dict(val.to_dict()) print(message.to_bytes()) + if sys.argv[1] == 'decode': decode(sys.argv[2]) else: diff --git a/setup.py b/setup.py index db60d82..5bef173 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,23 @@ #!/usr/bin/env python +''' +pycapnp distutils setup.py +''' + from __future__ import print_function -use_cython = False - -from setuptools import setup import os import sys -from buildutils import test_build, fetch_libcapnp, build_libcapnp, info + +from distutils.command.clean import clean as _clean from distutils.errors import CompileError from distutils.extension import Extension +from setuptools import setup + +from buildutils import test_build, fetch_libcapnp, build_libcapnp, info + +use_cython = False + _this_dir = os.path.dirname(__file__) MAJOR = 0 @@ -20,10 +28,14 @@ VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) # Write version info def write_version_py(filename=None): + ''' + Generate pycapnp version + ''' cnt = """\ version = '%s' short_version = '%s' +# flake8: noqa E402 F401 from .lib.capnp import _CAPNP_VERSION_MAJOR as LIBCAPNP_VERSION_MAJOR from .lib.capnp import _CAPNP_VERSION_MINOR as LIBCAPNP_VERSION_MINOR from .lib.capnp import _CAPNP_VERSION_MICRO as LIBCAPNP_VERSION_MICRO @@ -39,6 +51,7 @@ from .lib.capnp import _CAPNP_VERSION as LIBCAPNP_VERSION finally: a.close() + write_version_py() # Try to convert README using pandoc @@ -49,13 +62,14 @@ try: changelog = '\nChangelog\n=============\n' + changelog long_description += changelog except (IOError, ImportError): - if len(sys.argv) and sys.argv[-1] == 'sdist': + if sys.argv and sys.argv[-1] == 'sdist': raise long_description = '' -# Clean command, invoked with `python setup.py clean` -from distutils.command.clean import clean as _clean class clean(_clean): + ''' + Clean command, invoked with `python setup.py clean` + ''' def run(self): _clean.run(self) for x in [ 'capnp/lib/capnp.cpp', 'capnp/lib/capnp.h', 'capnp/version.py' ]: @@ -65,6 +79,7 @@ class clean(_clean): except OSError: pass + # set use_cython if lib/capnp.cpp is not detected capnp_compiled_file = os.path.join(os.path.dirname(__file__), 'capnp', 'lib', 'capnp.cpp') if not os.path.isfile(capnp_compiled_file): @@ -87,7 +102,7 @@ try: libcapnp_url = sys.argv[libcapnp_url_index + 1] sys.argv.remove("--libcapnp-url") sys.argv.remove(libcapnp_url) -except: +except Exception: pass if use_cython: @@ -96,6 +111,9 @@ else: from distutils.command.build_ext import build_ext as build_ext_c class build_libcapnp_ext(build_ext_c): + ''' + Build capnproto library + ''' def build_extension(self, ext): build_ext_c.build_extension(self, ext) @@ -114,7 +132,12 @@ class build_libcapnp_ext(build_ext_c): need_build = True if need_build: - info("*WARNING* no libcapnp detected or rebuild forced. Will download and build it from source now. If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. Downloading and building libcapnp may take a while.") + info( + "*WARNING* no libcapnp detected or rebuild forced. " + "Will download and build it from source now. " + "If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. " + "Downloading and building libcapnp may take a while." + ) bundle_dir = os.path.join(_this_dir, "bundled") if not os.path.exists(bundle_dir): os.mkdir(bundle_dir) @@ -130,9 +153,10 @@ class build_libcapnp_ext(build_ext_c): return build_ext_c.run(self) + if use_cython: from Cython.Build import cythonize - import Cython + import Cython # noqa: F401 extensions = cythonize('capnp/lib/*.pyx') else: extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"], @@ -145,9 +169,14 @@ setup( name="pycapnp", packages=["capnp"], version=VERSION, - package_data={'capnp': ['*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*']}, + package_data={ + 'capnp': [ + '*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', + 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*' + ] + }, ext_modules=extensions, - cmdclass = { + cmdclass={ 'clean': clean, 'build_ext': build_libcapnp_ext }, @@ -161,10 +190,10 @@ setup( license='BSD', author="Jason Paryani", author_email="pypi-contact@jparyani.com", - url = 'https://github.com/jparyani/pycapnp', - download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, - keywords = ['capnp', 'capnproto', "Cap'n Proto"], - classifiers = [ + url='https://github.com/jparyani/pycapnp', + download_url='https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, + keywords=['capnp', 'capnproto', "Cap'n Proto"], + classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', @@ -172,12 +201,9 @@ setup( 'Operating System :: POSIX', 'Programming Language :: C++', 'Programming Language :: Cython', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Communications'], ) diff --git a/test/test_capability.py b/test/test_capability.py index 71114ac..5ec1fdb 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -1,8 +1,7 @@ import pytest -import capnp -import os import time +import capnp import test_capability_capnp as capability class Server(capability.TestInterface.Server): @@ -160,6 +159,7 @@ class BadPipelineServer(capability.TestPipeline.Server): _results = _context.results _results.s = response.x + '_foo' _results.outBox.cap = Server(100) + def _error(error): raise Exception('test was a success') @@ -186,7 +186,7 @@ def test_pipeline_exception(): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -194,7 +194,7 @@ def test_pipeline_exception(): def test_casting(): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) diff --git a/test/test_capability_context.py b/test/test_capability_context.py index 4f2c111..0ddbb85 100644 --- a/test/test_capability_context.py +++ b/test/test_capability_context.py @@ -1,12 +1,13 @@ -import pytest -import capnp import os +import pytest + +import capnp this_dir = os.path.dirname(__file__) @pytest.fixture def capability(): - return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) + return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) class Server: def __init__(self, val=1): @@ -150,6 +151,7 @@ class BadPipelineServer: def _then(response): context.results.s = response.x + '_foo' context.results.outBox.cap = capability().TestInterface._new_server(Server(100)) + def _error(error): raise Exception('test was a success') @@ -176,7 +178,7 @@ def test_pipeline_exception_context(capability): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -184,7 +186,7 @@ def test_pipeline_exception_context(capability): def test_casting_context(capability): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) diff --git a/test/test_capability_old.py b/test/test_capability_old.py index beaa5ca..5eb4ef6 100644 --- a/test/test_capability_old.py +++ b/test/test_capability_old.py @@ -1,12 +1,13 @@ -import pytest -import capnp import os +import pytest + +import capnp this_dir = os.path.dirname(__file__) @pytest.fixture def capability(): - return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) + return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) class Server: def __init__(self, val=1): @@ -154,6 +155,7 @@ class BadPipelineServer: _results = _context.results _results.s = response.x + '_foo' _results.outBox.cap = capability().TestInterface._new_server(Server(100)) + def _error(error): raise Exception('test was a success') @@ -180,7 +182,7 @@ def test_pipeline_exception(capability): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -188,7 +190,7 @@ def test_pipeline_exception(capability): def test_casting(capability): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) diff --git a/test/test_large_read.py b/test/test_large_read.py index e912008..d9e76c1 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -1,9 +1,10 @@ -import pytest -import platform -import capnp import os +import platform import tempfile -import sys + +import pytest + +import capnp this_dir = os.path.dirname(__file__) @@ -49,7 +50,7 @@ def get_two_adjacent_messages(test_capnp): msg2 = test_capnp.Msg.new_message() m2 = msg2.to_bytes() - return m1 + m2 + return m1 + m2 def test_large_read_multiple_bytes(test_capnp): data = get_two_adjacent_messages(test_capnp) @@ -81,19 +82,3 @@ def test_large_read_mutltiple_bytes_memoryview(test_capnp): data = get_two_adjacent_messages(test_capnp) + b' ' for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): pass - -@pytest.mark.skipif(sys.version_info[0] == 3, reason="Legacy buffer support only for python 2.7") -def test_large_read_mutltiple_bytes_buffer(test_capnp): - data = get_two_adjacent_messages(test_capnp) - for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): - pass - - with pytest.raises(capnp.KjException): - data = get_two_adjacent_messages(test_capnp)[:-1] - for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): - pass - - with pytest.raises(capnp.KjException): - data = get_two_adjacent_messages(test_capnp) + b' ' - for m in test_capnp.Msg.read_multiple_bytes(buffer(data)): - pass diff --git a/test/test_load.py b/test/test_load.py index 7f6d6af..f544f18 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -7,15 +7,15 @@ this_dir = os.path.dirname(__file__) @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) @pytest.fixture def foo(): - return capnp.load(os.path.join(this_dir, 'foo.capnp')) + return capnp.load(os.path.join(this_dir, 'foo.capnp')) @pytest.fixture def bar(): - return capnp.load(os.path.join(this_dir, 'bar.capnp')) + return capnp.load(os.path.join(this_dir, 'bar.capnp')) def test_basic_load(): capnp.load(os.path.join(this_dir, 'addressbook.capnp')) @@ -56,13 +56,13 @@ def test_failed_import(): bar.foo = foo def test_defualt_import_hook(): - import addressbook_capnp + import addressbook_capnp # noqa: F401 def test_dash_import(): - import addressbook_with_dashes_capnp + import addressbook_with_dashes_capnp # noqa: F401 def test_spaces_import(): - import addressbook_with_spaces_capnp + import addressbook_with_spaces_capnp # noqa: F401 def test_add_import_hook(): capnp.add_import_hook([this_dir]) @@ -86,4 +86,4 @@ def test_remove_import_hook(): del sys.modules['addressbook_capnp'] # hack to deal with it being imported already with pytest.raises(ImportError): - import addressbook_capnp + import addressbook_capnp # noqa: F401 diff --git a/test/test_object.py b/test/test_object.py index f87e124..3fee49a 100644 --- a/test/test_object.py +++ b/test/test_object.py @@ -7,7 +7,7 @@ this_dir = os.path.dirname(__file__) @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) def test_object_basic(addressbook): diff --git a/test/test_regression.py b/test/test_regression.py index 93a8a26..12bde05 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,4 +1,4 @@ - # -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- import pytest import capnp @@ -16,7 +16,7 @@ else: @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) def test_addressbook_message_classes(addressbook): def writeAddressBook(fd): @@ -71,7 +71,7 @@ def test_addressbook_message_classes(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') writeAddressBook(f.fileno()) @@ -130,7 +130,7 @@ def test_addressbook(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') @@ -192,7 +192,7 @@ def test_addressbook_resizable(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') @@ -262,7 +262,7 @@ def test_addressbook_explicit_fields(addressbook): assert bobPhones[1]._get_by_field(phone_fields['number']) == "555-7654" assert bobPhones[1]._get_by_field(phone_fields['type']) == 'work' employment = bob._get_by_field(person_fields['employment']) - employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) == None + employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) is None f = open('example', 'w') @@ -371,8 +371,8 @@ def check_list(reader, expected): assert reader[i] == v def check_all_types(reader): - assert reader.voidField == None - assert reader.boolField == True + assert reader.voidField is None + assert reader.boolField assert reader.int8Field == -123 assert reader.int16Field == -12345 assert reader.int32Field == -12345678 @@ -387,8 +387,8 @@ def check_all_types(reader): assert reader.dataField == b"bar" subReader = reader.structField - assert subReader.voidField == None - assert subReader.boolField == True + assert subReader.voidField is None + assert subReader.boolField assert subReader.int8Field == -12 assert subReader.int16Field == 3456 assert subReader.int32Field == -78901234 @@ -495,7 +495,7 @@ def test_build_first_segment_size(all_types): expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() assert str(root) + '\n' == expectedText - root = all_types.TestAllTypes.new_message(1024*1024) + root = all_types.TestAllTypes.new_message(1024 * 1024) init_all_types(root) expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() assert str(root) + '\n' == expectedText diff --git a/test/test_response.py b/test/test_response.py index c9bff3d..ae3eb40 100644 --- a/test/test_response.py +++ b/test/test_response.py @@ -1,8 +1,3 @@ -import pytest -import capnp -import os -import time - import test_response_capnp class FooServer(test_response_capnp.Foo.Server): diff --git a/test/test_rpc.py b/test/test_rpc.py index c51797e..a814d68 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,6 +1,5 @@ import pytest import capnp -import os import socket import test_capability_capnp @@ -30,7 +29,7 @@ def test_simple_rpc(): read, write = socket.socketpair(socket.AF_UNIX) restorer = SimpleRestorer() - server = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, restorer) client = capnp.TwoPartyClient(read) ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') @@ -47,7 +46,7 @@ def test_simple_rpc_with_options(): read, write = socket.socketpair(socket.AF_UNIX) restorer = SimpleRestorer() - server = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, restorer) # This traversal limit is too low to receive the response in, so we expect # an exception during the call. client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) @@ -58,13 +57,13 @@ def test_simple_rpc_with_options(): remote = cap.foo(i=5) with pytest.raises(capnp.KjException): - response = remote.wait() + _ = remote.wait() def test_simple_rpc_restore_func(): read, write = socket.socketpair(socket.AF_UNIX) - server = capnp.TwoPartyServer(write, restore_func) + _ = capnp.TwoPartyServer(write, restore_func) client = capnp.TwoPartyClient(read) ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') @@ -86,7 +85,7 @@ def text_restore_func(objectId): def test_ez_rpc(): read, write = socket.socketpair(socket.AF_UNIX) - server = capnp.TwoPartyServer(write, text_restore_func) + _ = capnp.TwoPartyServer(write, text_restore_func) client = capnp.TwoPartyClient(read) cap = client.ez_restore('testInterface') @@ -108,7 +107,7 @@ def test_ez_rpc(): def test_simple_rpc_bootstrap(): read, write = socket.socketpair(socket.AF_UNIX) - server = capnp.TwoPartyServer(write, bootstrap=Server(100)) + _ = capnp.TwoPartyServer(write, bootstrap=Server(100)) client = capnp.TwoPartyClient(read) cap = client.bootstrap() diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index ecf73ec..6beea16 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -1,21 +1,23 @@ -import capnp +import gc import os import socket -import gc import subprocess +import sys # add examples dir to sys.path import time -import sys # add examples dir to sys.path +import capnp + examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') sys.path.append(examples_dir) -import calculator_client -import calculator_server + +import calculator_client # noqa: E402 +import calculator_server # noqa: E402 def test_calculator(): read, write = socket.socketpair(socket.AF_UNIX) - server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) + _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) calculator_client.main(read) @@ -57,7 +59,7 @@ def test_calculator_gc(): evaluate_impl_orig = calculator_server.evaluate_impl calculator_server.evaluate_impl = new_evaluate_impl(evaluate_impl_orig) - server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) + _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) calculator_client.main(read) calculator_server.evaluate_impl = evaluate_impl_orig diff --git a/test/test_serialization.py b/test/test_serialization.py index a3ff183..1f26d15 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -42,7 +42,10 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="TODO: Investigate why this works on CPython but fails on PyPy.") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="TODO: Investigate why this works on CPython but fails on PyPy." +) def test_roundtrip_segments(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -79,7 +82,10 @@ def test_roundtrip_bytes_fail(all_types): with pytest.raises(TypeError): all_types.TestAllTypes.from_bytes(42) -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test.") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test." +) def test_roundtrip_bytes_packed(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -146,7 +152,10 @@ def test_roundtrip_bytes_multiple_packed(all_types): i += 1 assert i == 3 -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now.") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now." +) def test_roundtrip_dict(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) diff --git a/test/test_struct.py b/test/test_struct.py index 953a8be..7b23b5a 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -77,7 +77,10 @@ def test_which_reader(addressbook): addresses.which -@pytest.mark.skipif(capnp.version.LIBCAPNP_VERSION < 5000, reason="Using ints as enums requires v0.5.0+ of the C++ capnp library") +@pytest.mark.skipif( + capnp.version.LIBCAPNP_VERSION < 5000, + reason="Using ints as enums requires v0.5.0+ of the C++ capnp library" +) def test_enum(addressbook): addresses = addressbook.AddressBook.new_message() people = addresses.init('people', 2) @@ -188,13 +191,8 @@ def test_set_dict_union(addressbook): assert person.employment.employer.name == 'foo' -try: - basestring # attempt to evaluate basestring - def isstr(s): - return isinstance(s, basestring) -except NameError: - def isstr(s): - return isinstance(s, str) +def isstr(s): + return isinstance(s, str) def test_to_dict_enum(addressbook): @@ -227,7 +225,12 @@ def test_to_dict_verbose(addressbook): def test_to_dict_ordered(addressbook): - person = addressbook.Person.new_message(**{'name': 'Alice', 'phones': [{'type': 'mobile', 'number': '555-1212'}], 'id': 123, 'employment': {'school': 'MIT'}, 'email': 'alice@example.com'}) + person = addressbook.Person.new_message(**{ + 'name': 'Alice', + 'phones': [{'type': 'mobile', 'number': '555-1212'}], + 'id': 123, + 'employment': {'school': 'MIT'}, 'email': 'alice@example.com' + }) if sys.version_info >= (2, 7): assert list(person.to_dict(ordered=True).keys()) == ['id', 'name', 'email', 'phones', 'employment'] @@ -246,4 +249,4 @@ def test_nested_list(addressbook): struct.list[1][0] = 2 struct.list[1][1] = 3 - assert struct.to_dict()["list"] == [[1], [2,3]] + assert struct.to_dict()["list"] == [[1], [2, 3]] diff --git a/test/test_threads.py b/test/test_threads.py index 8604ecf..218557b 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -1,20 +1,38 @@ -import capnp -import pytest -import test_capability_capnp +''' +thread test +''' + +import platform import socket import threading -import platform -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +import pytest + +import capnp +import test_capability_capnp + +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_making_event_loop(): + ''' + Event loop test + ''' capnp.remove_event_loop(True) capnp.create_event_loop() capnp.remove_event_loop() capnp.create_event_loop() -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_making_threaded_event_loop(): + ''' + Threaded event loop test + ''' capnp.remove_event_loop(True) capnp.create_event_loop(True) @@ -23,23 +41,40 @@ def test_making_threaded_event_loop(): class Server(test_capability_capnp.TestInterface.Server): - + ''' + Server + ''' def __init__(self, val=1): self.val = val def foo(self, i, j, **kwargs): + ''' + foo + ''' return str(i * 5 + self.val) class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): + ''' + SimpleRestorer + ''' def restore(self, ref_id): + ''' + Restore + ''' assert ref_id.tag == 'testInterface' return Server(100) -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_using_threads(): + ''' + Thread test + ''' capnp.remove_event_loop(True) capnp.create_event_loop(True) @@ -47,7 +82,7 @@ def test_using_threads(): def run_server(): restorer = SimpleRestorer() - server = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, restorer) capnp.wait_forever() server_thread = threading.Thread(target=run_server) From d62e9b80f78c4dcae4fd9cfb58c13620d033c303 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Fri, 27 Sep 2019 00:30:08 -0700 Subject: [PATCH 106/126] Always use Cython --- setup.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 5bef173..f070213 100644 --- a/setup.py +++ b/setup.py @@ -16,8 +16,6 @@ from setuptools import setup from buildutils import test_build, fetch_libcapnp, build_libcapnp, info -use_cython = False - _this_dir = os.path.dirname(__file__) MAJOR = 0 @@ -80,11 +78,6 @@ class clean(_clean): pass -# set use_cython if lib/capnp.cpp is not detected -capnp_compiled_file = os.path.join(os.path.dirname(__file__), 'capnp', 'lib', 'capnp.cpp') -if not os.path.isfile(capnp_compiled_file): - use_cython = True - # hack to parse commandline arguments force_bundled_libcapnp = "--force-bundled-libcapnp" in sys.argv if force_bundled_libcapnp: @@ -95,7 +88,7 @@ if force_system_libcapnp: force_cython = "--force-cython" in sys.argv if force_cython: sys.argv.remove("--force-cython") - use_cython = True + # Always use cython, ignoring option libcapnp_url = None try: libcapnp_url_index = sys.argv.index("--libcapnp-url") @@ -105,10 +98,7 @@ try: except Exception: pass -if use_cython: - from Cython.Distutils import build_ext as build_ext_c -else: - from distutils.command.build_ext import build_ext as build_ext_c +from Cython.Distutils import build_ext as build_ext_c class build_libcapnp_ext(build_ext_c): ''' @@ -154,16 +144,9 @@ class build_libcapnp_ext(build_ext_c): return build_ext_c.run(self) -if use_cython: - from Cython.Build import cythonize - import Cython # noqa: F401 - extensions = cythonize('capnp/lib/*.pyx') -else: - extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"], - include_dirs=["."], - language='c++', - extra_compile_args=['--std=c++14'], - libraries=['capnpc', 'capnp-rpc', 'capnp', 'kj-async', 'kj'])] +from Cython.Build import cythonize +import Cython # noqa: F401 +extensions = cythonize('capnp/lib/*.pyx') setup( name="pycapnp", From 75e0e7e84c3a6d7f458466433c7f8b8da246d40d Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Fri, 27 Sep 2019 01:09:28 -0700 Subject: [PATCH 107/126] Fixed or waved all pytest failures and errors - Needed to include cleanup_global_schema_parser() to handle duplicate imports of the same .capnp file * Duplicate IDs are a problem as pytest does not fully cleanup between tests - Marked some tests as xfail as I'm not sure the test is supposed to work anymore with recent versions of capnproto --- capnp/lib/capnp.pyx | 7 +++++++ test/all-types.txt | 2 +- test/test_capability_context.py | 19 +++++++++++++++++++ test/test_capability_old.py | 18 ++++++++++++++++++ test/test_load.py | 9 +++++++++ test/test_rpc.py | 8 ++++---- 6 files changed, 58 insertions(+), 5 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index e5d420e..8824122 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -4041,6 +4041,13 @@ def _write_packed_message_to_fd(int fd, _MessageBuilder message): _global_schema_parser = None +def cleanup_global_schema_parser(): + """Unloads all of the schema from the current context""" + global _global_schema_parser + if _global_schema_parser: + del _global_schema_parser + _global_schema_parser = None + def load(file_name, display_name=None, imports=[]): """Load a Cap'n Proto schema from a file diff --git a/test/all-types.txt b/test/all-types.txt index a9e3dcc..a85df4c 100644 --- a/test/all-types.txt +++ b/test/all-types.txt @@ -25,7 +25,7 @@ uInt64Field = 345678901234567890, float32Field = -1.25e-10, float64Field = 345, - textField = "\xe2\x98\x83", + textField = "☃", dataField = "qux", structField = ( voidField = void, diff --git a/test/test_capability_context.py b/test/test_capability_context.py index 0ddbb85..e4d6fb6 100644 --- a/test/test_capability_context.py +++ b/test/test_capability_context.py @@ -5,8 +5,11 @@ import capnp this_dir = os.path.dirname(__file__) +# flake8: noqa: E501 + @pytest.fixture def capability(): + capnp.cleanup_global_schema_parser() return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) class Server: @@ -116,7 +119,15 @@ def test_simple_client_context(capability): with pytest.raises(Exception): remote = client.foo(baz=5) +@pytest.mark.xfail def test_pipeline_context(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f87c1ac6e40 7f87c17c3250 7f87c17be260 7f87c17c49f0 7f87c17c0f50 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409 + ''' client = capability.TestPipeline._new_client(PipelineServer()) foo_client = capability.TestInterface._new_client(Server()) @@ -221,7 +232,15 @@ class TailCallee: results.t = context.params.t results.c = capability().TestCallOrder._new_server(TailCallOrder()) +@pytest.mark.xfail def test_tail_call(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:75: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f87c17c5540 7f87c17c51b0 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c388ace7 + ''' callee_server = TailCallee() caller_server = TailCaller() diff --git a/test/test_capability_old.py b/test/test_capability_old.py index 5eb4ef6..cdcf46a 100644 --- a/test/test_capability_old.py +++ b/test/test_capability_old.py @@ -5,6 +5,8 @@ import capnp this_dir = os.path.dirname(__file__) +# flake8: noqa: E501 + @pytest.fixture def capability(): return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) @@ -117,7 +119,15 @@ def test_simple_client(capability): with pytest.raises(Exception): remote = client.foo(baz=5) +@pytest.mark.xfail def test_pipeline(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f680f7fce40 7f680f4f9250 7f680f4f4260 7f680f4fa9f0 7f680f4f6f50 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409 + ''' client = capability.TestPipeline._new_client(PipelineServer()) foo_client = capability.TestInterface._new_client(Server()) @@ -225,7 +235,15 @@ class TailCallee: results.t = t results.c = capability().TestCallOrder._new_server(TailCallOrder()) +@pytest.mark.xfail def test_tail_call(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:104: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f680f4fb540 7f680f4fb1b0 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409 7f68115eb767 7f68115ece7e 7f68115c0ce7 + ''' callee_server = TailCallee() caller_server = TailCaller() diff --git a/test/test_load.py b/test/test_load.py index f544f18..174d793 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -56,6 +56,9 @@ def test_failed_import(): bar.foo = foo def test_defualt_import_hook(): + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + import addressbook_capnp # noqa: F401 def test_dash_import(): @@ -67,6 +70,9 @@ def test_spaces_import(): def test_add_import_hook(): capnp.add_import_hook([this_dir]) + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + import addressbook_capnp addressbook_capnp.AddressBook.new_message() @@ -75,6 +81,9 @@ def test_multiple_add_import_hook(): capnp.add_import_hook() capnp.add_import_hook([this_dir]) + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + import addressbook_capnp addressbook_capnp.AddressBook.new_message() diff --git a/test/test_rpc.py b/test/test_rpc.py index a814d68..3d0ac40 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -52,11 +52,11 @@ def test_simple_rpc_with_options(): client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) with pytest.raises(capnp.KjException): + cap = client.restore(ref) + cap = cap.cast_as(test_capability_capnp.TestInterface) + + remote = cap.foo(i=5) _ = remote.wait() From 58a5c5fc1f24fbe9279c13c6ea413e7cd6c31c43 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Fri, 27 Sep 2019 01:14:11 -0700 Subject: [PATCH 108/126] Initial pythonpackage.yml for GitHub Actions --- .github/workflows/pythonpackage.yml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/pythonpackage.yml diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..bda2ebc --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,46 @@ +name: Python package + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 4 + fail-fast: false + matrix: + python-version: [3.5, 3.6, 3.7] + os: [ubuntu-latest, macOS-latest] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + # Install capnproto + curl -O https://capnproto.org/capnproto-c++-0.7.0.tar.gz + tar zxf capnproto-c++-0.7.0.tar.gz + cd capnproto-c++-0.7.0 + ./configure + make -j check + sudo make install + - name: Build pycapnp and install + run: | + pip install . + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude benchmark + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude benchmark + - name: Test with pytest + run: | + pip install pytest + pytest From 78776de647bdf7e951217b79f3dba1d2b461c954 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Fri, 27 Sep 2019 14:40:54 -0700 Subject: [PATCH 109/126] Adding examples as pytest tests - This way they will be included in CI checks - Decreased the delay time in the thread-like examples to speed up tests (probably could decrease the time some more) - Added an async version of the calculator test - Forcing python3 support for example scripts --- examples/addressbook.py | 2 + examples/async_calculator_client.py | 341 ++++++++++++++++++++++++++++ examples/async_calculator_server.py | 182 +++++++++++++++ examples/async_client.py | 2 +- examples/async_server.py | 4 +- examples/async_ssl_client.py | 6 +- examples/async_ssl_server.py | 10 +- examples/calculator_client.py | 2 +- examples/calculator_server.py | 2 +- examples/thread_client.py | 2 +- examples/thread_server.py | 4 +- test/test_examples.py | 52 +++++ 12 files changed, 596 insertions(+), 13 deletions(-) create mode 100755 examples/async_calculator_client.py create mode 100755 examples/async_calculator_server.py create mode 100644 test/test_examples.py diff --git a/examples/addressbook.py b/examples/addressbook.py index b6ae64f..2bd869a 100755 --- a/examples/addressbook.py +++ b/examples/addressbook.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from __future__ import print_function import capnp # noqa: F401 diff --git a/examples/async_calculator_client.py b/examples/async_calculator_client.py new file mode 100755 index 0000000..de9c823 --- /dev/null +++ b/examples/async_calculator_client.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 + +from __future__ import print_function +import argparse +import asyncio +import socket +import capnp + +import calculator_capnp + + +class PowerFunction(calculator_capnp.Calculator.Function.Server): + + '''An implementation of the Function interface wrapping pow(). Note that + we're implementing this on the client side and will pass a reference to + the server. The server will then be able to make calls back to the client.''' + + def call(self, params, **kwargs): + '''Note the **kwargs. This is very necessary to include, since + protocols can add parameters over time. Also, by default, a _context + variable is passed to all server methods, but you can also return + results directly as python objects, and they'll be added to the + results struct in the correct order''' + + return pow(params[0], params[1]) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Calculator server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Pass "calculator" to ez_restore (there's also a `restore` function that + # takes a struct or AnyPointer as an argument), and then cast the returned + # capability to it's proper type. This casting is due to capabilities not + # having a reference to their schema + calculator = client.bootstrap().cast_as(calculator_capnp.Calculator) + + '''Make a request that just evaluates the literal value 123. + + What's interesting here is that evaluate() returns a "Value", which is + another interface and therefore points back to an object living on the + server. We then have to call read() on that object to read it. + However, even though we are making two RPC's, this block executes in + *one* network round trip because of promise pipelining: we do not wait + for the first call to complete before we send the second call to the + server.''' + + print('Evaluating a literal... ', end="") + + # Make the request. Note we are using the shorter function form (instead + # of evaluate_request), and we are passing a dictionary that represents a + # struct and its member to evaluate + eval_promise = calculator.evaluate({"literal": 123}) + + # This is equivalent to: + ''' + request = calculator.evaluate_request() + request.expression.literal = 123 + + # Send it, which returns a promise for the result (without blocking). + eval_promise = request.send() + ''' + + # Using the promise, create a pipelined request to call read() on the + # returned object. Note that here we are using the shortened method call + # syntax read(), which is mostly just sugar for read_request().send() + read_promise = eval_promise.value.read() + + # Now that we've sent all the requests, wait for the response. Until this + # point, we haven't waited at all! + response = await read_promise.a_wait() + assert response.value == 123 + + print("PASS") + + '''Make a request to evaluate 123 + 45 - 67. + + The Calculator interface requires that we first call getOperator() to + get the addition and subtraction functions, then call evaluate() to use + them. But, once again, we can get both functions, call evaluate(), and + then read() the result -- four RPCs -- in the time of *one* network + round trip, because of promise pipelining.''' + + print("Using add and subtract... ", end='') + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "subtract" function from the server. + subtract = calculator.getOperator(op='subtract').func + + # Build the request to evaluate 123 + 45 - 67. Note the form is 'evaluate' + # + '_request', where 'evaluate' is the name of the method we want to call + request = calculator.evaluate_request() + subtract_call = request.expression.init('call') + subtract_call.function = subtract + subtract_params = subtract_call.init('params', 2) + subtract_params[1].literal = 67.0 + + add_call = subtract_params[0].init('call') + add_call.function = add + add_params = add_call.init('params', 2) + add_params[0].literal = 123 + add_params[1].literal = 45 + + # Send the evaluate() request, read() the result, and wait for read() to finish. + eval_promise = request.send() + read_promise = eval_promise.value.read() + + response = await read_promise.a_wait() + assert response.value == 101 + + print("PASS") + + ''' + Note: a one liner version of building the previous request (I highly + recommend not doing it this way for such a complicated structure, but I + just wanted to demonstrate it is possible to set all of the fields with a + dictionary): + + eval_promise = calculator.evaluate( +{'call': {'function': subtract, + 'params': [{'call': {'function': add, + 'params': [{'literal': 123}, + {'literal': 45}]}}, + {'literal': 67.0}]}}) + ''' + + '''Make a request to evaluate 4 * 6, then use the result in two more + requests that add 3 and 5. + + Since evaluate() returns its result wrapped in a `Value`, we can pass + that `Value` back to the server in subsequent requests before the first + `evaluate()` has actually returned. Thus, this example again does only + one network round trip.''' + + print("Pipelining eval() calls... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "multiply" function from the server. + multiply = calculator.getOperator(op='multiply').func + + # Build the request to evaluate 4 * 6 + request = calculator.evaluate_request() + + multiply_call = request.expression.init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[0].literal = 4 + multiply_params[1].literal = 6 + + multiply_result = request.send().value + + # Use the result in two calls that add 3 and add 5. + + add_3_request = calculator.evaluate_request() + add_3_call = add_3_request.expression.init("call") + add_3_call.function = add + add_3_params = add_3_call.init("params", 2) + add_3_params[0].previousResult = multiply_result + add_3_params[1].literal = 3 + add_3_promise = add_3_request.send().value.read() + + add_5_request = calculator.evaluate_request() + add_5_call = add_5_request.expression.init("call") + add_5_call.function = add + add_5_params = add_5_call.init("params", 2) + add_5_params[0].previousResult = multiply_result + add_5_params[1].literal = 5 + add_5_promise = add_5_request.send().value.read() + + # Now wait for the results. + assert (await add_3_promise.a_wait()).value == 27 + assert (await add_5_promise.a_wait()).value == 29 + + print("PASS") + + '''Our calculator interface supports defining functions. Here we use it + to define two functions and then make calls to them as follows: + + f(x, y) = x * 100 + y + g(x) = f(x, x + 1) * 2; + f(12, 34) + g(21) + + Once again, the whole thing takes only one network round trip.''' + + print("Defining functions... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "multiply" function from the server. + multiply = calculator.getOperator(op='multiply').func + + # Define f. + request = calculator.defFunction_request() + request.paramCount = 2 + + # Build the function body. + add_call = request.body.init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[1].parameter = 1 # y + + multiply_call = add_params[0].init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[0].parameter = 0 # x + multiply_params[1].literal = 100 + + f = request.send().func + + # Define g. + request = calculator.defFunction_request() + request.paramCount = 1 + + # Build the function body. + multiply_call = request.body.init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[1].literal = 2 + + f_call = multiply_params[0].init("call") + f_call.function = f + f_params = f_call.init("params", 2) + f_params[0].parameter = 0 + + add_call = f_params[1].init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[0].parameter = 0 + add_params[1].literal = 1 + + g = request.send().func + + # OK, we've defined all our functions. Now create our eval requests. + + # f(12, 34) + f_eval_request = calculator.evaluate_request() + f_call = f_eval_request.expression.init("call") + f_call.function = f + f_params = f_call.init("params", 2) + f_params[0].literal = 12 + f_params[1].literal = 34 + f_eval_promise = f_eval_request.send().value.read() + + # g(21) + g_eval_request = calculator.evaluate_request() + g_call = g_eval_request.expression.init("call") + g_call.function = g + g_call.init('params', 1)[0].literal = 21 + g_eval_promise = g_eval_request.send().value.read() + + # Wait for the results. + assert (await f_eval_promise.a_wait()).value == 1234 + assert (await g_eval_promise.a_wait()).value == 4244 + + print("PASS") + + '''Make a request that will call back to a function defined locally. + + Specifically, we will compute 2^(4 + 5). However, exponent is not + defined by the Calculator server. So, we'll implement the Function + interface locally and pass it to the server for it to use when + evaluating the expression. + + This example requires two network round trips to complete, because the + server calls back to the client once before finishing. In this + particular case, this could potentially be optimized by using a tail + call on the server side -- see CallContext::tailCall(). However, to + keep the example simpler, we haven't implemented this optimization in + the sample server.''' + + print("Using a callback... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + + # Build the eval request for 2^(4+5). + request = calculator.evaluate_request() + + pow_call = request.expression.init("call") + pow_call.function = PowerFunction() + pow_params = pow_call.init("params", 2) + pow_params[0].literal = 2 + + add_call = pow_params[1].init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[0].literal = 4 + add_params[1].literal = 5 + + # Send the request and wait. + response = await request.send().value.read().a_wait() + assert response.value == 512 + + print("PASS") + +if __name__ == '__main__': + asyncio.run(main(parse_args().host)) diff --git a/examples/async_calculator_server.py b/examples/async_calculator_server.py new file mode 100755 index 0000000..1316571 --- /dev/null +++ b/examples/async_calculator_server.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 + +from __future__ import print_function +import argparse +import asyncio +import socket +import random +import capnp + +import calculator_capnp + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + await client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +def read_value(value): + '''Helper function to asynchronously call read() on a Calculator::Value and + return a promise for the result. (In the future, the generated code might + include something like this automatically.)''' + + return value.read().then(lambda result: result.value) + + +def evaluate_impl(expression, params=None): + '''Implementation of CalculatorImpl::evaluate(), also shared by + FunctionImpl::call(). In the latter case, `params` are the parameter + values passed to the function; in the former case, `params` is just an + empty list.''' + + which = expression.which() + + if which == 'literal': + return capnp.Promise(expression.literal) + elif which == 'previousResult': + return read_value(expression.previousResult) + elif which == 'parameter': + assert expression.parameter < len(params) + return capnp.Promise(params[expression.parameter]) + elif which == 'call': + call = expression.call + func = call.function + + # Evaluate each parameter. + paramPromises = [evaluate_impl(param, params) for param in call.params] + + joinedParams = capnp.join_promises(paramPromises) + # When the parameters are complete, call the function. + ret = (joinedParams + .then(lambda vals: func.call(vals)) + .then(lambda result: result.value)) + + return ret + else: + raise ValueError("Unknown expression type: " + which) + + +class ValueImpl(calculator_capnp.Calculator.Value.Server): + + "Simple implementation of the Calculator.Value Cap'n Proto interface." + + def __init__(self, value): + self.value = value + + def read(self, **kwargs): + return self.value + + +class FunctionImpl(calculator_capnp.Calculator.Function.Server): + + '''Implementation of the Calculator.Function Cap'n Proto interface, where the + function is defined by a Calculator.Expression.''' + + def __init__(self, paramCount, body): + self.paramCount = paramCount + self.body = body.as_builder() + + def call(self, params, _context, **kwargs): + '''Note that we're returning a Promise object here, and bypassing the + helper functionality that normally sets the results struct from the + returned object. Instead, we set _context.results directly inside of + another promise''' + + assert len(params) == self.paramCount + # using setattr because '=' is not allowed inside of lambdas + return evaluate_impl(self.body, params).then(lambda value: setattr(_context.results, 'value', value)) + + +class OperatorImpl(calculator_capnp.Calculator.Function.Server): + + '''Implementation of the Calculator.Function Cap'n Proto interface, wrapping + basic binary arithmetic operators.''' + + def __init__(self, op): + self.op = op + + def call(self, params, **kwargs): + assert len(params) == 2 + + op = self.op + + if op == 'add': + return params[0] + params[1] + elif op == 'subtract': + return params[0] - params[1] + elif op == 'multiply': + return params[0] * params[1] + elif op == 'divide': + return params[0] / params[1] + else: + raise ValueError('Unknown operator') + + +class CalculatorImpl(calculator_capnp.Calculator.Server): + + "Implementation of the Calculator Cap'n Proto interface." + + def evaluate(self, expression, _context, **kwargs): + return evaluate_impl(expression).then(lambda value: setattr(_context.results, 'value', ValueImpl(value))) + + def defFunction(self, paramCount, body, _context, **kwargs): + return FunctionImpl(paramCount, body) + + def getOperator(self, op, **kwargs): + return OperatorImpl(op) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=CalculatorImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ) + except: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + family=socket.AF_INET6 + ) + + async with server: + await server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/async_client.py b/examples/async_client.py index 6808faa..7ffed54 100755 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/examples/async_server.py b/examples/async_server.py index 5030ec3..5d534fb 100755 --- a/examples/async_server.py +++ b/examples/async_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function @@ -20,7 +20,7 @@ class ExampleImpl(thread_capnp.Example.Server): .then(lambda _: self.subscribeStatus(subscriber)) def longRunning(self, **kwargs): - return capnp.getTimer().after_delay(3 * 10**9) + return capnp.getTimer().after_delay(1 * 10**9) async def myreader(server, reader): diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py index a95d03a..802044e 100755 --- a/examples/async_ssl_client.py +++ b/examples/async_ssl_client.py @@ -1,9 +1,10 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import asyncio import argparse +import os import time import capnp import socket @@ -11,6 +12,7 @@ import ssl import thread_capnp +this_dir = os.path.dirname(os.path.abspath(__file__)) capnp.remove_event_loop() capnp.create_event_loop(threaded=True) @@ -55,7 +57,7 @@ async def main(host): port = host[1] # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile='selfsigned.cert') + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert')) # Handle both IPv4 and IPv6 cases try: diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py index ebbcf47..c162109 100755 --- a/examples/async_ssl_server.py +++ b/examples/async_ssl_server.py @@ -1,8 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import argparse +import os import capnp import thread_capnp @@ -11,6 +12,9 @@ import socket import ssl +this_dir = os.path.dirname(os.path.abspath(__file__)) + + class ExampleImpl(thread_capnp.Example.Server): "Implementation of the Example threading Cap'n Proto interface." @@ -21,7 +25,7 @@ class ExampleImpl(thread_capnp.Example.Server): .then(lambda _: self.subscribeStatus(subscriber)) def longRunning(self, **kwargs): - return capnp.getTimer().after_delay(3 * 10**9) + return capnp.getTimer().after_delay(1 * 10**9) async def myreader(server, reader): @@ -68,7 +72,7 @@ async def main(): # Setup SSL context ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ctx.load_cert_chain('selfsigned.cert', 'selfsigned.key') + ctx.load_cert_chain(os.path.join(this_dir, 'selfsigned.cert'), os.path.join(this_dir, 'selfsigned.key')) # Handle both IPv4 and IPv6 cases try: diff --git a/examples/calculator_client.py b/examples/calculator_client.py index f6569e4..85694da 100755 --- a/examples/calculator_client.py +++ b/examples/calculator_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import argparse diff --git a/examples/calculator_server.py b/examples/calculator_server.py index 32af680..38ca7ef 100755 --- a/examples/calculator_server.py +++ b/examples/calculator_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import argparse diff --git a/examples/thread_client.py b/examples/thread_client.py index fdd50ad..2b23f2d 100755 --- a/examples/thread_client.py +++ b/examples/thread_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function diff --git a/examples/thread_server.py b/examples/thread_server.py index 142bd1f..6a2b52f 100755 --- a/examples/thread_server.py +++ b/examples/thread_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function @@ -18,7 +18,7 @@ class ExampleImpl(thread_capnp.Example.Server): .then(lambda _: self.subscribeStatus(subscriber)) def longRunning(self, **kwargs): - return capnp.getTimer().after_delay(3 * 10**9) + return capnp.getTimer().after_delay(1 * 10**9) def parse_args(): diff --git a/test/test_examples.py b/test/test_examples.py new file mode 100644 index 0000000..2f39b7d --- /dev/null +++ b/test/test_examples.py @@ -0,0 +1,52 @@ +import gc +import os +import socket +import subprocess +import sys # add examples dir to sys.path +import time + +examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') + + +def run_subprocesses(address, server, client): + server = subprocess.Popen([os.path.join(examples_dir, server), address]) + time.sleep(1) # Give the server some small amount of time to start listening + client = subprocess.Popen([os.path.join(examples_dir, client), address]) + + ret = client.wait() + server.kill() + assert ret == 0 + + +def test_async_calculator_example(): + address = 'localhost:36432' + server = 'async_calculator_server.py' + client = 'async_calculator_client.py' + run_subprocesses(address, server, client) + + +def test_thread_example(): + address = 'localhost:36433' + server = 'thread_server.py' + client = 'thread_client.py' + run_subprocesses(address, server, client) + + +def test_addressbook_example(): + proc = subprocess.Popen([os.path.join(examples_dir, 'addressbook.py')]) + ret = proc.wait() + assert ret == 0 + + +def test_async_example(): + address = 'localhost:36434' + server = 'async_server.py' + client = 'async_client.py' + run_subprocesses(address, server, client) + + +def test_ssl_async_example(): + address = 'localhost:36435' + server = 'async_ssl_server.py' + client = 'async_ssl_client.py' + run_subprocesses(address, server, client) From 8361fa8597af8045b6393a0b4372198ef8370984 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Sun, 29 Sep 2019 23:54:49 -0700 Subject: [PATCH 110/126] Currently asyncio examples require Python 3.7+ - It should be possible to port asyncio examples to at least Python 3.6 * However, in my quick 10 minute attempt it wasn't as smooth as I'd hoped --- .github/workflows/pythonpackage.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index bda2ebc..41afa2a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -1,4 +1,4 @@ -name: Python package +name: Python Test Packaging on: [push, pull_request] @@ -10,7 +10,9 @@ jobs: max-parallel: 4 fail-fast: false matrix: - python-version: [3.5, 3.6, 3.7] + # Some asyncio commands require 3.7+ + # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work + python-version: [3.7] os: [ubuntu-latest, macOS-latest] steps: From a7efe4e3f89524e6f3a2d8d80ba92f91e39435a6 Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Thu, 18 May 2017 11:47:38 +0200 Subject: [PATCH 111/126] Fix setup.py for MSVC --- buildutils/detect.py | 11 +++++++---- buildutils/misc.py | 4 ++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/buildutils/detect.py b/buildutils/detect.py index 19459b1..03f1be4 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -57,11 +57,14 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): lpreargs = ['-m32'] else: lpreargs = ['-m64'] - extra = compiler_attrs.get('extra_compile_args', []) - extra += ['--std=c++14'] + extra_compile_args = compiler_attrs.get('extra_compile_args', []) + extra_compile_args += ['--std=c++14'] + extra_link_args = compiler_attrs.get('extra_link_args', []) + if cc.compiler_type == 'msvc': + extra_link_args += ['/MANIFEST'] - objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra) - cc.link_executable(objs, efile, extra_preargs=lpreargs) + objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra_compile_args) + cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args) return efile def compile_and_run(basedir, src, compiler=None, **compiler_attrs): diff --git a/buildutils/misc.py b/buildutils/misc.py index fabb52d..6f0c370 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -24,6 +24,8 @@ def customize_mingw(cc): if 'msvcr90' in cc.dll_libraries: cc.dll_libraries.remove('msvcr90') +def customize_msvc(cc): + pass def get_compiler(compiler, **compiler_attrs): """get and customize a compiler""" @@ -32,6 +34,8 @@ def get_compiler(compiler, **compiler_attrs): customize_compiler(cc) if cc.compiler_type == 'mingw32': customize_mingw(cc) + elif cc.compiler_type == 'msvc': + customize_msvc(cc) else: cc = compiler From 722579d671749cb22688e565941a3d0a19487d8d Mon Sep 17 00:00:00 2001 From: Yann Diorcet Date: Thu, 18 May 2017 11:47:51 +0200 Subject: [PATCH 112/126] Guess include and lib directory using capnp executable --- buildutils/detect.py | 4 ++-- setup.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/buildutils/detect.py b/buildutils/detect.py index 03f1be4..4a1c117 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -148,7 +148,7 @@ def detect_version(basedir, compiler=None, **compiler_attrs): return props -def test_build(): +def test_build(**compiler_attrs): """do a test build of libcapnp""" tmp_dir = tempfile.mkdtemp() @@ -156,7 +156,7 @@ def test_build(): # info("Configure: Autodetecting Cap'n Proto settings...") # info(" Custom Cap'n Proto dir: %s" % prefix) try: - detected = detect_version(tmp_dir) + detected = detect_version(tmp_dir, None, **compiler_attrs) finally: erase_dir(tmp_dir) diff --git a/setup.py b/setup.py index f070213..09ac805 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ import sys from distutils.command.clean import clean as _clean from distutils.errors import CompileError from distutils.extension import Extension +from distutils.spawn import find_executable from setuptools import setup @@ -113,10 +114,16 @@ class build_libcapnp_ext(build_ext_c): elif force_system_libcapnp: need_build = False else: + # Try to use capnp executable to find include and lib path + capnp_executable = find_executable("capnp") + if capnp_executable: + self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')] + self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')] + # Try to autodetect presence of library. Requires compile/run # step so only works for host (non-cross) compliation try: - test_build() + test_build(include_dirs=self.include_dirs, library_dirs=self.library_dirs) need_build = False except CompileError: need_build = True From 940ab9916d8e0d30f4548fcb1ace8987cd45db75 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Sat, 5 Oct 2019 14:50:56 -0700 Subject: [PATCH 113/126] Adding reconnecting async ssl example - async_reconnecting_ssl_client.py will automatically close and reconnect to a server when it becomes available (rather than hanging or dying when the server disappears) --- examples/async_reconnecting_ssl_client.py | 158 ++++++++++++++++++++++ examples/async_ssl_server.py | 3 + examples/thread.capnp | 1 + test/test_examples.py | 7 + 4 files changed, 169 insertions(+) create mode 100755 examples/async_reconnecting_ssl_client.py diff --git a/examples/async_reconnecting_ssl_client.py b/examples/async_reconnecting_ssl_client.py new file mode 100755 index 0000000..ca93836 --- /dev/null +++ b/examples/async_reconnecting_ssl_client.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import asyncio +import argparse +import os +import time +import capnp +import socket +import ssl +import time + +import thread_capnp + +this_dir = os.path.dirname(os.path.abspath(__file__)) +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + try: + # Must be a wait_for in order to give watch_connection a slot + # to try again + data = await asyncio.wait_for(reader.read(4096), timeout=1.0) + except asyncio.TimeoutError: + continue + client.write(data) + + +async def mywriter(client, writer): + while True: + try: + # Must be a wait_for in order to give watch_connection a slot + # to try again + data = await asyncio.wait_for(client.read(4096), timeout=1.0) + writer.write(data.tobytes()) + #await writer.drain() + except asyncio.TimeoutError: + continue + + +async def watch_connection(cap): + while True: + try: + await asyncio.wait_for(cap.alive().a_wait(), timeout=5) + await asyncio.sleep(1) + except asyncio.TimeoutError: + print("Watch timeout!") + asyncio.get_running_loop().stop() + return False + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert')) + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + ) + except OSError: + print("Try IPv6") + try: + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + family=socket.AF_INET6 + ) + except OSError: + return False + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Start watcher to restart socket connection if it is lost + overalltasks = [] + watcher = [watch_connection(cap)] + overalltasks.append(asyncio.gather(*watcher, return_exceptions=True)) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + overalltasks.append(asyncio.gather(*coroutines, return_exceptions=True)) + + # Start background task for subscriber + tasks = [background(cap)] + overalltasks.append(asyncio.gather(*tasks, return_exceptions=True)) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + + for task in overalltasks: + task.cancel() + + return True + +if __name__ == '__main__': + # Using asyncio.run hits an asyncio ssl bug + # https://bugs.python.org/issue36709 + # asyncio.run(main(parse_args().host), loop=loop, debug=True) + retry = True + while retry: + loop = asyncio.new_event_loop() + try: + retry = not loop.run_until_complete(main(parse_args().host)) + except RuntimeError: + # If an IO is hung, the event loop will be stopped + # and will throw RuntimeError exception + continue + if retry: + time.sleep(1) + print("Retrying...") + +# How this works +# - There are two retry mechanisms +# 1. Connection retry +# 2. alive RPC verification +# - The connection retry just loops the connection (IPv4+IPv6 until there is a connection or Ctrl+C) +# - The alive RPC verification attempts a very basic rpc call with a timeout +# * If there is a timeout, stop the current event loop +# * Use the RuntimeError exception to force a reconnect +# * myreader and mywriter must also be wrapped in wait_for in order for the events to get triggered correctly diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py index c162109..06a7d28 100755 --- a/examples/async_ssl_server.py +++ b/examples/async_ssl_server.py @@ -27,6 +27,9 @@ class ExampleImpl(thread_capnp.Example.Server): def longRunning(self, **kwargs): return capnp.getTimer().after_delay(1 * 10**9) + def alive(self, **kwargs): + return True + async def myreader(server, reader): while True: diff --git a/examples/thread.capnp b/examples/thread.capnp index ae32b8d..8caf56f 100644 --- a/examples/thread.capnp +++ b/examples/thread.capnp @@ -8,4 +8,5 @@ interface Example { longRunning @0 () -> (value: Bool); subscribeStatus @1 (subscriber: StatusSubscriber); + alive @2 () -> (value: Bool); } diff --git a/test/test_examples.py b/test/test_examples.py index 2f39b7d..d247549 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -50,3 +50,10 @@ def test_ssl_async_example(): server = 'async_ssl_server.py' client = 'async_ssl_client.py' run_subprocesses(address, server, client) + + +def test_ssl_reconnecting_async_example(): + address = 'localhost:36435' + server = 'async_ssl_server.py' + client = 'async_reconnecting_ssl_client.py' + run_subprocesses(address, server, client) From de61c304e105e53534afe2addb818dd317ab9c41 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Sat, 12 Oct 2019 16:21:58 -0700 Subject: [PATCH 114/126] Adding sleep delay - Reduces 99% CPU usage to around 1% - It might be useful to have the sleep/delay tunable for certain applications depending on the latency requirements --- capnp/lib/capnp.pyx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 8824122..367b78a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1955,7 +1955,7 @@ cdef class _RemotePromise: raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') while not helpers.pollRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope): - await asyncio.sleep(0) + await asyncio.sleep(0.01) ret = self._wait() self.is_consumed = True @@ -2322,7 +2322,7 @@ cdef class TwoPartyClient: bufsize ) while not reader.poll(): - await asyncio.sleep(0) + await asyncio.sleep(0.01) cdef array.array read_buffer = array.array('b', []) array.resize(read_buffer, reader.read_size()) @@ -2464,7 +2464,7 @@ cdef class TwoPartyServer: bufsize ) while not reader.poll(): - await asyncio.sleep(0) + await asyncio.sleep(0.01) cdef array.array read_buffer = array.array('b', []) array.resize(read_buffer, reader.read_size()) @@ -2511,7 +2511,7 @@ cdef class TwoPartyServer: async def poll_forever(self): while True: poll_once() - await asyncio.sleep(0) + await asyncio.sleep(0.01) cpdef run_forever(self): if self.port_promise is None: From 12ddd743ef1457c199d74ae117ac502a5d24f325 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Mon, 14 Oct 2019 11:19:33 -0700 Subject: [PATCH 115/126] Fork pycapnp to pycapnp-async - Breaking some earlier compatibility to cleanup build messages - As well as being able to publish pypi releases * Builds can be complicated to package correctly - Windows support --- README.md | 92 +++++++++++++++++++++++++++++++++++-------------------- setup.py | 20 ++++++------ 2 files changed, 68 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 30f8d28..d99d681 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,96 @@ -# pycapnp +# pycapnp-async + +[![Actions Status](https://github.com/haata/pycapnp-async/workflows/Python%20Test%20Packaging/badge.svg)](https://github.com/haata/pycapnp-async/actions) -More thorough docs are available at [http://jparyani.github.io/pycapnp/](http://jparyani.github.io/pycapnp/). ## Requirements -pycapnp's distribution has no requirements beyond a C++11 compatible compiler. GCC 4.8+ or Clang 3.3+ should work fine. +* C++14 supported compiler + - gcc 6.1+ (5+ may work) + - clang 6 (3.4+ may work) + - Visual Studio 2017+ +* cmake (needed for bundled capnproto) + - ninja (macOS + Linux) + - Visual Studio 2017+ + +* capnproto-0.7.0 + - Not necessary if using bundled capnproto + +32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake. + +pycapnp has additional development dependencies, including cython and pytest. See requirements.txt for them all. -pycapnp has additional development dependencies, including cython and py.test. See requirements.txt for them all. ## Building and installation -Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-4.8 pip install pycapnp`. +Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-8.2 pip install pycapnp`. Or you can clone the repo like so: - git clone https://github.com/jparyani/pycapnp.git - pip install --install-option '--force-cython' ./pycapnp - -Note: for OSX, if using clang from Xcode 5, you may need to set `CFLAGS` like so: - - CFLAGS='-stdlib=libc++' pip install pycapnp +```bash +git clone https://github.com/haata/pycapnp-async.git +cd pycapnp-async +pip install . +``` If you wish to install using the latest upstream C++ Cap'n Proto: - pip install --install-option "--libcapnp-url" --install-option "https://github.com/sandstorm-io/capnproto/archive/master.tar.gz" --install-option "--force-bundled-libcapnp" . +```bash +pip install \ + --install-option "--libcapnp-url" \ + --install-option "https://github.com/sandstorm-io/capnproto/archive/master.tar.gz" \ + --install-option "--force-bundled-libcapnp" . +``` + +To force bundled python: + +```bash +pip install --install-option "--force-bundled-libcapnp" . +``` ## Python Versions -Python 2.7, Python 3.4+, and PyPy 2.1+ are supported. +Python 3.7+ is supported. +Earlier versions of Python have asyncio bugs that might be possible to work around, but may require significant work (3.5 and 3.6). -One oddity to note is that `Text` type fields will be treated as byte strings under Python 2, and unicode strings under Python 3. `Data` fields will always be treated as byte strings. ## Development -This project uses [git-flow](http://jeffkreeftmeijer.com/2010/why-arent-you-using-git-flow/). Essentially, just make sure you do your changes in the `develop` branch. You can run the tests by installing pytest with `pip install pytest`, and then run `py.test` from the `test` directory. +Git flow has been abandoned, use master. + +To test, use a pipenv (or install requirements.txt and run pytest manually). +```bash +pip install pipenv +pipenv install +pipenv run pytest +``` + ### Binary Packages Building a dumb binary distribution: - python setup.py bdist_dumb +```bash +python setup.py bdist_dumb +``` Building a Python wheel distributiion: - python setup.py bdist_wheel +```bash +python setup.py bdist_wheel +``` -If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with: - python setup.py build --force-cython +### Pypi Upload Instructions + +Only necessary if uploading release to pypi.org. + +TODO + ## Documentation/Example + There is some basic documentation [here](http://jparyani.github.io/pycapnp/). The examples directory has one example that shows off pycapnp quite nicely. Here it is, reproduced: @@ -161,17 +201,3 @@ if __name__ == '__main__': server(write_end) client(read_end) ``` - -## Common Problems - -If you get an error on installation like: - - ... - gcc-4.8: error: capnp/capnp.c: No such file or directory - - gcc-4.8: fatal error: no input files - -Then you have too old a version of setuptools. Run `pip install -U setuptools` then try again. - - -[![Build Status](https://travis-ci.org/jparyani/pycapnp.png?branch=develop)](https://travis-ci.org/jparyani/pycapnp) diff --git a/setup.py b/setup.py index 09ac805..5c47573 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python ''' -pycapnp distutils setup.py +pycapnp-async distutils setup.py ''' from __future__ import print_function @@ -20,15 +20,15 @@ from buildutils import test_build, fetch_libcapnp, build_libcapnp, info _this_dir = os.path.dirname(__file__) MAJOR = 0 -MINOR = 6 -MICRO = 4 +MINOR = 7 +MICRO = 0 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) # Write version info def write_version_py(filename=None): ''' - Generate pycapnp version + Generate pycapnp-async version ''' cnt = """\ version = '%s' @@ -156,7 +156,7 @@ import Cython # noqa: F401 extensions = cythonize('capnp/lib/*.pyx') setup( - name="pycapnp", + name="pycapnp-async", packages=["capnp"], version=VERSION, package_data={ @@ -178,10 +178,10 @@ setup( description="A cython wrapping of the C++ Cap'n Proto library", long_description=long_description, license='BSD', - author="Jason Paryani", - author_email="pypi-contact@jparyani.com", - url='https://github.com/jparyani/pycapnp', - download_url='https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, + author="Jacob Alexander", + author_email="haata@kiibohd.com", + url='https://github.com/haata/pycapnp-async', + download_url='https://github.com/haata/pycapnp-async/archive/v%s.zip' % VERSION, keywords=['capnp', 'capnproto', "Cap'n Proto"], classifiers=[ 'Development Status :: 4 - Beta', @@ -191,8 +191,6 @@ setup( 'Operating System :: POSIX', 'Programming Language :: C++', 'Programming Language :: Cython', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Communications'], From f6dd08dda699dcd62b01775753ee8fc621839e5d Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Mon, 14 Oct 2019 23:19:39 -0700 Subject: [PATCH 116/126] Removing deprecated Restorer and ezRestore references - Not recommended to be used in new designs - Just pollutes warning messages during compilation (hiding ones that should be fixed) - Updated test code to use bootstrap - Sped up some of the test code that was just sleeping while waiting for the server (now polling for the socket) --- capnp/helpers/helpers.pxd | 9 +- capnp/helpers/non_circular.pxd | 2 - capnp/helpers/rpcHelper.h | 118 -------------------------- capnp/includes/capnp_cpp.pxd | 3 +- capnp/lib/capnp.pxd | 4 +- capnp/lib/capnp.pyx | 147 +++------------------------------ test/test_examples.py | 17 +++- test/test_rpc.py | 85 ++----------------- test/test_rpc_calculator.py | 30 ++++++- test/test_threads.py | 23 +----- 10 files changed, 70 insertions(+), 368 deletions(-) diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index 6e506f8..f34b530 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -1,4 +1,4 @@ -from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer +from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer from capnp.includes.schema_cpp cimport ByteArray @@ -27,15 +27,8 @@ cdef extern from "capnp/helpers/capabilityHelper.h": VoidPromise convert_to_voidpromise(PyPromise&) cdef extern from "capnp/helpers/rpcHelper.h": - Capability.Client restoreHelper(RpcSystem&) - Capability.Client restoreHelper(RpcSystem&, MessageBuilder&) - Capability.Client restoreHelper(RpcSystem&, MessageReader&) - Capability.Client restoreHelper(RpcSystem&, AnyPointer.Reader&) - Capability.Client restoreHelper(RpcSystem&, AnyPointer.Builder&) Capability.Client bootstrapHelper(RpcSystem&) Capability.Client bootstrapHelperServer(RpcSystem&) - RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&) - PyPromise connectServerRestorer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr) PyPromise connectServer(TaskSet &, Capability.Client, AsyncIoContext *, StringPtr) cdef extern from "capnp/helpers/serialize.h": diff --git a/capnp/helpers/non_circular.pxd b/capnp/helpers/non_circular.pxd index b0a9ffd..c220673 100644 --- a/capnp/helpers/non_circular.pxd +++ b/capnp/helpers/non_circular.pxd @@ -12,8 +12,6 @@ cdef extern from "capnp/helpers/capabilityHelper.h": PyRefCounter(PyObject *) cdef extern from "capnp/helpers/rpcHelper.h": - cdef cppclass PyRestorer: - PyRestorer(PyObject *) cdef cppclass ErrorHandler: pass diff --git a/capnp/helpers/rpcHelper.h b/capnp/helpers/rpcHelper.h index 54f3081..44f098f 100644 --- a/capnp/helpers/rpcHelper.h +++ b/capnp/helpers/rpcHelper.h @@ -6,73 +6,6 @@ #include "Python.h" #include "capabilityHelper.h" -extern "C" { - capnp::Capability::Client * call_py_restorer(PyObject *, capnp::AnyPointer::Reader &); -} - -class PyRestorer final: public capnp::SturdyRefRestorer { -public: - PyRestorer(PyObject * _py_restorer): py_restorer(_py_restorer) { - // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere. - // Py_INCREF(py_restorer); - } - - // ~PyRestorer() { - // Py_DECREF(py_restorer); - // } - - capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override { - GILAcquire gil; - capnp::Capability::Client * ret = call_py_restorer(py_restorer, objectId); - check_py_error(); - capnp::Capability::Client stack_ret(*ret); - delete ret; - - return stack_ret; - } - -private: - PyObject * py_restorer; -}; - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::MessageBuilder & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId.getRoot()); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::MessageReader & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId.getRoot()); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::AnyPointer::Reader & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::AnyPointer::Builder & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - - capnp::MallocMessageBuilder blankMessage(8); - auto objectId = blankMessage.getRoot(); - return client.restore(hostId, objectId); -} - capnp::Capability::Client bootstrapHelper(capnp::RpcSystem& client) { capnp::MallocMessageBuilder hostIdMessage(8); auto hostId = hostIdMessage.initRoot(); @@ -87,63 +20,12 @@ capnp::Capability::Client bootstrapHelperServer(capnp::RpcSystem -capnp::RpcSystem makeRpcClientWithRestorer( - capnp::VatNetwork& network, - PyRestorer& restorer) { - using namespace capnp; - return RpcSystem(network, restorer); -} - -struct ServerContextRestorer { - kj::Own stream; - capnp::TwoPartyVatNetwork network; - capnp::RpcSystem rpcSystem; - - ServerContextRestorer(kj::Own&& stream, capnp::SturdyRefRestorer& restorer) - : stream(kj::mv(stream)), - network(*this->stream, capnp::rpc::twoparty::Side::SERVER), - rpcSystem(makeRpcServer(network, restorer)) {} -}; - class ErrorHandler : public kj::TaskSet::ErrorHandler { void taskFailed(kj::Exception&& exception) override { kj::throwFatalException(kj::mv(exception)); } }; -void acceptLoopRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::Own&& listener) { - auto ptr = listener.get(); - tasks.add(ptr->accept().then(kj::mvCapture(kj::mv(listener), - [&](kj::Own&& listener, - kj::Own&& connection) { - acceptLoopRestorer(tasks, restorer, kj::mv(listener)); - - auto server = kj::heap(kj::mv(connection), restorer); - - // Arrange to destroy the server context when all references are gone, or when the - // EzRpcServer is destroyed (which will destroy the TaskSet). - tasks.add(server->network.onDisconnect().attach(kj::mv(server))); - }))); -} - -kj::Promise connectServerRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::AsyncIoContext * context, kj::StringPtr bindAddress) { - auto paf = kj::newPromiseAndFulfiller(); - auto portPromise = paf.promise.fork(); - - tasks.add(context->provider->getNetwork().parseAddress(bindAddress) - .then(kj::mvCapture(paf.fulfiller, - [&](kj::Own>&& portFulfiller, - kj::Own&& addr) { - auto listener = addr->listen(); - portFulfiller->fulfill(listener->getPort()); - acceptLoopRestorer(tasks, restorer, kj::mv(listener)); - }))); - - return portPromise.addBranch().then([&](unsigned int port) { return PyLong_FromUnsignedLong(port); }); -} - struct ServerContext { kj::Own stream; diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index 450a05e..9e53c51 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -6,7 +6,7 @@ cdef extern from "capnp/helpers/checkCompiler.h": from libcpp cimport bool from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions -from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyRestorer, PyEventPort, ErrorHandler +from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyEventPort, ErrorHandler from capnp.includes.types cimport * cdef extern from "capnp/common.h" namespace " ::capnp": @@ -365,7 +365,6 @@ cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp": TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side, ReaderOptions) VoidPromise onDisconnect() VoidPromise onDrained() - RpcSystem makeRpcServer(TwoPartyVatNetwork&, PyRestorer&) RpcSystem makeRpcServerBootstrap"makeRpcServer"(TwoPartyVatNetwork&, Capability.Client) RpcSystem makeRpcClient(TwoPartyVatNetwork&) diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index 4d9c7e1..21c8950 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -2,7 +2,7 @@ from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp -from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder, TwoWayPipe +from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, RpcSystem, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder, TwoWayPipe from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode from capnp.includes.types cimport * from capnp.helpers.non_circular cimport reraise_kj_exception @@ -103,14 +103,12 @@ cdef class _Schema: cpdef as_struct(self) cpdef as_interface(self) cpdef as_enum(self) - cpdef get_dependency(self, id) cpdef get_proto(self) cdef class _InterfaceSchema: cdef C_InterfaceSchema thisptr cdef object __method_names, __method_names_inherited, __methods, __methods_inherited cdef _init(self, C_InterfaceSchema other) - cpdef get_dependency(self, id) cdef class _DynamicEnum: cdef capnp.DynamicEnum thisptr diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 367b78a..1310ea1 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -10,7 +10,6 @@ cimport cython -from capnp.helpers.helpers cimport makeRpcClientWithRestorer from capnp.helpers.helpers cimport AsyncIoStreamReadHelper from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope @@ -24,7 +23,6 @@ from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_WRITABLE from types import ModuleType as _ModuleType import os as _os import sys as _sys -import imp as _imp import traceback as _traceback from functools import partial as _partial import warnings as _warnings @@ -120,17 +118,6 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_ return NULL -cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except * with gil: - restorer = _restorer - reader = _DynamicObjectReader()._init(_reader, None) - - ret = restorer._restore(reader) - cdef _DynamicCapabilityServer server = ret - cdef _InterfaceSchema schema = ret.schema - - return new C_Capability.Client(helpers.server_to_client(schema.thisptr, server)) - - cdef public convert_array_pyobject(PyArray & arr) with gil: return [arr[i] for i in range(arr.size())] @@ -2233,21 +2220,6 @@ cdef class _CapabilityClient: s = schema return _DynamicCapabilityClient()._init(self.thisptr.castAs(s.thisptr), self._parent) -cdef class _Restorer: - cdef PyRestorer * thisptr - cdef public object restore, _parent - - def __init__(self, restore, parent=None): - self.thisptr = new PyRestorer(self) - self.restore = restore - self._parent = parent - - def __dealloc__(self): - del self.thisptr - - def _restore(self, obj): - return self.restore(obj) - cdef class _TwoPartyVatNetwork: cdef Own[C_TwoPartyVatNetwork] thisptr cdef _AsyncIoStream stream @@ -2264,27 +2236,14 @@ cdef class _TwoPartyVatNetwork: cpdef on_disconnect(self) except +reraise_kj_exception: return _VoidPromise()._init(deref(self.thisptr).onDisconnect(), self) -cdef _Restorer _convert_restorer(restorer): - if isinstance(restorer, _RestorerImpl): - return _Restorer(restorer._restore, restorer) - elif type(restorer) is _Restorer: - return restorer - elif hasattr(restorer, 'restore'): - return _Restorer(restorer.restore, restorer) - elif callable(restorer): - return _Restorer(restorer) - else: - raise KjException("Restorer object ({}) isn't able to be used as a restore".format(str(restorer))) - cdef class TwoPartyClient: cdef RpcSystem * thisptr cdef public _TwoPartyVatNetwork _network cdef public object _orig_stream - cdef public _Restorer _restorer cdef public _AsyncIoStream _stream cdef public _TwoWayPipe _pipe - def __init__(self, socket=None, restorer=None, traversal_limit_in_words=None, nesting_limit=None): + def __init__(self, socket=None, traversal_limit_in_words=None, nesting_limit=None): if isinstance(socket, basestring): socket = self._connect(socket) @@ -2299,15 +2258,7 @@ cdef class TwoPartyClient: self._pipe = _TwoWayPipe() self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.CLIENT, opts) - if restorer is None: - self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) - self._restorer = None - else: - _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) - self._restorer = _convert_restorer(restorer) - self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self._network.thisptr), deref(self._restorer.thisptr))) - - Py_INCREF(self._restorer) + self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) if self._orig_stream: Py_INCREF(self._orig_stream) Py_INCREF(self._stream) @@ -2355,47 +2306,6 @@ cdef class TwoPartyClient: sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) return sock - cpdef restore(self, objectId) except +reraise_kj_exception: - _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) - cdef _MessageBuilder builder - cdef _MessageReader reader - cdef _DynamicObjectBuilder object_builder - cdef _DynamicObjectReader object_reader - - if objectId is None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr)), self) - elif type(objectId) is _DynamicObjectBuilder: - object_builder = objectId - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(object_builder.thisptr)), self) - elif type(objectId) is _DynamicObjectReader: - object_reader = objectId - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), object_reader.thisptr), self) - else: - if not hasattr(objectId, 'is_root'): - raise KjException("objectId was not a valid Cap'n Proto struct") - if not objectId.is_root: - raise KjException("objectId must be the root of a Cap'n Proto message, ie. addressbook_capnp.Person.new_message()") - - try: - builder = objectId._parent - except: - reader = objectId._parent - - if builder is not None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(builder.thisptr)), self) - elif reader is not None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(reader.thisptr)), self) - else: - raise KjException("objectId unexpectedly was not convertible to the proper type") - - cpdef ez_restore(self, textId) except +reraise_kj_exception: - # ez-rpc from the C++ API uses Text under the hood - ref = _MallocMessageBuilder().get_root_as_any() - # objectId is an AnyPointer, so we have a special method for setting it to text - ref.set_as_text(textId) - - return self.restore(ref) - cpdef bootstrap(self) except +reraise_kj_exception: return _CapabilityClient()._init(helpers.bootstrapHelper(deref(self.thisptr)), self) @@ -2406,7 +2316,6 @@ cdef class TwoPartyServer: cdef RpcSystem * thisptr cdef public _TwoPartyVatNetwork _network cdef public object _orig_stream, _server_socket, _disconnect_promise - cdef public _Restorer _restorer cdef public _AsyncIoStream _stream cdef public _TwoWayPipe _pipe cdef object _port @@ -2414,18 +2323,17 @@ cdef class TwoPartyServer: cdef capnp.TaskSet * _task_set cdef capnp.ErrorHandler _error_handler - def __init__(self, socket=None, restorer=None, server_socket=None, bootstrap=None, + def __init__(self, socket=None, server_socket=None, bootstrap=None, traversal_limit_in_words=None, nesting_limit=None): - if not restorer and not bootstrap: - raise KjException("You must provide either a bootstrap interface or a restorer (deperecated) to a server constructor.") + if not bootstrap: + raise KjException("You must provide a bootstrap interface to a server constructor.") cdef _InterfaceSchema schema cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - self._restorer = None self._bootstrap = None if isinstance(socket, basestring): - self._connect(socket, restorer, bootstrap) + self._connect(socket, bootstrap) return self._orig_stream = socket @@ -2444,12 +2352,7 @@ cdef class TwoPartyServer: self._bootstrap = bootstrap schema = bootstrap.schema self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) - elif restorer: - _warnings.warn('Restorers are deprecated. Please use the new bootstrap methods.', UserWarning) - self._restorer = _convert_restorer(restorer) - self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr))) - Py_INCREF(self._restorer) Py_INCREF(self._orig_stream) Py_INCREF(self._stream) Py_INCREF(self._pipe) @@ -2479,23 +2382,19 @@ cdef class TwoPartyServer: len(data) ).wait(self._pipe._event_loop.thisptr.waitScope) - cpdef _connect(self, host_string, restorer, bootstrap): + cpdef _connect(self, host_string, bootstrap): cdef _InterfaceSchema schema cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() cdef capnp.StringPtr temp_string = capnp.StringPtr(host_string, len(host_string)) self._task_set = new capnp.TaskSet(self._error_handler) - if restorer: - self._restorer = _convert_restorer(restorer) - self.port_promise = Promise()._init(helpers.connectServerRestorer(deref(self._task_set), deref(self._restorer.thisptr), loop.thisptr, temp_string)) - else: - self._bootstrap = bootstrap - Py_INCREF(self._bootstrap) - schema = bootstrap.schema - self.port_promise = Promise()._init(helpers.connectServer(deref(self._task_set), helpers.server_to_client(schema.thisptr, bootstrap), loop.thisptr, temp_string)) + + self._bootstrap = bootstrap + Py_INCREF(self._bootstrap) + schema = bootstrap.schema + self.port_promise = Promise()._init(helpers.connectServer(deref(self._task_set), helpers.server_to_client(schema.thisptr, bootstrap), loop.thisptr, temp_string)) def _decref(self): Py_DECREF(self._bootstrap) - Py_DECREF(self._restorer) Py_INCREF(self._pipe) Py_DECREF(self._orig_stream) Py_DECREF(self._stream) @@ -2589,11 +2488,6 @@ cdef class _Schema: cpdef as_enum(self): return _EnumSchema()._init(self.thisptr.asEnum()) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - cpdef get_proto(self): return _NodeReader().init(self.thisptr.getProto()) @@ -2676,11 +2570,6 @@ cdef class _StructSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - def __richcmp__(_StructSchema self, _StructSchema other, mode): if mode == 2: return self.thisptr == other.thisptr @@ -2814,11 +2703,6 @@ cdef class _InterfaceSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - def __repr__(self): return '' % self.node.displayName @@ -2991,20 +2875,13 @@ cdef _new_message(self, kwargs, num_first_segment_words): msg.from_dict(kwargs) return msg -class _RestorerImpl(object): - pass - class _StructModuleWhich(object): pass class _StructModule(object): def __init__(self, schema, name): - def _restore(self, obj): - return self.restore(obj.as_struct(self.schema)) self.schema = schema - self.Restorer = type(name + '.Restorer', (_RestorerImpl,), {'schema':schema, '_restore':_restore}) - # Add enums for union fields for field, raw_field in zip(schema.node.struct.fields, schema.fields_list): if field.which() == 'group': diff --git a/test/test_examples.py b/test/test_examples.py index d247549..1223549 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -10,7 +10,22 @@ examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') def run_subprocesses(address, server, client): server = subprocess.Popen([os.path.join(examples_dir, server), address]) - time.sleep(1) # Give the server some small amount of time to start listening + retries = 30 + addr, port = address.split(':') + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" client = subprocess.Popen([os.path.join(examples_dir, client), address]) ret = client.wait() diff --git a/test/test_rpc.py b/test/test_rpc.py index 3d0ac40..535cba1 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,3 +1,7 @@ +''' +rpc test +''' + import pytest import capnp import socket @@ -7,103 +11,28 @@ import test_capability_capnp class Server(test_capability_capnp.TestInterface.Server): - def __init__(self, val=1): + def __init__(self, val=100): self.val = val def foo(self, i, j, **kwargs): return str(i * 5 + self.val) -def restore_func(ref_id): - return Server(100) - - -class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): - - def restore(self, ref_id): - assert ref_id.tag == 'testInterface' - return Server(100) - - -def test_simple_rpc(): - read, write = socket.socketpair(socket.AF_UNIX) - - restorer = SimpleRestorer() - _ = capnp.TwoPartyServer(write, restorer) - client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - def test_simple_rpc_with_options(): read, write = socket.socketpair(socket.AF_UNIX) - restorer = SimpleRestorer() - _ = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, bootstrap=Server()) # This traversal limit is too low to receive the response in, so we expect # an exception during the call. client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') with pytest.raises(capnp.KjException): - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) + cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) remote = cap.foo(i=5) _ = remote.wait() -def test_simple_rpc_restore_func(): - read, write = socket.socketpair(socket.AF_UNIX) - - _ = capnp.TwoPartyServer(write, restore_func) - client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - -def text_restore_func(objectId): - text = objectId.as_text() - assert text == 'testInterface' - return Server(100) - - -def test_ez_rpc(): - read, write = socket.socketpair(socket.AF_UNIX) - - _ = capnp.TwoPartyServer(write, text_restore_func) - client = capnp.TwoPartyClient(read) - - cap = client.ez_restore('testInterface') - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - cap = client.restore(test_capability_capnp.TestSturdyRefObjectId.new_message()) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - - with pytest.raises(capnp.KjException): - response = remote.wait() - def test_simple_rpc_bootstrap(): read, write = socket.socketpair(socket.AF_UNIX) diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index 6beea16..d5b3d10 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -23,7 +23,35 @@ def test_calculator(): def run_subprocesses(address): server = subprocess.Popen([examples_dir + '/calculator_server.py', address]) - time.sleep(2) # Give the server some small amount of time to start listening + retries = 30 + if 'unix' in address: + addr = address.split(':')[1] + while True: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + result = sock.connect_ex(addr) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" + else: + addr, port = address.split(':') + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" client = subprocess.Popen([examples_dir + '/calculator_client.py', address]) ret = client.wait() diff --git a/test/test_threads.py b/test/test_threads.py index 218557b..b53e2c0 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -44,7 +44,7 @@ class Server(test_capability_capnp.TestInterface.Server): ''' Server ''' - def __init__(self, val=1): + def __init__(self, val=100): self.val = val def foo(self, i, j, **kwargs): @@ -54,19 +54,6 @@ class Server(test_capability_capnp.TestInterface.Server): return str(i * 5 + self.val) -class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): - ''' - SimpleRestorer - ''' - - def restore(self, ref_id): - ''' - Restore - ''' - assert ref_id.tag == 'testInterface' - return Server(100) - - @pytest.mark.skipif( platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" @@ -81,8 +68,7 @@ def test_using_threads(): read, write = socket.socketpair(socket.AF_UNIX) def run_server(): - restorer = SimpleRestorer() - _ = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, bootstrap=Server()) capnp.wait_forever() server_thread = threading.Thread(target=run_server) @@ -90,10 +76,7 @@ def test_using_threads(): server_thread.start() client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) + cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) remote = cap.foo(i=5) response = remote.wait() From 10355a74ace3a1edab5ec8c7968569239cc53bfd Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Mon, 14 Oct 2019 23:32:37 -0700 Subject: [PATCH 117/126] More agressive warning fixes capnp/lib/capnp.cpp:35163:294: warning: moving a temporary object prevents copy elision [-Wpessimizing-move] ...*)__pyx_t_3), new ::capnp::DynamicStruct::Pipeline(std::move((( ::capn... ^ capnp/lib/capnp.cpp:35163:294: note: remove std::move call here ...std::move((( ::capnp::DynamicValue::Pipeline)__pyx_v_self->thisptr->get(__pyx_t_5)).releaseAs< ::capnp::DynamicStruct>())... ^~~~~~~~~~ ~ capnp/lib/capnp.cpp:39540:53: warning: moving a temporary object prevents copy elision [-Wpessimizing-move] __pyx_v_self->thisptr = new ::kj::AsyncIoContext(std::move( ::kj::se... ^ capnp/lib/capnp.cpp:39540:53: note: remove std::move call here ...= new ::kj::AsyncIoContext(std::move( ::kj::setupAsyncIo())); ^~~~~~~~~~ ~ capnp/lib/capnp.cpp:47838:294: warning: moving a temporary object prevents copy elision [-Wpessimizing-move] ...*)__pyx_t_3), new ::capnp::DynamicStruct::Pipeline(std::move((( ::capn... ^ capnp/lib/capnp.cpp:47838:294: note: remove std::move call here ...std::move((( ::capnp::DynamicValue::Pipeline)__pyx_v_self->thisptr->get(__pyx_t_5)).releaseAs< ::capnp::DynamicStruct>())... ^~~~~~~~~~ ~ --- capnp/lib/capnp.pyx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 1310ea1..a222e2b 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1477,7 +1477,7 @@ cdef class _DynamicStructPipeline: if type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init((self.thisptr.get(field)).asCapability(), self._parent) elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) + return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline((self.thisptr.get(field)).asStruct()), self._parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -1636,7 +1636,7 @@ cdef class _EventLoop: self._init() cdef _init(self) except +reraise_kj_exception: - self.thisptr = new capnp.AsyncIoContext(moveAsyncContext(capnp.setupAsyncIo())) + self.thisptr = new capnp.AsyncIoContext(capnp.setupAsyncIo()) def __dealloc__(self): del self.thisptr #TODO:MEMORY: fix problems with Promises still being around @@ -1976,7 +1976,7 @@ cdef class _RemotePromise: if type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init((self.thisptr.get(field)).asCapability(), self._parent) elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) + return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline((self.thisptr.get(field)).asStruct()), self._parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -3128,6 +3128,7 @@ cdef class SchemaParser: self._last_import_array = importArray ret = _ParsedSchema() + # TODO (HaaTa): Convert to parseFromDirectory() as per deprecation note ret._init_child(self.thisptr.parseDiskFile(displayName, diskPath, importArray.asArrayPtr())) return ret From f2651facde3c104f58af8000b3e13e02f98e26dc Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 15 Oct 2019 00:42:34 -0700 Subject: [PATCH 118/126] Updating capnproto bundling code to use CMake and ninja - Must faster - Also showing output by default (easier to diagnose errors) * ninja has minimal verbosity (unless there are errors) --- buildutils/build.py | 61 +++++++++++++++++++++++++++++++------------- buildutils/bundle.py | 6 +---- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/buildutils/build.py b/buildutils/build.py index 5797c46..c85a5cc 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -2,6 +2,8 @@ import subprocess import os +import shutil +import sys import tempfile def build_libcapnp(bundle_dir, build_dir, verbose=False): @@ -11,23 +13,46 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): bundle_dir = os.path.abspath(bundle_dir) capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') build_dir = os.path.abspath(build_dir) + tmp_dir = os.path.join(capnp_dir, 'build') + if not os.path.exists(tmp_dir): + os.mkdir(tmp_dir) - with tempfile.TemporaryFile() as f: - stdout = f - if verbose: - stdout = None - cxxflags = os.environ.get('CXXFLAGS', None) - os.environ['CXXFLAGS'] = (cxxflags or '') + ' -fPIC -O2 -DNDEBUG' - conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout) - returncode = conf.wait() - if returncode != 0: - raise RuntimeError('Configure failed') + cxxflags = os.environ.get('CXXFLAGS', None) + os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG' - make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout) - returncode = make.wait() - if cxxflags is None: - del os.environ['CXXFLAGS'] - else: - os.environ['CXXFLAGS'] = cxxflags - if returncode != 0: - raise RuntimeError('Make failed') + # Enable ninja for compilation if available + build_type = [] + if shutil.which('ninja'): + build_type = ['-G', 'Ninja'] + + # TODO Determine VS version + + args = [ + 'cmake', + '-DCMAKE_POSITION_INDEPENDENT_CODE=1', + '-DBUILD_TESTING=OFF', + '-DBUILD_SHARED_LIBS=OFF', + '-DCMAKE_INSTALL_PREFIX:PATH={}'.format(build_dir), + capnp_dir, + ] + args.extend(build_type) + conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) + returncode = conf.wait() + if returncode != 0: + raise RuntimeError('CMake failed') + + # Run build through cmake + build = subprocess.Popen([ + 'cmake', + '--build', + '.', + '--target', + 'install', + ], cwd=tmp_dir, stdout=sys.stdout) + returncode = build.wait() + if cxxflags is None: + del os.environ['CXXFLAGS'] + else: + os.environ['CXXFLAGS'] = cxxflags + if returncode != 0: + raise RuntimeError('capnproto compilation failed') diff --git a/buildutils/bundle.py b/buildutils/bundle.py index 817797b..c9e83df 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -33,7 +33,7 @@ pjoin = os.path.join # Constants # -bundled_version = (0, 7, 4) +bundled_version = (0, 7, 0) libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp_name @@ -92,10 +92,6 @@ def fetch_libcapnp(savedir, url=None): shutil.move(with_version, dest) else: cpp_dir = os.path.join(with_version, 'c++') - conf = Popen(['autoreconf', '-i'], cwd=cpp_dir) - returncode = conf.wait() - if returncode != 0: - raise RuntimeError('Autoreconf failed. Make sure autotools are installed on your system.') shutil.move(cpp_dir, dest) From 0e830c212721a6cc62eef9e08598e5ec91699ec5 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 15 Oct 2019 01:04:00 -0700 Subject: [PATCH 119/126] Minor test fixes for Linux with IPv6 --- test/test_examples.py | 2 +- test/test_rpc_calculator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_examples.py b/test/test_examples.py index 1223549..dfbff5e 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -68,7 +68,7 @@ def test_ssl_async_example(): def test_ssl_reconnecting_async_example(): - address = 'localhost:36435' + address = 'localhost:36436' server = 'async_ssl_server.py' client = 'async_reconnecting_ssl_client.py' run_subprocesses(address, server, client) diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index d5b3d10..ed54094 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -60,7 +60,7 @@ def run_subprocesses(address): def test_calculator_tcp(): - address = '127.0.0.1:36431' + address = 'localhost:36431' run_subprocesses(address) From 7789ebbf96593801591bba25fbf6f0f6b883678e Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 15 Oct 2019 01:08:11 -0700 Subject: [PATCH 120/126] Fixing flake8 linting errors --- buildutils/build.py | 1 - examples/async_calculator_client.py | 34 +++++++++++------------ examples/async_calculator_server.py | 17 ++++++------ examples/async_reconnecting_ssl_client.py | 5 ++-- test/test_examples.py | 2 -- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/buildutils/build.py b/buildutils/build.py index c85a5cc..efbe739 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -4,7 +4,6 @@ import subprocess import os import shutil import sys -import tempfile def build_libcapnp(bundle_dir, build_dir, verbose=False): ''' diff --git a/examples/async_calculator_client.py b/examples/async_calculator_client.py index de9c823..90f29f9 100755 --- a/examples/async_calculator_client.py +++ b/examples/async_calculator_client.py @@ -26,16 +26,16 @@ class PowerFunction(calculator_capnp.Calculator.Function.Server): async def myreader(client, reader): - while True: - data = await reader.read(4096) - client.write(data) + while True: + data = await reader.read(4096) + client.write(data) async def mywriter(client, writer): - while True: - data = await client.read(4096) - writer.write(data.tobytes()) - await writer.drain() + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() def parse_args(): @@ -52,16 +52,16 @@ async def main(host): port = host[1] # Handle both IPv4 and IPv6 cases try: - print("Try IPv4") - reader, writer = await asyncio.open_connection( - addr, port, - ) - except: - print("Try IPv6") - reader, writer = await asyncio.open_connection( - addr, port, - family=socket.AF_INET6 - ) + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) client = capnp.TwoPartyClient() diff --git a/examples/async_calculator_server.py b/examples/async_calculator_server.py index 1316571..8324902 100755 --- a/examples/async_calculator_server.py +++ b/examples/async_calculator_server.py @@ -4,23 +4,22 @@ from __future__ import print_function import argparse import asyncio import socket -import random import capnp import calculator_capnp async def myreader(client, reader): - while True: - data = await reader.read(4096) - await client.write(data) + while True: + data = await reader.read(4096) + await client.write(data) async def mywriter(client, writer): - while True: - data = await client.read(4096) - writer.write(data.tobytes()) - await writer.drain() + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() def read_value(value): @@ -167,7 +166,7 @@ async def main(): myserver, addr, port, ) - except: + except Exception: print("Try IPv6") server = await asyncio.start_server( myserver, diff --git a/examples/async_reconnecting_ssl_client.py b/examples/async_reconnecting_ssl_client.py index ca93836..a4967bf 100755 --- a/examples/async_reconnecting_ssl_client.py +++ b/examples/async_reconnecting_ssl_client.py @@ -6,10 +6,10 @@ import asyncio import argparse import os import time -import capnp import socket import ssl -import time + +import capnp import thread_capnp @@ -51,7 +51,6 @@ async def mywriter(client, writer): # to try again data = await asyncio.wait_for(client.read(4096), timeout=1.0) writer.write(data.tobytes()) - #await writer.drain() except asyncio.TimeoutError: continue diff --git a/test/test_examples.py b/test/test_examples.py index dfbff5e..1477f59 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -1,8 +1,6 @@ -import gc import os import socket import subprocess -import sys # add examples dir to sys.path import time examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') From 67d52769367cc4e9249673fb64a1f7f98d4cdf09 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 15 Oct 2019 09:05:01 -0700 Subject: [PATCH 121/126] Defaulting to built-in capnproto - Adding code to detect if bundled capnproto is already built --- .github/workflows/pythonpackage.yml | 8 +------- buildutils/build.py | 11 +++++++---- setup.py | 13 +++++++++++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 41afa2a..cd6c5dc 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -25,15 +25,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - # Install capnproto - curl -O https://capnproto.org/capnproto-c++-0.7.0.tar.gz - tar zxf capnproto-c++-0.7.0.tar.gz - cd capnproto-c++-0.7.0 - ./configure - make -j check - sudo make install - name: Build pycapnp and install run: | + python setup.py build # Not necessary, but shows output on stdout pip install . - name: Lint with flake8 run: | diff --git a/buildutils/build.py b/buildutils/build.py index efbe739..b1c33fa 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -13,8 +13,11 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') build_dir = os.path.abspath(build_dir) tmp_dir = os.path.join(capnp_dir, 'build') - if not os.path.exists(tmp_dir): - os.mkdir(tmp_dir) + + # Clean the tmp build directory every time + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + os.mkdir(tmp_dir) cxxflags = os.environ.get('CXXFLAGS', None) os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG' @@ -38,7 +41,7 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = conf.wait() if returncode != 0: - raise RuntimeError('CMake failed') + raise RuntimeError('CMake failed {}'.format(returncode)) # Run build through cmake build = subprocess.Popen([ @@ -54,4 +57,4 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): else: os.environ['CXXFLAGS'] = cxxflags if returncode != 0: - raise RuntimeError('capnproto compilation failed') + raise RuntimeError('capnproto compilation failed: {}'.format(returncode)) diff --git a/setup.py b/setup.py index 5c47573..5aa9829 100644 --- a/setup.py +++ b/setup.py @@ -141,9 +141,18 @@ class build_libcapnp_ext(build_ext_c): build_dir = os.path.join(_this_dir, "build") if not os.path.exists(build_dir): os.mkdir(build_dir) - fetch_libcapnp(bundle_dir, libcapnp_url) - build_libcapnp(bundle_dir, build_dir) + # Check if we've already built capnproto + capnp_bin = os.path.join(build_dir, 'bin', 'capnp') + if os.name == 'nt': + capnp_bin = os.path.join(build_dir, 'bin', 'capnp.exe') + + if not os.path.exists(capnp_bin): + # Not built, fetch and build + fetch_libcapnp(bundle_dir, libcapnp_url) + build_libcapnp(bundle_dir, build_dir) + else: + info("capnproto already built at {}".format(build_dir)) self.include_dirs += [os.path.join(build_dir, 'include')] self.library_dirs += [os.path.join(build_dir, 'lib')] From 063522d30877e4d4a4321b1f645af54fa361cd42 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Tue, 15 Oct 2019 18:09:11 -0700 Subject: [PATCH 122/126] Adding Windows compilation support - Automatically determining build arch from running Python shell * Should work across all platforms --- .github/workflows/pythonpackage.yml | 2 +- buildutils/build.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index cd6c5dc..daf985d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,7 +13,7 @@ jobs: # Some asyncio commands require 3.7+ # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work python-version: [3.7] - os: [ubuntu-latest, macOS-latest] + os: [ubuntu-latest, macOS-latest, windows-latest] steps: - uses: actions/checkout@v1 diff --git a/buildutils/build.py b/buildutils/build.py index b1c33fa..b9d6ff1 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -3,9 +3,10 @@ import subprocess import os import shutil +import struct import sys -def build_libcapnp(bundle_dir, build_dir, verbose=False): +def build_libcapnp(bundle_dir, build_dir): ''' Build capnproto ''' @@ -27,7 +28,18 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): if shutil.which('ninja'): build_type = ['-G', 'Ninja'] - # TODO Determine VS version + # Determine python shell architecture + python_arch = 8 * struct.calcsize("P") + build_arch = [] + if os.name == 'nt': + if python_arch == 64: + build_arch_flag = "x64" + elif python_arch == 32: + build_arch_flag = "Win32" + else: + raise RuntimeError('Unknown windows build arch') + build_arch = ['-A', build_arch_flag] + print('Building module for {}'.format(python_arch)) args = [ 'cmake', @@ -38,6 +50,7 @@ def build_libcapnp(bundle_dir, build_dir, verbose=False): capnp_dir, ] args.extend(build_type) + args.extend(build_arch) conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = conf.wait() if returncode != 0: From 8062e6f4018bb7d999b9761c1540991b02447f4c Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Thu, 17 Oct 2019 00:20:03 -0700 Subject: [PATCH 123/126] Adding Windows 32-bit and 64-bit builds - Basic tests are working - May need some adjustments to get all tests working - Cleaned up bundling to take Python arch into account when building with multiple architectures --- .github/workflows/pythonpackage.yml | 4 +--- buildutils/build.py | 13 ++++++++++--- buildutils/detect.py | 3 ++- capnp/helpers/checkCompiler.h | 11 ++++------- capnp/includes/capnp_cpp.pxd | 1 - capnp/includes/schema_cpp.pxd | 1 - capnp/lib/capnp.pyx | 1 - capnp/templates/module.pyx | 1 - setup.py | 21 ++++++++++++++++----- 9 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index daf985d..2fb80b2 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -27,14 +27,12 @@ jobs: pip install -r requirements.txt - name: Build pycapnp and install run: | - python setup.py build # Not necessary, but shows output on stdout + python setup.py build pip install . - name: Lint with flake8 run: | pip install flake8 - # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude benchmark - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude benchmark - name: Test with pytest run: | diff --git a/buildutils/build.py b/buildutils/build.py index b9d6ff1..f355d1c 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -13,7 +13,7 @@ def build_libcapnp(bundle_dir, build_dir): bundle_dir = os.path.abspath(bundle_dir) capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') build_dir = os.path.abspath(build_dir) - tmp_dir = os.path.join(capnp_dir, 'build') + tmp_dir = os.path.join(capnp_dir, 'build{}'.format(8 * struct.calcsize("P"))) # Clean the tmp build directory every time if os.path.exists(tmp_dir): @@ -31,6 +31,7 @@ def build_libcapnp(bundle_dir, build_dir): # Determine python shell architecture python_arch = 8 * struct.calcsize("P") build_arch = [] + build_flags = [] if os.name == 'nt': if python_arch == 64: build_arch_flag = "x64" @@ -39,8 +40,12 @@ def build_libcapnp(bundle_dir, build_dir): else: raise RuntimeError('Unknown windows build arch') build_arch = ['-A', build_arch_flag] + build_flags = ['--config', 'Release'] print('Building module for {}'.format(python_arch)) + if not shutil.which('cmake'): + raise RuntimeError('Could not find cmake in your path!') + args = [ 'cmake', '-DCMAKE_POSITION_INDEPENDENT_CODE=1', @@ -57,13 +62,15 @@ def build_libcapnp(bundle_dir, build_dir): raise RuntimeError('CMake failed {}'.format(returncode)) # Run build through cmake - build = subprocess.Popen([ + args = [ 'cmake', '--build', '.', '--target', 'install', - ], cwd=tmp_dir, stdout=sys.stdout) + ] + args.extend(build_flags) + build = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = build.wait() if cxxflags is None: del os.environ['CXXFLAGS'] diff --git a/buildutils/detect.py b/buildutils/detect.py index 4a1c117..b9771f2 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -58,7 +58,8 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): else: lpreargs = ['-m64'] extra_compile_args = compiler_attrs.get('extra_compile_args', []) - extra_compile_args += ['--std=c++14'] + if os.name != 'nt': + extra_compile_args += ['--std=c++14'] extra_link_args = compiler_attrs.get('extra_link_args', []) if cc.compiler_type == 'msvc': extra_link_args += ['/MANIFEST'] diff --git a/capnp/helpers/checkCompiler.h b/capnp/helpers/checkCompiler.h index ed8d427..c5e34c1 100644 --- a/capnp/helpers/checkCompiler.h +++ b/capnp/helpers/checkCompiler.h @@ -1,11 +1,8 @@ -#ifdef __GNUC__ - #if __clang__ - #if __cplusplus >= 201103L && !__has_include() - #warning "Your compiler supports C++11 but your C++ standard library does not. If your system has libc++ installed (as should be the case on e.g. Mac OSX), try adding -stdlib=libc++ to your CFLAGS (ignore the other warning that says to use CXXFLAGS)." - #endif - #endif +#ifdef _MSC_VER +#pragma comment(lib, "Ws2_32.lib") +#pragma comment(lib, "advapi32.lib") #endif #include "capnp/dynamic.h" -static_assert(CAPNP_VERSION >= 5000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.5 and then re-install this python library"); +static_assert(CAPNP_VERSION >= 7000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.7 and then re-install this python library"); \ No newline at end of file diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index 9e53c51..4ba2fc4 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -1,6 +1,5 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++14 cdef extern from "capnp/helpers/checkCompiler.h": pass diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index c0ba23f..2c6ca2e 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -1,6 +1,5 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++14 from libc.stdint cimport * from capnp_cpp cimport DynamicOrphan diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index a222e2b..3c2f490 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1,6 +1,5 @@ # capnp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++14 # distutils: libraries = capnpc capnp-rpc capnp kj-async kj # distutils: include_dirs = . # cython: c_string_type = str diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index 1dcf7f5..e0ed81c 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -1,6 +1,5 @@ # addressbook_fast.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++14 # distutils: include_dirs = {{include_dir}} # distutils: libraries = capnpc capnp capnp-rpc # distutils: sources = {{file.filename}}.cpp diff --git a/setup.py b/setup.py index 5aa9829..db57b62 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ pycapnp-async distutils setup.py from __future__ import print_function import os +import struct import sys from distutils.command.clean import clean as _clean @@ -13,7 +14,7 @@ from distutils.errors import CompileError from distutils.extension import Extension from distutils.spawn import find_executable -from setuptools import setup +from setuptools import setup, find_packages, Extension from buildutils import test_build, fetch_libcapnp, build_libcapnp, info @@ -138,7 +139,7 @@ class build_libcapnp_ext(build_ext_c): bundle_dir = os.path.join(_this_dir, "bundled") if not os.path.exists(bundle_dir): os.mkdir(bundle_dir) - build_dir = os.path.join(_this_dir, "build") + build_dir = os.path.join(_this_dir, "build{}".format(8 * struct.calcsize("P"))) if not os.path.exists(build_dir): os.mkdir(build_dir) @@ -159,10 +160,20 @@ class build_libcapnp_ext(build_ext_c): return build_ext_c.run(self) +extra_compile_args = ['--std=c++14'] +extra_link_args = [] +if os.name == 'nt': + extra_compile_args = ['/std:c++14', '/MD'] + extra_link_args = ['/MANIFEST'] -from Cython.Build import cythonize +import Cython.Build import Cython # noqa: F401 -extensions = cythonize('capnp/lib/*.pyx') +extensions = [Extension( + '*', ['capnp/lib/*.pyx'], + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + language='c++', +)] setup( name="pycapnp-async", @@ -174,7 +185,7 @@ setup( 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*' ] }, - ext_modules=extensions, + ext_modules=Cython.Build.cythonize(extensions), cmdclass={ 'clean': clean, 'build_ext': build_libcapnp_ext From 964f5141806e5c92e51e41170c337c4dbd837da5 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Thu, 17 Oct 2019 00:45:14 -0700 Subject: [PATCH 124/126] Code gardening - Removing unused code sections --- buildutils/build.py | 2 +- buildutils/bundle.py | 89 +----------------------- buildutils/config.py | 137 ------------------------------------- buildutils/constants.py | 84 ----------------------- buildutils/detect.py | 21 ------ buildutils/setup_travis.sh | 15 ---- 6 files changed, 3 insertions(+), 345 deletions(-) delete mode 100644 buildutils/constants.py delete mode 100755 buildutils/setup_travis.sh diff --git a/buildutils/build.py b/buildutils/build.py index f355d1c..7b54c7e 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -28,7 +28,7 @@ def build_libcapnp(bundle_dir, build_dir): if shutil.which('ninja'): build_type = ['-G', 'Ninja'] - # Determine python shell architecture + # Determine python shell architecture for Windows python_arch = 8 * struct.calcsize("P") build_arch = [] build_flags = [] diff --git a/buildutils/bundle.py b/buildutils/bundle.py index c9e83df..acc39ed 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -13,19 +13,10 @@ import os import shutil -import stat -import sys import tarfile -from subprocess import Popen, PIPE -try: - # py2 - from urllib2 import urlopen -except ImportError: - # py3 - from urllib.request import urlopen - -from .msg import fatal, info, warn +from urllib.request import urlopen +from .msg import info pjoin = os.path.join @@ -94,79 +85,3 @@ def fetch_libcapnp(savedir, url=None): cpp_dir = os.path.join(with_version, 'c++') shutil.move(cpp_dir, dest) - -def stage_platform_hpp(capnproot): - """stage platform.hpp into libcapnp sources - - Tries ./configure first (except on Windows), - then falls back on included platform.hpp previously generated. - """ - - platform_hpp = pjoin(capnproot, 'src', 'platform.hpp') - if os.path.exists(platform_hpp): - info("already have platform.hpp") - return - if os.name == 'nt': - # stage msvc platform header - platform_dir = pjoin(capnproot, 'builds', 'msvc') - else: - info("attempting ./configure to generate platform.hpp") - - p = Popen('./configure', cwd=capnproot, shell=True, - stdout=PIPE, stderr=PIPE, - ) - _, e = p.communicate() - if p.returncode: - warn("failed to configure libcapnp:\n%s" % e) - if sys.platform == 'darwin': - platform_dir = pjoin(HERE, 'include_darwin') - elif sys.platform.startswith('freebsd'): - platform_dir = pjoin(HERE, 'include_freebsd') - elif sys.platform.startswith('linux-armv'): - platform_dir = pjoin(HERE, 'include_linux-armv') - else: - platform_dir = pjoin(HERE, 'include_linux') - else: - return - - info("staging platform.hpp from: %s" % platform_dir) - shutil.copy(pjoin(platform_dir, 'platform.hpp'), platform_hpp) - - -def copy_and_patch_libcapnp(capnp, libcapnp): - """copy libcapnp into source dir, and patch it if necessary. - - This command is necessary prior to running a bdist on Linux or OS X. - """ - if sys.platform.startswith('win'): - return - # copy libcapnp into capnp for bdist - local = localpath('capnp', libcapnp) - if not capnp and not os.path.exists(local): - fatal("Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` " - "or copy libcapnp into capnp/ manually prior to running bdist.") - try: - # resolve real file through symlinks - lib = os.path.realpath(pjoin(capnp, 'lib', libcapnp)) - print ("copying %s -> %s" % (lib, local)) - shutil.copy(lib, local) - except Exception: - if not os.path.exists(local): - fatal("Could not copy libcapnp into capnp/, which is necessary for bdist. " - "Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` " - "or copy libcapnp into capnp/ manually.") - - if sys.platform == 'darwin': - # chmod u+w on the lib, - # which can be user-read-only for some reason - mode = os.stat(local).st_mode - os.chmod(local, mode | stat.S_IWUSR) - # patch install_name on darwin, instead of using rpath - cmd = ['install_name_tool', '-id', '@loader_path/../%s' % libcapnp, local] - try: - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - except OSError: - fatal("install_name_tool not found, cannot patch libcapnp for bundling.") - _, err = p.communicate() - if p.returncode: - fatal("Could not patch bundled libcapnp install_name: %s" % err, p.returncode) diff --git a/buildutils/config.py b/buildutils/config.py index 6701259..277b776 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -11,147 +11,10 @@ # the file COPYING.BSD, distributed as part of this software. # -import sys -import os -import json - -from .msg import debug, warn - -try: - from configparser import ConfigParser -except Exception: - from ConfigParser import ConfigParser - -pjoin = os.path.join - # # Utility functions (adapted from h5py: http://h5py.googlecode.com) # - -def load_config(name, base='conf'): - """Load config dict from JSON""" - fname = pjoin(base, name + '.json') - if not os.path.exists(fname): - return {} - try: - with open(fname) as f: - cfg = json.load(f) - except Exception as e: - warn("Couldn't load %s: %s" % (fname, e)) - cfg = {} - return cfg - - -def save_config(name, data, base='conf'): - """Save config dict to JSON""" - if not os.path.exists(base): - os.mkdir(base) - fname = pjoin(base, name + '.json') - with open(fname, 'w') as f: - json.dump(data, f, indent=2) - - def v_str(v_tuple): """turn (2,0,1) into '2.0.1'.""" return ".".join(str(x) for x in v_tuple) - -def get_eargs(): - """ Look for options in environment vars """ - - settings = {} - - zmq = os.environ.get("ZMQ_PREFIX", None) - if zmq is not None: - debug("Found environ var ZMQ_PREFIX=%s" % zmq) - settings['zmq_prefix'] = zmq - - return settings - -def cfg2dict(cfg): - """turn a ConfigParser into a nested dict - - because ConfigParser objects are dumb. - """ - d = {} - for section in cfg.sections(): - d[section] = dict(cfg.items(section)) - return d - -def get_cfg_args(): - """ Look for options in setup.cfg """ - - if not os.path.exists('setup.cfg'): - return {} - cfg = ConfigParser() - cfg.read('setup.cfg') - cfg = cfg2dict(cfg) - - g = cfg.setdefault('global', {}) - # boolean keys: - for key in ['libzmq_extension', - 'bundle_libzmq_dylib', - 'no_libzmq_extension', - 'have_sys_un_h', - 'skip_check_zmq', - ]: - if key in g: - g[key] = eval(g[key]) - - # globals go to top level - cfg.update(cfg.pop('global')) - return cfg - -def config_from_prefix(prefix): - """Get config from zmq prefix""" - settings = {} - if prefix.lower() in ('default', 'auto', ''): - settings['zmq_prefix'] = '' - settings['libzmq_extension'] = False - settings['no_libzmq_extension'] = False - elif prefix.lower() in ('bundled', 'extension'): - settings['zmq_prefix'] = '' - settings['libzmq_extension'] = True - settings['no_libzmq_extension'] = False - else: - settings['zmq_prefix'] = prefix - settings['libzmq_extension'] = False - settings['no_libzmq_extension'] = True - return settings - -def merge(into, d): - """merge two containers - - into is updated, d has priority - """ - if isinstance(into, dict): - for key in d.keys(): - if key not in into: - into[key] = d[key] - else: - into[key] = merge(into[key], d[key]) - return into - if isinstance(into, list): - return into + d - return d - -def discover_settings(conf_base=None): - """ Discover custom settings for ZMQ path""" - settings = { - 'zmq_prefix': '', - 'libzmq_extension': False, - 'no_libzmq_extension': False, - 'skip_check_zmq': False, - 'build_ext': {}, - 'bdist_egg': {}, - } - if sys.platform.startswith('win'): - settings['have_sys_un_h'] = False - - if conf_base: - # lowest priority - merge(settings, load_config('config', conf_base)) - merge(settings, get_cfg_args()) - merge(settings, get_eargs()) - - return settings diff --git a/buildutils/constants.py b/buildutils/constants.py deleted file mode 100644 index fa39722..0000000 --- a/buildutils/constants.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -script for generating files that involve repetitive updates for zmq constants. - -Run this after updating utils/constant_names - -Currently generates the following files from templates: - -- constant_enums.pxi -- constants.pxi -- zmq_constants.h - -""" - -# Copyright (C) PyZMQ Developers -# Distributed under the terms of the Modified BSD License. - -import os -import sys - -from . import info -pjoin = os.path.join - -root = os.path.abspath(pjoin(os.path.dirname(__file__), os.path.pardir)) - -sys.path.insert(0, pjoin(root, 'zmq', 'utils')) -from constant_names import all_names, no_prefix # noqa: E402 - -ifndef_t = """#ifndef {0} - #define {0} (_PYZMQ_UNDEFINED) -#endif -""" - -def cython_enums(): - """generate `enum: ZMQ_CONST` block for constant_enums.pxi""" - lines = [] - for name in all_names: - if no_prefix(name): - lines.append('enum: ZMQ_{0} "{0}"'.format(name)) - else: - lines.append('enum: ZMQ_{0}'.format(name)) - - return dict(ZMQ_ENUMS='\n '.join(lines)) - -def ifndefs(): - """generate `#ifndef ZMQ_CONST` block for zmq_constants.h""" - lines = ['#define _PYZMQ_UNDEFINED (-9999)'] - for name in all_names: - if not no_prefix(name): - name = 'ZMQ_%s' % name - lines.append(ifndef_t.format(name)) - return dict(ZMQ_IFNDEFS='\n'.join(lines)) - -def constants_pyx(): - """generate CONST = ZMQ_CONST and __all__ for constants.pxi""" - all_lines = [] - assign_lines = [] - for name in all_names: - if name == "NULL": - # avoid conflict with NULL in Cython - assign_lines.append("globals()['NULL'] = ZMQ_NULL") - else: - assign_lines.append('{0} = ZMQ_{0}'.format(name)) - all_lines.append(' "{0}",'.format(name)) - return dict(ASSIGNMENTS='\n'.join(assign_lines), ALL='\n'.join(all_lines)) - -def generate_file(fname, ns_func, dest_dir="."): - """generate a constants file from its template""" - with open(pjoin(root, 'buildutils', 'templates', '%s' % fname), 'r') as f: - tpl = f.read() - out = tpl.format(**ns_func()) - dest = pjoin(dest_dir, fname) - info("generating %s from template" % dest) - with open(dest, 'w') as f: - f.write(out) - -def render_constants(): - """render generated constant files from templates""" - generate_file("constant_enums.pxi", cython_enums, pjoin(root, 'zmq', 'backend', 'cython')) - generate_file("constants.pxi", constants_pyx, pjoin(root, 'zmq', 'backend', 'cython')) - generate_file("zmq_constants.h", ifndefs, pjoin(root, 'zmq', 'utils')) - - -if __name__ == '__main__': - render_constants() diff --git a/buildutils/detect.py b/buildutils/detect.py index b9771f2..7d71345 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -21,7 +21,6 @@ import logging import platform from distutils import ccompiler from distutils.ccompiler import get_default_compiler -from subprocess import Popen, PIPE import tempfile from .misc import get_compiler, get_output_error @@ -68,26 +67,6 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args) return efile -def compile_and_run(basedir, src, compiler=None, **compiler_attrs): - """Compile and run""" - if not os.path.exists(basedir): - os.makedirs(basedir) - cfile = pjoin(basedir, os.path.basename(src)) - shutil.copy(src, cfile) - try: - cc = get_compiler(compiler, **compiler_attrs) - efile = test_compilation(cfile, compiler=cc) - patch_lib_paths(efile, cc.library_dirs) - result = Popen(efile, stdout=PIPE, stderr=PIPE) - so, se = result.communicate() - # for py3k: - so = so.decode() - se = se.decode() - finally: - shutil.rmtree(basedir) - - return result.returncode, so, se - def detect_version(basedir, compiler=None, **compiler_attrs): """Compile, link & execute a test program, in empty directory `basedir`. diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh deleted file mode 100755 index 7a89a3e..0000000 --- a/buildutils/setup_travis.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -exo pipefail - -CAPNP_VERSION=0.5.2 - -sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -sudo apt-get -qq update -sudo apt-get -qq install g++-4.8 libstdc++-4.8-dev -sudo update-alternatives --quiet --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.8 --slave /usr/bin/gcov gcov /usr/bin/gcov-4.8 -sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 - -if ! [ -z "${BUILD_CAPNP}" ]; then - wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz && tar xzvf capnproto-c++-${CAPNP_VERSION}.tar.gz && cd capnproto-c++-${CAPNP_VERSION} && ./configure && make -j6 && sudo make install && sudo ldconfig && cd .. -fi From 62eccff15044a1fe3e13fe9398ccf3aca8878a9b Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Fri, 18 Oct 2019 12:25:04 -0700 Subject: [PATCH 125/126] Adding capnp extension validation check - To catch odd import problems on Windows --- capnp/lib/capnp.pyx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 3c2f490..4a3a974 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3213,6 +3213,9 @@ cdef class SchemaParser: if not _os.path.isfile(file_name): raise IOError("File not found: " + file_name) + if not file_name.endswith('.capnp'): + raise ValueError("File does not end with .capnp, {}".format(file_name)) + if display_name is None: display_name = _os.path.basename(file_name) From 362cce345b12b55440a6baa1b8ba2698e8937df3 Mon Sep 17 00:00:00 2001 From: Jacob Alexander Date: Sat, 19 Oct 2019 00:22:59 -0700 Subject: [PATCH 126/126] Fixing Windows tests - Adding import path filter to exclude non-directories Otherwise kj will through exceptions - Skipped AF_UNIX socket test - Use default socket configuration when it doesn't matter the type of socket used - Open files with utf8 encoding (needed for text validation) - Explictly call python executable when running external scripts - Fix path creation to always use os.path.join - Added timeout to client wait in some tests - Some broken tests still remain (most likely asyncio related) --- capnp/lib/capnp.pyx | 8 +++++++- examples/async_client.py | 1 - examples/async_server.py | 1 - test/test_examples.py | 11 +++++++---- test/test_regression.py | 16 ++++++++-------- test/test_rpc.py | 4 ++-- test/test_rpc_calculator.py | 12 ++++++++---- test/test_threads.py | 2 +- 8 files changed, 33 insertions(+), 22 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 4a3a974..c8b9aba 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3224,7 +3224,13 @@ cdef class SchemaParser: module._parser = parser - fileSchema = parser._parse_disk_file(display_name, file_name, imports) + # Some systems (Windows running pytest) add non-directories to the sys.path used for imports + # Filter these out so kj doesn't implode when searching paths + filtered_imports = [] + for imp in imports: + if _os.path.isdir(imp): + filtered_imports.append(imp) + fileSchema = parser._parse_disk_file(display_name, file_name, filtered_imports) _load(fileSchema, module) abs_path = _os.path.abspath(file_name) diff --git a/examples/async_client.py b/examples/async_client.py index 7ffed54..63c3e42 100755 --- a/examples/async_client.py +++ b/examples/async_client.py @@ -39,7 +39,6 @@ async def mywriter(client, writer): while True: data = await client.read(4096) writer.write(data.tobytes()) - await writer.drain() async def background(cap): diff --git a/examples/async_server.py b/examples/async_server.py index 5d534fb..e6fa225 100755 --- a/examples/async_server.py +++ b/examples/async_server.py @@ -36,7 +36,6 @@ async def mywriter(server, writer): while True: data = await server.read(4096) writer.write(data.tobytes()) - await writer.drain() async def myserver(reader, writer): diff --git a/test/test_examples.py b/test/test_examples.py index 1477f59..1167418 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -1,13 +1,15 @@ import os import socket import subprocess +import sys import time examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') def run_subprocesses(address, server, client): - server = subprocess.Popen([os.path.join(examples_dir, server), address]) + cmd = [sys.executable, os.path.join(examples_dir, server), address] + server = subprocess.Popen(cmd) retries = 30 addr, port = address.split(':') while True: @@ -24,9 +26,10 @@ def run_subprocesses(address, server, client): retries -= 1 if retries == 0: assert False, "Timed out waiting for server to start" - client = subprocess.Popen([os.path.join(examples_dir, client), address]) + cmd = [sys.executable, os.path.join(examples_dir, client), address] + client = subprocess.Popen(cmd) - ret = client.wait() + ret = client.wait(timeout=30) server.kill() assert ret == 0 @@ -46,7 +49,7 @@ def test_thread_example(): def test_addressbook_example(): - proc = subprocess.Popen([os.path.join(examples_dir, 'addressbook.py')]) + proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')]) ret = proc.wait() assert ret == 0 diff --git a/test/test_regression.py b/test/test_regression.py index 12bde05..acddcee 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -309,7 +309,7 @@ def init_all_types(builder): subBuilder.uInt64Field = 345678901234567890 subBuilder.float32Field = -1.25e-10 subBuilder.float64Field = 345 - subBuilder.textField = b"\xe2\x98\x83".decode('utf-8') # This is u"☃", but py3.2 doesn't support u + subBuilder.textField = "☃" subBuilder.dataField = b"qux" subSubBuilder = subBuilder.structField subSubBuilder.textField = "nested" @@ -486,26 +486,26 @@ def check_all_types(reader): def test_build(all_types): root = all_types.TestAllTypes.new_message() init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_build_first_segment_size(all_types): root = all_types.TestAllTypes.new_message(1) init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText root = all_types.TestAllTypes.new_message(1024 * 1024) init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_binary_read(all_types): - f = open(os.path.join(this_dir, 'all-types.binary'), 'r') + f = open(os.path.join(this_dir, 'all-types.binary'), 'r', encoding='utf8') root = all_types.TestAllTypes.read(f) check_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText # Test set_root(). @@ -518,11 +518,11 @@ def test_binary_read(all_types): check_all_types(builder2.get_root(all_types.TestAllTypes)) def test_packed_read(all_types): - f = open(os.path.join(this_dir, 'all-types.packed'), 'r') + f = open(os.path.join(this_dir, 'all-types.packed'), 'r', encoding='utf8') root = all_types.TestAllTypes.read_packed(f) check_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_binary_write(all_types): diff --git a/test/test_rpc.py b/test/test_rpc.py index 535cba1..ba1c5e8 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -19,7 +19,7 @@ class Server(test_capability_capnp.TestInterface.Server): def test_simple_rpc_with_options(): - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() _ = capnp.TwoPartyServer(write, bootstrap=Server()) # This traversal limit is too low to receive the response in, so we expect @@ -34,7 +34,7 @@ def test_simple_rpc_with_options(): def test_simple_rpc_bootstrap(): - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() _ = capnp.TwoPartyServer(write, bootstrap=Server(100)) client = capnp.TwoPartyClient(read) diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index ed54094..50ba4e6 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -1,5 +1,6 @@ import gc import os +import pytest import socket import subprocess import sys # add examples dir to sys.path @@ -15,14 +16,15 @@ import calculator_server # noqa: E402 def test_calculator(): - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) calculator_client.main(read) def run_subprocesses(address): - server = subprocess.Popen([examples_dir + '/calculator_server.py', address]) + cmd = [sys.executable, os.path.join(examples_dir, 'calculator_server.py'), address] + server = subprocess.Popen(cmd) retries = 30 if 'unix' in address: addr = address.split(':')[1] @@ -52,7 +54,8 @@ def run_subprocesses(address): retries -= 1 if retries == 0: assert False, "Timed out waiting for server to start" - client = subprocess.Popen([examples_dir + '/calculator_client.py', address]) + cmd = [sys.executable, os.path.join(examples_dir, 'calculator_client.py'), address] + client = subprocess.Popen(cmd) ret = client.wait() server.kill() @@ -64,6 +67,7 @@ def test_calculator_tcp(): run_subprocesses(address) +@pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows") def test_calculator_unix(): path = '/tmp/pycapnp-test' try: @@ -81,7 +85,7 @@ def test_calculator_gc(): return old_evaluate_impl(*args, **kwargs) return call - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() # inject a gc.collect to the beginning of every evaluate_impl call evaluate_impl_orig = calculator_server.evaluate_impl diff --git a/test/test_threads.py b/test/test_threads.py index b53e2c0..bd7ae71 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -65,7 +65,7 @@ def test_using_threads(): capnp.remove_event_loop(True) capnp.create_event_loop(True) - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() def run_server(): _ = capnp.TwoPartyServer(write, bootstrap=Server())