diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000..2fb80b2 --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,40 @@ +name: Python Test Packaging + +on: [push, pull_request] + +jobs: + build: + + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 4 + fail-fast: false + matrix: + # Some asyncio commands require 3.7+ + # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work + python-version: [3.7] + os: [ubuntu-latest, macOS-latest, windows-latest] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + - name: Build pycapnp and install + run: | + python setup.py build + pip install . + - name: Lint with flake8 + run: | + pip install flake8 + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude benchmark + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude benchmark + - name: Test with pytest + run: | + pip install pytest + pytest diff --git a/.gitignore b/.gitignore index ad81cce..ff5b24b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,9 @@ capnp/*.cpp capnp/version.py MANIFEST docs/_build + +capnp/lib/capnp.cpp +capnp/lib/capnp.h +bundled/ +example +*.iml diff --git a/.travis.yml b/.travis.yml index e65ce0d..a852039 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,13 @@ +# Use older, non-container infrastructure to allow sudo +sudo: required + language: python python: - - 2.6 - 2.7 - - 3.3 - 3.4 + - 3.5 + - 3.6 - pypy env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab8d43..ef14edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,68 @@ +## v0.6.4 (2019-01-31) +- Fix bugs in `read_multiple_bytes` (thanks to @tsh56) +- Remove end-of-life Python versions 2.6, 3.2, and 3.3. Add CI tests for 3.6 +- Expose SchemaParser in Cython header + +## v0.6.3 (2018-01-14) +- Bump bundled capnp version to v0.6.1 (thanks to @E8Yuval) +- Fix a memleak in RemotePromise (thanks to @E8Yuval) + +## v0.6.2 (2017-11-30) +- Add support for buffers/memoryviews in `from_bytes` (thanks to @aldanor) + +## v0.6.1 (2017-07-27) +- Fixed upload to PyPi (forgot to cythonize) + +## v0.6.0 (2017-07-27) +- Update bundled capnp version to v0.6.0 and fix related problems (thanks to @benmoran) +- Fix memleak with KjException (thanks to @tsh56) + +## v0.5.12 (2017-04-18) +- Bump bundled capnp version to v0.5.3.1 + +## v0.5.11 (2017-04-10) +- Make enums hashable (thanks to @madeleine-empirical) +- Rework logic on when to build bundled libcapnp. Fixes cross-compilation (thanks to @benizl) +- Add traversal_limit_in_words and nesting_limit to RPC classes (thanks to @asilversempirical) +- Include class attributes in __dir__. This allows for code completion of class methods (thanks to @chaoflow ) +- Allow setting lists with python tuples (thanks to @chaoflow) +- Fix traversal_limit_in_words and nesting_limit being ignored by `from_bytes` (thanks to @plesner) + +## v0.5.10 (2016-11-28) +- Fix bug that prevented event loop from actually being lazy initialized +- Fix possible recursive loop in KjException +- Add `clear_write_flag` method to builder classes + +## v0.5.9 (2016-07-07) +- Make the event loop be lazy initialized +- Add support for segment (de)serialization (thanks to @gcv). See to_segments/from_segments methods. +- Fix response objects not referencing parents correctly +- Add test for large reads + +## v0.5.8 (2016-05-27) +- Fix build problem with Cython v0.24 +- Include the changelog in the manifest (should fix install problems if pandoc is present) +- Include the traceback in exceptions +- Make sure to encode to utf-8, not the default encoding (thanks to @novas0x2a) +- Add --libcapnp-url option in installer to allow installing arbitrary libcapnp versions +- Support mmap objects for reading with from_bytes (thanks to @bpiwowar) +- Change read_multiple and read_multiple_packed to copy by default +- Fix mistakenly discarding the file parameter on reads +- Add reraise_kj_exception to the prettyPrint functions. (thanks to @kdienes) +- Fix KjException init (missing wrapper). (thanks to @E8-Storage) +- Add `result_type` to InterfaceMethodSchema + + +## v0.5.7 (2015-06-16) +- Update bundled libcapnp to v0.5.2 +- Add warnings for using old restorer methods. You should use `bootstrap` instead +- Fix warning from PyEventPort +- Handle AnyPointers better as arguments to RPC functions +- Add support for using keyword arguments with a named struct in an RPC +- Add bootstrap method to TwoPartyServer +- Add `init` method to lists +- Add support for unix sockets in RPC + ## v0.5.6 (2015-04-13) - Fix a serious bug in TwoPartyServer that was preventing it from working when passed a string address. - Fix bugs that were exposed by defining KJDEBUG (thanks @davidcarne for finding this) diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..69eae40 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,44 @@ +# Deployment instructions for PyPi + +This file is meant for maintainers of pycapnp, and documents the process for uploading to PyPI. + +## Pre-requisites + +``` +pip install pypandoc cython +``` + +## Run tests + +I typically sanity check by running the tests once again locally, but as long as Travis is green, you're probably fine. + +## Add a commit that bumps the version + +Bump the version in setup.py, and add descriptions of all the changes to CHANGELOG.md (see 19e1b189caa786c7f572e679d6bb94aadfbdb5e0 for an example commit). + +## Run the build and upload + +Run the following command to clean up old artifacts, run the build, and then upload the result to PyPI + +``` +rm -rf bundled/ capnp/version.py capnp/lib/capnp.{h,cpp} build; python setup.py build && python setup.py sdist upload -r PyPI +``` + +## Test the PyPI release + +I manually test the PyPI release after it's been uploaded. I have a few virtualenvs that I manually run the following command in (run this from the pycapnp directory since it runs the tests at the end): +``` +yes | pip uninstall pycapnp; pip install pycapnp && py.test test +``` + +I usually test the following configurations: +- Python 2.7 with and without cython installed +- Python 3.6 with and without cython installed + +This step could probably benefit greatly from some automation. Perhaps even Travis could handle it, but I'm not sure how best to trigger Travis from a PyPI release. + +## Tag the github release + +Tag the release on the develop branch (not the master branch). Sadly, I've stopped using git-flow, and at this point it might be worth moving back to using just master, but that would take some amount of work and I worry that it would break open PRs. Definitely worth considering if development picks back up. + +Version numbers roughly follow semver, although I try to loosely follow upstream Cap'n Proto C++ versions as well. So when pycapnp officially starts using v0.7.0 of the C++ library, pycapnp's version should be bumped to v0.7.0 as well. diff --git a/MANIFEST.in b/MANIFEST.in index fc2d9af..1ea2323 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include README.md +include CHANGELOG.md include requirements.txt include buildutils/* diff --git a/README.md b/README.md index d4e9f9c..d99d681 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,96 @@ -# pycapnp +# pycapnp-async + +[![Actions Status](https://github.com/haata/pycapnp-async/workflows/Python%20Test%20Packaging/badge.svg)](https://github.com/haata/pycapnp-async/actions) -More thorough docs are available at [http://jparyani.github.io/pycapnp/](http://jparyani.github.io/pycapnp/). ## Requirements -pycapnp's distribution has no requirements beyond a C++11 compatible compiler. GCC 4.8+ or Clang 3.3+ should work fine. +* C++14 supported compiler + - gcc 6.1+ (5+ may work) + - clang 6 (3.4+ may work) + - Visual Studio 2017+ +* cmake (needed for bundled capnproto) + - ninja (macOS + Linux) + - Visual Studio 2017+ + +* capnproto-0.7.0 + - Not necessary if using bundled capnproto + +32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake. + +pycapnp has additional development dependencies, including cython and pytest. See requirements.txt for them all. -pycapnp has additional development dependencies, including cython and py.test. See requirements.txt for them all. ## Building and installation -Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-4.8 pip install pycapnp`. +Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-8.2 pip install pycapnp`. Or you can clone the repo like so: - git clone https://github.com/jparyani/pycapnp.git - pip install --install-option '--force-cython' ./pycapnp +```bash +git clone https://github.com/haata/pycapnp-async.git +cd pycapnp-async +pip install . +``` -Note: for OSX, if using clang from Xcode 5, you may need to set `CFLAGS` like so: +If you wish to install using the latest upstream C++ Cap'n Proto: + +```bash +pip install \ + --install-option "--libcapnp-url" \ + --install-option "https://github.com/sandstorm-io/capnproto/archive/master.tar.gz" \ + --install-option "--force-bundled-libcapnp" . +``` + +To force bundled python: + +```bash +pip install --install-option "--force-bundled-libcapnp" . +``` - CFLAGS='-stdlib=libc++' pip install pycapnp ## Python Versions -Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. +Python 3.7+ is supported. +Earlier versions of Python have asyncio bugs that might be possible to work around, but may require significant work (3.5 and 3.6). -One oddity to note is that `Text` type fields will be treated as byte strings under Python 2, and unicode strings under Python 3. `Data` fields will always be treated as byte strings. ## Development -This project uses [git-flow](http://jeffkreeftmeijer.com/2010/why-arent-you-using-git-flow/). Essentially, just make sure you do your changes in the `develop` branch. You can run the tests by installing pytest with `pip install pytest`, and then run `py.test` from the `test` directory. +Git flow has been abandoned, use master. + +To test, use a pipenv (or install requirements.txt and run pytest manually). +```bash +pip install pipenv +pipenv install +pipenv run pytest +``` + ### Binary Packages -In order to build binary packages from this source code, you must specify the `--disable-cython` option: - Building a dumb binary distribution: - python setup.py bdist_dumb --disable-cython +```bash +python setup.py bdist_dumb +``` Building a Python wheel distributiion: - python setup.py bdist_wheel --disable-cython +```bash +python setup.py bdist_wheel +``` -If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with: - python setup.py build --force-cython +### Pypi Upload Instructions + +Only necessary if uploading release to pypi.org. + +TODO + ## Documentation/Example + There is some basic documentation [here](http://jparyani.github.io/pycapnp/). The examples directory has one example that shows off pycapnp quite nicely. Here it is, reproduced: @@ -158,17 +201,3 @@ if __name__ == '__main__': server(write_end) client(read_end) ``` - -## Common Problems - -If you get an error on installation like: - - ... - gcc-4.8: error: capnp/capnp.c: No such file or directory - - gcc-4.8: fatal error: no input files - -Then you have too old a version of setuptools. Run `pip install -U setuptools` then try again. - - -[![Build Status](https://travis-ci.org/jparyani/pycapnp.png?branch=develop)](https://travis-ci.org/jparyani/pycapnp) diff --git a/benchmark/bin/run_all.py b/benchmark/bin/run_all.py index da6d9a3..60421a0 100755 --- a/benchmark/bin/run_all.py +++ b/benchmark/bin/run_all.py @@ -6,10 +6,13 @@ import sys import os import json import argparse +import time + +_this_dir = os.path.dirname(__file__) def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l capnproto -l protobuf", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp']) + parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l pyproto -l pyproto_cpp", action='append', default=['pycapnp']) parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true') parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true') parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0) @@ -26,26 +29,24 @@ def run_one(prefix, name, mode, iters, faster, compression): if compression != 'none': res_type += '_' + compression - command = ["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)] + command = [os.path.join(_this_dir, prefix+"-"+name), mode, reuse, compression, str(iters)] + start = time.time() + print('running: ' + ' '.join(command), file=sys.stderr) p = Popen(command, stdout=PIPE, stderr=PIPE) - res = p.communicate()[1] + res = p.wait() + end = time.time() data = {} if p.returncode != 0: - sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + res + '\n') + sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + p.stderr.read() + '\n') sys.stderr.flush() - else: - res = res.strip() - - for line in res.split('\n'): - vals = line.split() - data[vals[0]] = float(vals[1]) data['type'] = res_type data['mode'] = mode data['name'] = name data['iters'] = iters + data['time'] = end - start return data diff --git a/buildutils/__init__.py b/buildutils/__init__.py index 1da698d..65bb10d 100644 --- a/buildutils/__init__.py +++ b/buildutils/__init__.py @@ -2,6 +2,7 @@ Largely adapted from h5py """ +# flake8: noqa F401 F403 from .msg import * from .config import * diff --git a/buildutils/build.py b/buildutils/build.py index 82ffaab..7b54c7e 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -2,29 +2,79 @@ import subprocess import os -import tempfile +import shutil +import struct +import sys -def build_libcapnp(bundle_dir, build_dir, verbose=False): - bundle_dir = os.path.abspath(bundle_dir) - capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') - build_dir = os.path.abspath(build_dir) +def build_libcapnp(bundle_dir, build_dir): + ''' + Build capnproto + ''' + bundle_dir = os.path.abspath(bundle_dir) + capnp_dir = os.path.join(bundle_dir, 'capnproto-c++') + build_dir = os.path.abspath(build_dir) + tmp_dir = os.path.join(capnp_dir, 'build{}'.format(8 * struct.calcsize("P"))) + + # Clean the tmp build directory every time + if os.path.exists(tmp_dir): + shutil.rmtree(tmp_dir) + os.mkdir(tmp_dir) - with tempfile.TemporaryFile() as f: - stdout = f - if verbose: - stdout = None cxxflags = os.environ.get('CXXFLAGS', None) - os.environ['CXXFLAGS'] = (cxxflags or '') + ' -fPIC -O2 -DNDEBUG' - conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout) + os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG' + + # Enable ninja for compilation if available + build_type = [] + if shutil.which('ninja'): + build_type = ['-G', 'Ninja'] + + # Determine python shell architecture for Windows + python_arch = 8 * struct.calcsize("P") + build_arch = [] + build_flags = [] + if os.name == 'nt': + if python_arch == 64: + build_arch_flag = "x64" + elif python_arch == 32: + build_arch_flag = "Win32" + else: + raise RuntimeError('Unknown windows build arch') + build_arch = ['-A', build_arch_flag] + build_flags = ['--config', 'Release'] + print('Building module for {}'.format(python_arch)) + + if not shutil.which('cmake'): + raise RuntimeError('Could not find cmake in your path!') + + args = [ + 'cmake', + '-DCMAKE_POSITION_INDEPENDENT_CODE=1', + '-DBUILD_TESTING=OFF', + '-DBUILD_SHARED_LIBS=OFF', + '-DCMAKE_INSTALL_PREFIX:PATH={}'.format(build_dir), + capnp_dir, + ] + args.extend(build_type) + args.extend(build_arch) + conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = conf.wait() if returncode != 0: - raise RuntimeError('Configure failed') + raise RuntimeError('CMake failed {}'.format(returncode)) - make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout) - returncode = make.wait() + # Run build through cmake + args = [ + 'cmake', + '--build', + '.', + '--target', + 'install', + ] + args.extend(build_flags) + build = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) + returncode = build.wait() if cxxflags is None: - del os.environ['CXXFLAGS'] + del os.environ['CXXFLAGS'] else: - os.environ['CXXFLAGS'] = cxxflags + os.environ['CXXFLAGS'] = cxxflags if returncode != 0: - raise RuntimeError('Make failed') + raise RuntimeError('capnproto compilation failed: {}'.format(returncode)) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index f1292c4..acc39ed 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -1,12 +1,11 @@ """utilities for fetching build dependencies.""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # Distributed under the terms of the Modified BSD License. # # This bundling code is largely adapted from pyzmq-static's get.sh by # Brandon Craig-Rhodes, which is itself BSD licensed. -#----------------------------------------------------------------------------- # # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq # for original project. @@ -14,40 +13,31 @@ import os import shutil -import stat -import sys import tarfile -from glob import glob -from subprocess import Popen, PIPE -try: - # py2 - from urllib2 import urlopen -except ImportError: - # py3 - from urllib.request import urlopen - -from .msg import fatal, debug, info, warn +from urllib.request import urlopen +from .msg import info pjoin = os.path.join -#----------------------------------------------------------------------------- +# # Constants -#----------------------------------------------------------------------------- +# -bundled_version = (0,5,1,2) -libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version) -libcapnp_url = "https://capnproto.org/" + libcapnp +bundled_version = (0, 7, 0) +libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) +libcapnp_url = "https://capnproto.org/" + libcapnp_name HERE = os.path.dirname(__file__) ROOT = os.path.dirname(HERE) -#----------------------------------------------------------------------------- +# # Utilities -#----------------------------------------------------------------------------- +# def untgz(archive): + """Remove .tar.gz""" return archive.replace('.tar.gz', '') def localpath(*args): @@ -69,97 +59,29 @@ def fetch_archive(savedir, url, fname, force=False): f.write(req.read()) return dest -#----------------------------------------------------------------------------- +# # libcapnp -#----------------------------------------------------------------------------- +# -def fetch_libcapnp(savedir): +def fetch_libcapnp(savedir, url=None): """download and extract libcapnp""" + is_preconfigured = False + if url is None: + url = libcapnp_url + is_preconfigured = True dest = pjoin(savedir, 'capnproto-c++') if os.path.exists(dest): info("already have %s" % dest) return - fname = fetch_archive(savedir, libcapnp_url, libcapnp) + fname = fetch_archive(savedir, url, libcapnp_name) tf = tarfile.open(fname) with_version = pjoin(savedir, tf.firstmember.path) tf.extractall(savedir) tf.close() # remove version suffix: - shutil.move(with_version, dest) - -def stage_platform_hpp(capnproot): - """stage platform.hpp into libcapnp sources - - Tries ./configure first (except on Windows), - then falls back on included platform.hpp previously generated. - """ - - platform_hpp = pjoin(capnproot, 'src', 'platform.hpp') - if os.path.exists(platform_hpp): - info("already have platform.hpp") - return - if os.name == 'nt': - # stage msvc platform header - platform_dir = pjoin(capnproot, 'builds', 'msvc') + if is_preconfigured: + shutil.move(with_version, dest) else: - info("attempting ./configure to generate platform.hpp") - - p = Popen('./configure', cwd=capnproot, shell=True, - stdout=PIPE, stderr=PIPE, - ) - o,e = p.communicate() - if p.returncode: - warn("failed to configure libcapnp:\n%s" % e) - if sys.platform == 'darwin': - platform_dir = pjoin(HERE, 'include_darwin') - elif sys.platform.startswith('freebsd'): - platform_dir = pjoin(HERE, 'include_freebsd') - elif sys.platform.startswith('linux-armv'): - platform_dir = pjoin(HERE, 'include_linux-armv') - else: - platform_dir = pjoin(HERE, 'include_linux') - else: - return - - info("staging platform.hpp from: %s" % platform_dir) - shutil.copy(pjoin(platform_dir, 'platform.hpp'), platform_hpp) - - -def copy_and_patch_libcapnp(capnp, libcapnp): - """copy libcapnp into source dir, and patch it if necessary. - - This command is necessary prior to running a bdist on Linux or OS X. - """ - if sys.platform.startswith('win'): - return - # copy libcapnp into capnp for bdist - local = localpath('capnp',libcapnp) - if not capnp and not os.path.exists(local): - fatal("Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` " - "or copy libcapnp into capnp/ manually prior to running bdist.") - try: - # resolve real file through symlinks - lib = os.path.realpath(pjoin(capnp, 'lib', libcapnp)) - print ("copying %s -> %s"%(lib, local)) - shutil.copy(lib, local) - except Exception: - if not os.path.exists(local): - fatal("Could not copy libcapnp into capnp/, which is necessary for bdist. " - "Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` " - "or copy libcapnp into capnp/ manually.") - - if sys.platform == 'darwin': - # chmod u+w on the lib, - # which can be user-read-only for some reason - mode = os.stat(local).st_mode - os.chmod(local, mode | stat.S_IWUSR) - # patch install_name on darwin, instead of using rpath - cmd = ['install_name_tool', '-id', '@loader_path/../%s'%libcapnp, local] - try: - p = Popen(cmd, stdout=PIPE,stderr=PIPE) - except OSError: - fatal("install_name_tool not found, cannot patch libcapnp for bundling.") - out,err = p.communicate() - if p.returncode: - fatal("Could not patch bundled libcapnp install_name: %s"%err, p.returncode) + cpp_dir = os.path.join(with_version, 'c++') + shutil.move(cpp_dir, dest) diff --git a/buildutils/config.py b/buildutils/config.py index c674655..277b776 100644 --- a/buildutils/config.py +++ b/buildutils/config.py @@ -1,5 +1,5 @@ """Config functions""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # # This file is part of pyzmq, copied and adapted from h5py. @@ -9,149 +9,12 @@ # # Distributed under the terms of the New BSD License. The full license is in # the file COPYING.BSD, distributed as part of this software. -#----------------------------------------------------------------------------- +# -import sys -import os -import json - -try: - from configparser import ConfigParser -except: - from ConfigParser import ConfigParser - -pjoin = os.path.join -from .msg import debug, fatal, warn - -#----------------------------------------------------------------------------- +# # Utility functions (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- - - -def load_config(name, base='conf'): - """Load config dict from JSON""" - fname = pjoin(base, name + '.json') - if not os.path.exists(fname): - return {} - try: - with open(fname) as f: - cfg = json.load(f) - except Exception as e: - warn("Couldn't load %s: %s" % (fname, e)) - cfg = {} - return cfg - - -def save_config(name, data, base='conf'): - """Save config dict to JSON""" - if not os.path.exists(base): - os.mkdir(base) - fname = pjoin(base, name+'.json') - with open(fname, 'w') as f: - json.dump(data, f, indent=2) - +# def v_str(v_tuple): """turn (2,0,1) into '2.0.1'.""" return ".".join(str(x) for x in v_tuple) - -def get_eargs(): - """ Look for options in environment vars """ - - settings = {} - - zmq = os.environ.get("ZMQ_PREFIX", None) - if zmq is not None: - debug("Found environ var ZMQ_PREFIX=%s" % zmq) - settings['zmq_prefix'] = zmq - - return settings - -def cfg2dict(cfg): - """turn a ConfigParser into a nested dict - - because ConfigParser objects are dumb. - """ - d = {} - for section in cfg.sections(): - d[section] = dict(cfg.items(section)) - return d - -def get_cfg_args(): - """ Look for options in setup.cfg """ - - if not os.path.exists('setup.cfg'): - return {} - cfg = ConfigParser() - cfg.read('setup.cfg') - cfg = cfg2dict(cfg) - - g = cfg.setdefault('global', {}) - # boolean keys: - for key in ['libzmq_extension', - 'bundle_libzmq_dylib', - 'no_libzmq_extension', - 'have_sys_un_h', - 'skip_check_zmq', - ]: - if key in g: - g[key] = eval(g[key]) - - # globals go to top level - cfg.update(cfg.pop('global')) - return cfg - -def config_from_prefix(prefix): - """Get config from zmq prefix""" - settings = {} - if prefix.lower() in ('default', 'auto', ''): - settings['zmq_prefix'] = '' - settings['libzmq_extension'] = False - settings['no_libzmq_extension'] = False - elif prefix.lower() in ('bundled', 'extension'): - settings['zmq_prefix'] = '' - settings['libzmq_extension'] = True - settings['no_libzmq_extension'] = False - else: - settings['zmq_prefix'] = prefix - settings['libzmq_extension'] = False - settings['no_libzmq_extension'] = True - return settings - -def merge(into, d): - """merge two containers - - into is updated, d has priority - """ - if isinstance(into, dict): - for key in d.keys(): - if key not in into: - into[key] = d[key] - else: - into[key] = merge(into[key], d[key]) - return into - elif isinstance(into, list): - return into + d - else: - return d - -def discover_settings(conf_base=None): - """ Discover custom settings for ZMQ path""" - settings = { - 'zmq_prefix': '', - 'libzmq_extension': False, - 'no_libzmq_extension': False, - 'skip_check_zmq': False, - 'build_ext': {}, - 'bdist_egg': {}, - } - if sys.platform.startswith('win'): - settings['have_sys_un_h'] = False - - if conf_base: - # lowest priority - merge(settings, load_config('config', conf_base)) - merge(settings, get_cfg_args()) - merge(settings, get_eargs()) - - return settings diff --git a/buildutils/constants.py b/buildutils/constants.py deleted file mode 100644 index e98c650..0000000 --- a/buildutils/constants.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -script for generating files that involve repetitive updates for zmq constants. - -Run this after updating utils/constant_names - -Currently generates the following files from templates: - -- constant_enums.pxi -- constants.pxi -- zmq_constants.h - -""" - -# Copyright (C) PyZMQ Developers -# Distributed under the terms of the Modified BSD License. - -import os -import sys - -from . import info -pjoin = os.path.join - -root = os.path.abspath(pjoin(os.path.dirname(__file__), os.path.pardir)) - -sys.path.insert(0, pjoin(root, 'zmq', 'utils')) -from constant_names import all_names, no_prefix - -ifndef_t = """#ifndef {0} - #define {0} (_PYZMQ_UNDEFINED) -#endif -""" - -def cython_enums(): - """generate `enum: ZMQ_CONST` block for constant_enums.pxi""" - lines = [] - for name in all_names: - if no_prefix(name): - lines.append('enum: ZMQ_{0} "{0}"'.format(name)) - else: - lines.append('enum: ZMQ_{0}'.format(name)) - - return dict(ZMQ_ENUMS='\n '.join(lines)) - -def ifndefs(): - """generate `#ifndef ZMQ_CONST` block for zmq_constants.h""" - lines = ['#define _PYZMQ_UNDEFINED (-9999)'] - for name in all_names: - if not no_prefix(name): - name = 'ZMQ_%s' % name - lines.append(ifndef_t.format(name)) - return dict(ZMQ_IFNDEFS='\n'.join(lines)) - -def constants_pyx(): - """generate CONST = ZMQ_CONST and __all__ for constants.pxi""" - all_lines = [] - assign_lines = [] - for name in all_names: - if name == "NULL": - # avoid conflict with NULL in Cython - assign_lines.append("globals()['NULL'] = ZMQ_NULL") - else: - assign_lines.append('{0} = ZMQ_{0}'.format(name)) - all_lines.append(' "{0}",'.format(name)) - return dict(ASSIGNMENTS='\n'.join(assign_lines), ALL='\n'.join(all_lines)) - -def generate_file(fname, ns_func, dest_dir="."): - """generate a constants file from its template""" - with open(pjoin(root, 'buildutils', 'templates', '%s' % fname), 'r') as f: - tpl = f.read() - out = tpl.format(**ns_func()) - dest = pjoin(dest_dir, fname) - info("generating %s from template" % dest) - with open(dest, 'w') as f: - f.write(out) - -def render_constants(): - """render generated constant files from templates""" - generate_file("constant_enums.pxi", cython_enums, pjoin(root, 'zmq', 'backend', 'cython')) - generate_file("constants.pxi", constants_pyx, pjoin(root, 'zmq', 'backend', 'cython')) - generate_file("zmq_constants.h", ifndefs, pjoin(root, 'zmq', 'utils')) - -if __name__ == '__main__': - render_constants() diff --git a/buildutils/detect.py b/buildutils/detect.py index 8fd628d..7d71345 100644 --- a/buildutils/detect.py +++ b/buildutils/detect.py @@ -1,5 +1,5 @@ """Detect zmq version""" -#----------------------------------------------------------------------------- +# # Copyright (C) PyZMQ Developers # # This file is part of pyzmq, copied and adapted from h5py. @@ -9,7 +9,7 @@ # # Distributed under the terms of the New BSD License. The full license is in # the file COPYING.BSD, distributed as part of this software. -#----------------------------------------------------------------------------- +# # # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq # for original project. @@ -21,7 +21,6 @@ import logging import platform from distutils import ccompiler from distutils.ccompiler import get_default_compiler -from subprocess import Popen, PIPE import tempfile from .misc import get_compiler, get_output_error @@ -29,20 +28,20 @@ from .patch import patch_lib_paths pjoin = os.path.join -#----------------------------------------------------------------------------- +# # Utility functions (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- +# def test_compilation(cfile, compiler=None, **compiler_attrs): """Test simple compilation with given settings""" cc = get_compiler(compiler, **compiler_attrs) - efile, ext = os.path.splitext(cfile) + efile, _ = os.path.splitext(cfile) cpreargs = lpreargs = [] if sys.platform == 'darwin': # use appropriate arch for compiler - if platform.architecture()[0]=='32bit': + if platform.architecture()[0] == '32bit': if platform.processor() == 'powerpc': cpu = 'ppc' else: @@ -53,36 +52,21 @@ def test_compilation(cfile, compiler=None, **compiler_attrs): # allow for missing UB arch, since it will still work: lpreargs = ['-undefined', 'dynamic_lookup'] if sys.platform == 'sunos5': - if platform.architecture()[0]=='32bit': + if platform.architecture()[0] == '32bit': lpreargs = ['-m32'] else: lpreargs = ['-m64'] - extra = compiler_attrs.get('extra_compile_args', []) - extra += ['--std=c++11'] + extra_compile_args = compiler_attrs.get('extra_compile_args', []) + if os.name != 'nt': + extra_compile_args += ['--std=c++14'] + extra_link_args = compiler_attrs.get('extra_link_args', []) + if cc.compiler_type == 'msvc': + extra_link_args += ['/MANIFEST'] - objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra) - cc.link_executable(objs, efile, extra_preargs=lpreargs) + objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra_compile_args) + cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args) return efile -def compile_and_run(basedir, src, compiler=None, **compiler_attrs): - if not os.path.exists(basedir): - os.makedirs(basedir) - cfile = pjoin(basedir, os.path.basename(src)) - shutil.copy(src, cfile) - try: - cc = get_compiler(compiler, **compiler_attrs) - efile = test_compilation(cfile, compiler=cc) - patch_lib_paths(efile, cc.library_dirs) - result = Popen(efile, stdout=PIPE, stderr=PIPE) - so, se = result.communicate() - # for py3k: - so = so.decode() - se = se.decode() - finally: - shutil.rmtree(basedir) - - return result.returncode, so, se - def detect_version(basedir, compiler=None, **compiler_attrs): """Compile, link & execute a test program, in empty directory `basedir`. @@ -130,11 +114,11 @@ def detect_version(basedir, compiler=None, **compiler_attrs): rc, so, se = get_output_error([efile]) if rc: - msg = "Error running version detection script:\n%s\n%s" % (so,se) + msg = "Error running version detection script:\n%s\n%s" % (so, se) logging.error(msg) raise IOError(msg) - handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))} + handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))} props = {} for line in (x for x in so.split('\n') if x): @@ -144,7 +128,7 @@ def detect_version(basedir, compiler=None, **compiler_attrs): return props -def test_build(): +def test_build(**compiler_attrs): """do a test build of libcapnp""" tmp_dir = tempfile.mkdtemp() @@ -152,7 +136,7 @@ def test_build(): # info("Configure: Autodetecting Cap'n Proto settings...") # info(" Custom Cap'n Proto dir: %s" % prefix) try: - detected = detect_version(tmp_dir) + detected = detect_version(tmp_dir, None, **compiler_attrs) finally: erase_dir(tmp_dir) @@ -161,8 +145,9 @@ def test_build(): return detected -def erase_dir(dir): +def erase_dir(path): + """Erase directory""" try: - shutil.rmtree(dir) + shutil.rmtree(path) except Exception: pass diff --git a/buildutils/misc.py b/buildutils/misc.py index 72d01b7..6f0c370 100644 --- a/buildutils/misc.py +++ b/buildutils/misc.py @@ -4,7 +4,6 @@ # Distributed under the terms of the Modified BSD License. import os -import sys import logging from distutils import ccompiler from distutils.sysconfig import customize_compiler @@ -14,13 +13,8 @@ from subprocess import Popen, PIPE pjoin = os.path.join -if sys.version_info[0] >= 3: - u = lambda x: x -else: - u = lambda x: x.decode('utf8', 'replace') - - def customize_mingw(cc): + """customize mingw""" # strip -mno-cygwin from mingw32 (Python Issue #12641) for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]: if '-mno-cygwin' in cmd: @@ -30,14 +24,18 @@ def customize_mingw(cc): if 'msvcr90' in cc.dll_libraries: cc.dll_libraries.remove('msvcr90') +def customize_msvc(cc): + pass def get_compiler(compiler, **compiler_attrs): """get and customize a compiler""" if compiler is None or isinstance(compiler, str): cc = ccompiler.new_compiler(compiler=compiler) - # customize_compiler(cc) + customize_compiler(cc) if cc.compiler_type == 'mingw32': customize_mingw(cc) + elif cc.compiler_type == 'msvc': + customize_msvc(cc) else: cc = compiler @@ -55,11 +53,10 @@ def get_output_error(cmd): try: result = Popen(cmd, stdout=PIPE, stderr=PIPE) except IOError as e: - return -1, u(''), u('Failed to run %r: %r' % (cmd, e)) + return -1, '', 'Failed to run %r: %r' % (cmd, e) so, se = result.communicate() # unicode: so = so.decode('utf8', 'replace') se = se.decode('utf8', 'replace') return result.returncode, so, se - diff --git a/buildutils/msg.py b/buildutils/msg.py index 70cd716..63f2e51 100644 --- a/buildutils/msg.py +++ b/buildutils/msg.py @@ -9,9 +9,9 @@ import os import sys import logging -#----------------------------------------------------------------------------- +# # Logging (adapted from h5py: http://h5py.googlecode.com) -#----------------------------------------------------------------------------- +# logger = logging.getLogger() @@ -22,18 +22,22 @@ else: logger.addHandler(logging.StreamHandler(sys.stderr)) def debug(msg): + """Debug""" logger.debug(msg) def info(msg): + """Info""" logger.info(msg) def fatal(msg, code=1): - logger.error("Fatal: " + msg) + """Fatal""" + logger.error("Fatal: %s", msg) exit(code) def warn(msg): - logger.error("Warning: " + msg) + """Warning""" + logger.error("Warning: %s", msg) def line(c='*', width=48): + """Horizontal rule""" print(c * (width // len(c))) - diff --git a/buildutils/patch.py b/buildutils/patch.py index 925b67b..dc58ac2 100644 --- a/buildutils/patch.py +++ b/buildutils/patch.py @@ -20,7 +20,7 @@ LIB_PAT = re.compile(r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), " def _get_libs(fname): rc, so, se = get_output_error(['otool', '-L', fname]) if rc: - logging.error("otool -L %s failed: %r" % (fname, se)) + logging.error("otool -L %s failed: %r", fname, se) return for line in so.splitlines()[1:]: m = LIB_PAT.match(line) @@ -33,6 +33,7 @@ def _find_library(lib, path): real_lib = os.path.join(d, lib) if os.path.exists(real_lib): return real_lib + return None def _install_name_change(fname, lib, real_lib): rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname]) @@ -41,15 +42,15 @@ def _install_name_change(fname, lib, real_lib): def patch_lib_paths(fname, library_dirs): """Load any weakly-defined libraries from their real location - + (only on OS X) - + - Find libraries with `otool -L` - Update with `install_name_tool -change` """ if sys.platform != 'darwin': return - + libs = _get_libs(fname) for lib in libs: if not lib.startswith(('@', '/')): @@ -58,4 +59,4 @@ def patch_lib_paths(fname, library_dirs): _install_name_change(fname, lib, real_lib) -__all__ = ['patch_lib_paths'] \ No newline at end of file +__all__ = ['patch_lib_paths'] diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh deleted file mode 100755 index 956dcf5..0000000 --- a/buildutils/setup_travis.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -exo pipefail - -CAPNP_VERSION=0.5.1.2 - -sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -sudo apt-get -qq update -sudo apt-get -qq install g++-4.8 libstdc++-4.8-dev -sudo update-alternatives --quiet --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-4.8 --slave /usr/bin/gcov gcov /usr/bin/gcov-4.8 -sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 - -if ! [ -z "${BUILD_CAPNP}" ]; then - wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz && tar xzvf capnproto-c++-${CAPNP_VERSION}.tar.gz && cd capnproto-c++-${CAPNP_VERSION} && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. -fi diff --git a/capnp/__init__.py b/capnp/__init__.py index 2a04e92..7d36a9a 100644 --- a/capnp/__init__.py +++ b/capnp/__init__.py @@ -31,8 +31,26 @@ Example Usage:: for phone in person.phones: print(phone.type, ':', phone.number) """ +# flake8: noqa F401 F403 F405 from .version import version as __version__ from .lib.capnp import * -from .lib.capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicResizableListBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan, _DynamicResizableListBuilder, _MallocMessageBuilder, _PackedFdMessageReader, _StreamFdMessageReader, _write_message_to_fd, _write_packed_message_to_fd, _StructModule, _InterfaceModule, _DynamicCapabilityClient, _CapabilityClient, _EventLoop +from .lib.capnp import ( + _CapabilityClient, + _DynamicCapabilityClient, + _DynamicListBuilder, + _DynamicListReader, + _DynamicOrphan, + _DynamicResizableListBuilder, + _DynamicStructBuilder, + _DynamicStructReader, + _EventLoop, + _InterfaceModule, + _MallocMessageBuilder, + _PackedFdMessageReader, + _StreamFdMessageReader, + _StructModule, + _write_message_to_fd, + _write_packed_message_to_fd, +) add_import_hook() # enable import hook by default diff --git a/capnp/_gen.py b/capnp/_gen.py index 5cb3eee..52087c1 100644 --- a/capnp/_gen.py +++ b/capnp/_gen.py @@ -7,58 +7,58 @@ from jinja2 import Environment, PackageLoader import os def find_type(code, id): - for node in code['nodes']: - if node['id'] == id: - return node + for node in code['nodes']: + if node['id'] == id: + return node - return None + return None def main(): - env = Environment(loader=PackageLoader('capnp', 'templates')) - env.filters['format_name'] = lambda name: name[name.find(':')+1:] + env = Environment(loader=PackageLoader('capnp', 'templates')) + env.filters['format_name'] = lambda name: name[name.find(':') + 1:] - code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) - code=code.to_dict() - code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] - for node in code['nodes']: - displayName = node['displayName'] - parent, path = displayName.split(':') - node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) - node['module_name'] = path.replace('.', '_') - node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) - node['schema'] = '_{}_Schema'.format(node['module_name']) - is_union = False - for field in node['struct']['fields']: - if field['discriminantValue'] != 65535: - is_union = True - field['c_name'] = field['name'][0].upper() + field['name'][1:] - if 'slot' in field: - field['type'] = field['slot']['type'].keys()[0] - if not isinstance(field['slot']['type'][field['type']], dict): - continue - sub_type = field['slot']['type'][field['type']].get('typeId', None) - if sub_type: - field['sub_type'] = find_type(code, sub_type) - sub_type = field['slot']['type'][field['type']].get('elementType', None) - if sub_type: - field['sub_type'] = sub_type - else: - field['type'] = find_type(code, field['group']['typeId']) - node['is_union'] = is_union + code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) + code = code.to_dict() + code['nodes'] = [node for node in code['nodes'] if 'struct' in node and node['scopeId'] != 0] + for node in code['nodes']: + displayName = node['displayName'] + parent, path = displayName.split(':') + node['module_path'] = parent.replace('.', '_') + '.' + '.'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['module_name'] = path.replace('.', '_') + node['c_module_path'] = '::'.join([x[0].upper() + x[1:] for x in path.split('.')]) + node['schema'] = '_{}_Schema'.format(node['module_name']) + is_union = False + for field in node['struct']['fields']: + if field['discriminantValue'] != 65535: + is_union = True + field['c_name'] = field['name'][0].upper() + field['name'][1:] + if 'slot' in field: + field['type'] = field['slot']['type'].keys()[0] + if not isinstance(field['slot']['type'][field['type']], dict): + continue + sub_type = field['slot']['type'][field['type']].get('typeId', None) + if sub_type: + field['sub_type'] = find_type(code, sub_type) + sub_type = field['slot']['type'][field['type']].get('elementType', None) + if sub_type: + field['sub_type'] = sub_type + else: + field['type'] = find_type(code, field['group']['typeId']) + node['is_union'] = is_union - include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) - module = env.get_template('module.pyx') + include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), '..')) + module = env.get_template('module.pyx') - for f in code['requestedFiles']: - filename = f['filename'].replace('.', '_') + '_cython.pyx' + for f in code['requestedFiles']: + filename = f['filename'].replace('.', '_') + '_cython.pyx' - file_code = dict(code) - file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] - with open(filename, 'w') as out: - out.write(module.render(code=file_code, file=f, include_dir=include_dir)) + file_code = dict(code) + file_code['nodes'] = [node for node in file_code['nodes'] if node['displayName'].startswith(f['filename'])] + with open(filename, 'w') as out: + out.write(module.render(code=file_code, file=f, include_dir=include_dir)) - setup = env.get_template('setup.py.tmpl') - with open('setup_capnp.py', 'w') as out: - out.write(setup.render(code=code)) - print('You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.') - print() + setup = env.get_template('setup.py.tmpl') + with open('setup_capnp.py', 'w') as out: + out.write(setup.render(code=code)) + print('You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.') + print() diff --git a/capnp/helpers/asyncHelper.h b/capnp/helpers/asyncHelper.h index 7d1b7ba..4d74e43 100644 --- a/capnp/helpers/asyncHelper.h +++ b/capnp/helpers/asyncHelper.h @@ -13,11 +13,13 @@ public: virtual bool wait() { GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("wait"), NULL); + return true; // TODO: get the bool result from python } virtual bool poll() { GILAcquire gil; PyObject_CallMethod(py_event_port, const_cast("poll"), NULL); + return true; // TODO: get the bool result from python } virtual void setRunnable(bool runnable) { @@ -37,6 +39,11 @@ void waitNeverDone(kj::WaitScope & scope) { kj::NEVER_DONE.wait(scope); } +void pollWaitScope(kj::WaitScope & scope) { + GILRelease gil; + scope.poll(); +} + kj::Timer * getTimer(kj::AsyncIoContext * context) { return &context->lowLevelProvider->getTimer(); } @@ -55,3 +62,8 @@ capnp::Response< ::capnp::DynamicStruct> * waitRemote(capnp::RemotePromise< ::ca GILRelease gil; return new capnp::Response< ::capnp::DynamicStruct>(promise->wait(scope)); } + +bool pollRemote(capnp::RemotePromise< ::capnp::DynamicStruct> * promise, kj::WaitScope & scope) { + GILRelease gil; + return promise->poll(scope); +} diff --git a/capnp/helpers/asyncIoHelper.h b/capnp/helpers/asyncIoHelper.h new file mode 100644 index 0000000..c848ebb --- /dev/null +++ b/capnp/helpers/asyncIoHelper.h @@ -0,0 +1,53 @@ +#pragma once + +#include "kj/async.h" +#include "kj/async-io.h" + +class AsyncIoStreamReadHelper { +public: + AsyncIoStreamReadHelper(kj::AsyncIoStream * _stream, kj::WaitScope * _scope, size_t bufsize) { + io_stream = _stream; + wait_scope = _scope; + ready = false; + buffer_read_size = 0; + buffer = new unsigned char[bufsize]; + promise = io_stream->read(buffer, 1, bufsize); + } + + ~AsyncIoStreamReadHelper() { + delete[] buffer; + } + + bool poll() { + bool result = promise.poll(*wait_scope); + if (result) { + ready = true; + buffer_read_size = promise.wait(*wait_scope); + } + return result; + } + + size_t read_size() { + if (!ready) { + return 0; + } + return buffer_read_size; + } + + void * read_buffer() { + if (!ready) { + return 0; + } + return buffer; + } + +private: + kj::AsyncIoStream * io_stream; + kj::WaitScope * wait_scope; + kj::Promise promise = nullptr; + + unsigned char *buffer; + size_t buffer_read_size; + + bool ready; +}; diff --git a/capnp/helpers/capabilityHelper.h b/capnp/helpers/capabilityHelper.h index 72ee96a..d6f7ff0 100644 --- a/capnp/helpers/capabilityHelper.h +++ b/capnp/helpers/capabilityHelper.h @@ -66,6 +66,7 @@ void reraise_kj_exception() { catch (kj::Exception& exn) { auto obj = wrap_kj_exception_for_reraise(exn); PyErr_SetObject((PyObject*)obj->ob_type, obj); + Py_DECREF(obj); } catch (const std::exception& exn) { PyErr_SetString(PyExc_RuntimeError, exn.what()); diff --git a/capnp/helpers/checkCompiler.h b/capnp/helpers/checkCompiler.h index ed8d427..c5e34c1 100644 --- a/capnp/helpers/checkCompiler.h +++ b/capnp/helpers/checkCompiler.h @@ -1,11 +1,8 @@ -#ifdef __GNUC__ - #if __clang__ - #if __cplusplus >= 201103L && !__has_include() - #warning "Your compiler supports C++11 but your C++ standard library does not. If your system has libc++ installed (as should be the case on e.g. Mac OSX), try adding -stdlib=libc++ to your CFLAGS (ignore the other warning that says to use CXXFLAGS)." - #endif - #endif +#ifdef _MSC_VER +#pragma comment(lib, "Ws2_32.lib") +#pragma comment(lib, "advapi32.lib") #endif #include "capnp/dynamic.h" -static_assert(CAPNP_VERSION >= 5000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.5 and then re-install this python library"); +static_assert(CAPNP_VERSION >= 7000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.7 and then re-install this python library"); \ No newline at end of file diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index 15b7197..f34b530 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -1,11 +1,13 @@ -from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer +from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer from capnp.includes.schema_cpp cimport ByteArray -from non_circular cimport reraise_kj_exception +from non_circular cimport reraise_kj_exception, AsyncIoStreamReadHelper from cpython.ref cimport PyObject +from libcpp cimport bool + cdef extern from "capnp/helpers/fixMaybe.h": EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception @@ -25,14 +27,8 @@ cdef extern from "capnp/helpers/capabilityHelper.h": VoidPromise convert_to_voidpromise(PyPromise&) cdef extern from "capnp/helpers/rpcHelper.h": - Capability.Client restoreHelper(RpcSystem&) - Capability.Client restoreHelper(RpcSystem&, MessageBuilder&) - Capability.Client restoreHelper(RpcSystem&, MessageReader&) - Capability.Client restoreHelper(RpcSystem&, AnyPointer.Reader&) - Capability.Client restoreHelper(RpcSystem&, AnyPointer.Builder&) Capability.Client bootstrapHelper(RpcSystem&) - RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&) - PyPromise connectServerRestorer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr) + Capability.Client bootstrapHelperServer(RpcSystem&) PyPromise connectServer(TaskSet &, Capability.Client, AsyncIoContext *, StringPtr) cdef extern from "capnp/helpers/serialize.h": @@ -40,7 +36,9 @@ cdef extern from "capnp/helpers/serialize.h": cdef extern from "capnp/helpers/asyncHelper.h": void waitNeverDone(WaitScope&) + void pollWaitScope(WaitScope&) Response * waitRemote(RemotePromise *, WaitScope&) + bool pollRemote(RemotePromise *, WaitScope&) PyObject * waitPyPromise(PyPromise *, WaitScope&) void waitVoidPromise(VoidPromise *, WaitScope&) Timer * getTimer(AsyncIoContext *) except +reraise_kj_exception diff --git a/capnp/helpers/non_circular.pxd b/capnp/helpers/non_circular.pxd index 58771dc..c220673 100644 --- a/capnp/helpers/non_circular.pxd +++ b/capnp/helpers/non_circular.pxd @@ -1,4 +1,6 @@ from cpython.ref cimport PyObject +from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope +from libcpp cimport bool cdef extern from "capnp/helpers/capabilityHelper.h": cppclass PythonInterfaceDynamicImpl: @@ -10,11 +12,16 @@ cdef extern from "capnp/helpers/capabilityHelper.h": PyRefCounter(PyObject *) cdef extern from "capnp/helpers/rpcHelper.h": - cdef cppclass PyRestorer: - PyRestorer(PyObject *) cdef cppclass ErrorHandler: pass cdef extern from "capnp/helpers/asyncHelper.h": cdef cppclass PyEventPort: PyEventPort(PyObject *) + +cdef extern from "capnp/helpers/asyncIoHelper.h": + cdef cppclass AsyncIoStreamReadHelper: + AsyncIoStreamReadHelper(AsyncIoStream *, WaitScope *, size_t) + bool poll() + size_t read_size() + void* read_buffer() diff --git a/capnp/helpers/rpcHelper.h b/capnp/helpers/rpcHelper.h index cd25a92..44f098f 100644 --- a/capnp/helpers/rpcHelper.h +++ b/capnp/helpers/rpcHelper.h @@ -6,73 +6,6 @@ #include "Python.h" #include "capabilityHelper.h" -extern "C" { - capnp::Capability::Client * call_py_restorer(PyObject *, capnp::AnyPointer::Reader &); -} - -class PyRestorer final: public capnp::SturdyRefRestorer { -public: - PyRestorer(PyObject * _py_restorer): py_restorer(_py_restorer) { - // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere. - // Py_INCREF(py_restorer); - } - - // ~PyRestorer() { - // Py_DECREF(py_restorer); - // } - - capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override { - GILAcquire gil; - capnp::Capability::Client * ret = call_py_restorer(py_restorer, objectId); - check_py_error(); - capnp::Capability::Client stack_ret(*ret); - delete ret; - - return stack_ret; - } - -private: - PyObject * py_restorer; -}; - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::MessageBuilder & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId.getRoot()); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::MessageReader & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId.getRoot()); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::AnyPointer::Reader & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client, capnp::AnyPointer::Builder & objectId) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return client.restore(hostId, objectId); -} - -capnp::Capability::Client restoreHelper(capnp::RpcSystem& client) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - - capnp::MallocMessageBuilder blankMessage(8); - auto objectId = blankMessage.getRoot(); - return client.restore(hostId, objectId); -} - capnp::Capability::Client bootstrapHelper(capnp::RpcSystem& client) { capnp::MallocMessageBuilder hostIdMessage(8); auto hostId = hostIdMessage.initRoot(); @@ -80,63 +13,19 @@ capnp::Capability::Client bootstrapHelper(capnp::RpcSystem -capnp::RpcSystem makeRpcClientWithRestorer( - capnp::VatNetwork& network, - PyRestorer& restorer) { - using namespace capnp; - return RpcSystem(network, restorer); +capnp::Capability::Client bootstrapHelperServer(capnp::RpcSystem& client) { + capnp::MallocMessageBuilder hostIdMessage(8); + auto hostId = hostIdMessage.initRoot(); + hostId.setSide(capnp::rpc::twoparty::Side::CLIENT); + return client.bootstrap(hostId); } -struct ServerContextRestorer { - kj::Own stream; - capnp::TwoPartyVatNetwork network; - capnp::RpcSystem rpcSystem; - - ServerContextRestorer(kj::Own&& stream, capnp::SturdyRefRestorer& restorer) - : stream(kj::mv(stream)), - network(*this->stream, capnp::rpc::twoparty::Side::SERVER), - rpcSystem(makeRpcServer(network, restorer)) {} -}; - class ErrorHandler : public kj::TaskSet::ErrorHandler { void taskFailed(kj::Exception&& exception) override { kj::throwFatalException(kj::mv(exception)); } }; -void acceptLoopRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::Own&& listener) { - auto ptr = listener.get(); - tasks.add(ptr->accept().then(kj::mvCapture(kj::mv(listener), - [&](kj::Own&& listener, - kj::Own&& connection) { - acceptLoopRestorer(tasks, restorer, kj::mv(listener)); - - auto server = kj::heap(kj::mv(connection), restorer); - - // Arrange to destroy the server context when all references are gone, or when the - // EzRpcServer is destroyed (which will destroy the TaskSet). - tasks.add(server->network.onDisconnect().attach(kj::mv(server))); - }))); -} - -kj::Promise connectServerRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::AsyncIoContext * context, kj::StringPtr bindAddress) { - auto paf = kj::newPromiseAndFulfiller(); - auto portPromise = paf.promise.fork(); - - tasks.add(context->provider->getNetwork().parseAddress(bindAddress) - .then(kj::mvCapture(paf.fulfiller, - [&](kj::Own>&& portFulfiller, - kj::Own&& addr) { - auto listener = addr->listen(); - portFulfiller->fulfill(listener->getPort()); - acceptLoopRestorer(tasks, restorer, kj::mv(listener)); - }))); - - return portPromise.addBranch().then([&](unsigned int port) { return PyLong_FromUnsignedLong(port); }); -} - struct ServerContext { kj::Own stream; diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index c246d0b..4ba2fc4 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -1,11 +1,11 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 cdef extern from "capnp/helpers/checkCompiler.h": pass -from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader -from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyRestorer, PyEventPort, ErrorHandler +from libcpp cimport bool +from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions +from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyEventPort, ErrorHandler from capnp.includes.types cimport * cdef extern from "capnp/common.h" namespace " ::capnp": @@ -45,7 +45,8 @@ cdef extern from "kj/exception.h" namespace " ::kj": cdef extern from "kj/memory.h" namespace " ::kj": cdef cppclass Own[T]: T& operator*() - Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side) + T* get() + Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side, ReaderOptions) Own[PromiseFulfillerPair] copyPromiseFulfillerPair" ::kj::heap< ::kj::PromiseFulfillerPair >"(PromiseFulfillerPair&) Own[PyRefCounter] makePyRefCounter" ::kj::heap< PyRefCounter >"(PyObject *) @@ -55,6 +56,7 @@ cdef extern from "kj/async.h" namespace " ::kj": Promise(Promise) Promise(T) T wait(WaitScope) + bool poll(WaitScope) # ForkedPromise fork() # Promise exclusiveJoin(Promise&& other) # Promise[T] eagerlyEvaluate() @@ -101,7 +103,14 @@ ctypedef Promise[PyArray] PyPromiseArray cdef extern from "kj/time.h" namespace " ::kj": cdef cppclass Duration: - Duration(int64_t) + Duration operator*(int64_t) + Duration NANOSECONDS + Duration MICROSECONDS + Duration MILLISECONDS + Duration SECONDS + Duration MINUTES + Duration HOURS + Duration DAYS # cdef cppclass TimePoint: # TimePoint(Duration) cdef cppclass Timer: @@ -109,18 +118,26 @@ cdef extern from "kj/time.h" namespace " ::kj": # VoidPromise atTime(TimePoint time) VoidPromise afterDelay(Duration delay) +cdef inline Duration Nanoseconds(int64_t nanos): + return NANOSECONDS * nanos + cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass AsyncIoStream: - pass + Promise[size_t] read(void*, size_t, size_t) + Promise[void] write(const void*, size_t) + cdef cppclass LowLevelAsyncIoProvider: # Own[AsyncInputStream] wrapInputFd(int) # Own[AsyncOutputStream] wrapOutputFd(int) Own[AsyncIoStream] wrapSocketFd(int) Timer& getTimer() except +reraise_kj_exception + cdef cppclass AsyncIoProvider: - pass + TwoWayPipe newTwoWayPipe() + cdef cppclass WaitScope: pass + cdef cppclass AsyncIoContext: AsyncIoContext(AsyncIoContext&) Own[LowLevelAsyncIoProvider] lowLevelProvider @@ -130,6 +147,9 @@ cdef extern from "kj/async-io.h" namespace " ::kj": cdef cppclass TaskSet: TaskSet(ErrorHandler &) + cdef cppclass TwoWayPipe: + Own[AsyncIoStream] ends[2] + AsyncIoContext setupAsyncIo() cdef extern from "capnp/schema.capnp.h" namespace " ::capnp": @@ -341,10 +361,9 @@ cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp": cdef Side SERVER" ::capnp::rpc::twoparty::Side::SERVER" cdef cppclass TwoPartyVatNetwork: - TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side) + TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side, ReaderOptions) VoidPromise onDisconnect() VoidPromise onDrained() - RpcSystem makeRpcServer(TwoPartyVatNetwork&, PyRestorer&) RpcSystem makeRpcServerBootstrap"makeRpcServer"(TwoPartyVatNetwork&, Capability.Client) RpcSystem makeRpcClient(TwoPartyVatNetwork&) @@ -427,6 +446,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": Reader(DynamicStruct.Reader& value) Reader(DynamicCapability.Client& value) Reader(PythonInterfaceDynamicImpl& value) + Reader(AnyPointer.Reader& value) Type getType() int64_t asInt"as"() uint64_t asUint"as"() diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index e615017..2c6ca2e 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -1,6 +1,5 @@ # schema.capnp.cpp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 from libc.stdint cimport * from capnp_cpp cimport DynamicOrphan @@ -628,6 +627,11 @@ cdef extern from "capnp/schema.capnp.h" namespace " ::capnp::schema": void setId(UInt64) Value getValue() void setValue(Value) + cdef cppclass ListNestedNodeReader"capnp::List::Reader": + ListNestedNodeReader() + ListNestedNodeReader(ListNestedNodeReader) + Node.NestedNode.Reader operator[](uint) + uint size() cdef extern from "capnp/message.h" namespace " ::capnp": cdef cppclass ReaderOptions: @@ -662,6 +666,8 @@ cdef extern from "capnp/message.h" namespace " ::capnp": DynamicStruct_Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema) void setRootDynamicStruct'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader) + ConstWordArrayArrayPtr getSegmentsForOutput'getSegmentsForOutput'() + AnyPointer.Builder getRootAnyPointer'getRoot< ::capnp::AnyPointer>'() DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema) @@ -686,6 +692,10 @@ cdef extern from "capnp/message.h" namespace " ::capnp": MallocMessageBuilder() MallocMessageBuilder(int) + cdef cppclass SegmentArrayMessageReader(MessageReader): + SegmentArrayMessageReader(ConstWordArrayArrayPtr array) except +reraise_kj_exception + SegmentArrayMessageReader(ConstWordArrayArrayPtr array, ReaderOptions) except +reraise_kj_exception + cdef cppclass FlatMessageBuilder(MessageBuilder): FlatMessageBuilder(WordArrayPtr array) FlatMessageBuilder(WordArrayPtr array, ReaderOptions) @@ -709,6 +719,16 @@ cdef extern from "kj/common.h" namespace " ::kj": ByteArrayPtr(byte *, size_t size) size_t size() byte& operator[](size_t index) + cdef cppclass ConstWordArrayPtr " ::kj::ArrayPtr< const ::capnp::word>": + ConstWordArrayPtr() + ConstWordArrayPtr(word *, size_t size) + size_t size() + const word* begin() + cdef cppclass ConstWordArrayArrayPtr " ::kj::ArrayPtr< const ::kj::ArrayPtr< const ::capnp::word>>": + ConstWordArrayArrayPtr() + ConstWordArrayArrayPtr(ConstWordArrayPtr*, size_t size) + size_t size() + ConstWordArrayPtr& operator[](size_t index) cdef extern from "kj/array.h" namespace " ::kj": # Cython can't handle Array[word] as a function argument @@ -765,6 +785,7 @@ cdef extern from "capnp/serialize.h" namespace " ::capnp": cdef cppclass FlatArrayMessageReader(MessageReader): FlatArrayMessageReader(WordArrayPtr array) except +reraise_kj_exception FlatArrayMessageReader(WordArrayPtr array, ReaderOptions) except +reraise_kj_exception + const word* getEnd() const void writeMessageToFd(int, MessageBuilder&) except +reraise_kj_exception diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index d7e3360..21c8950 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -1,6 +1,8 @@ +# cython: language_level = 2 + from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp -from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder +from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, RpcSystem, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder, TwoWayPipe from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode from capnp.includes.types cimport * from capnp.helpers.non_circular cimport reraise_kj_exception @@ -12,6 +14,18 @@ cdef class _StructSchemaField: cdef object _parent cdef _init(self, C_StructSchema.Field other, parent=?) +cdef class _StringArrayPtr: + cdef StringPtr * thisptr + cdef object parent + cdef size_t size + cdef ArrayPtr[StringPtr] asArrayPtr(self) except +reraise_kj_exception + +cdef class SchemaParser: + cdef C_SchemaParser * thisptr + cdef public dict modules_by_id + cdef list _all_imports + cdef _StringArrayPtr _last_import_array + cpdef _parse_disk_file(self, displayName, diskPath, imports) except +reraise_kj_exception cdef class _DynamicOrphan: cdef C_DynamicOrphan thisptr @@ -46,15 +60,16 @@ cdef class _DynamicStructBuilder: cdef DynamicStruct_Builder thisptr cdef public object _parent cdef public bint is_root - cdef bint _is_written + cdef public bint _is_written cdef object _schema cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?) cdef _check_write(self) - cpdef to_bytes(_DynamicStructBuilder self) - cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) - cpdef to_bytes_packed(_DynamicStructBuilder self) + cpdef to_bytes(_DynamicStructBuilder self) except +reraise_kj_exception + cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception + cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception + cpdef to_bytes_packed(_DynamicStructBuilder self) except +reraise_kj_exception cpdef _get(self, field) cpdef _set(self, field, value) @@ -88,21 +103,19 @@ cdef class _Schema: cpdef as_struct(self) cpdef as_interface(self) cpdef as_enum(self) - cpdef get_dependency(self, id) cpdef get_proto(self) cdef class _InterfaceSchema: cdef C_InterfaceSchema thisptr cdef object __method_names, __method_names_inherited, __methods, __methods_inherited cdef _init(self, C_InterfaceSchema other) - cpdef get_dependency(self, id) cdef class _DynamicEnum: cdef capnp.DynamicEnum thisptr cdef public object _parent cdef _init(self, capnp.DynamicEnum other, object parent) - cpdef _as_str(self) + cpdef _as_str(self) except +reraise_kj_exception cdef class _DynamicListBuilder: cdef C_DynamicList.Builder thisptr @@ -115,9 +128,12 @@ cdef class _DynamicListBuilder: cpdef adopt(self, index, _DynamicOrphan orphan) cpdef disown(self, index) + cpdef init(self, index, size) + cdef to_python_reader(C_DynamicValue.Reader self, object parent) cdef to_python_builder(C_DynamicValue.Builder self, object parent) cdef _to_dict(msg, bint verbose, bint ordered) cdef _from_list(_DynamicListBuilder msg, list d) +cdef _from_tuple(_DynamicListBuilder msg, tuple d) cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent) cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 3af300d..c8b9aba 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1,25 +1,27 @@ # capnp.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 # distutils: libraries = capnpc capnp-rpc capnp kj-async kj # distutils: include_dirs = . # cython: c_string_type = str # cython: c_string_encoding = default # cython: embedsignature = True +# cython: language_level = 2 cimport cython -from capnp.helpers.helpers cimport makeRpcClientWithRestorer +from capnp.helpers.helpers cimport AsyncIoStreamReadHelper +from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope from libc.stdlib cimport malloc, free from libc.string cimport memcpy from cython.operator cimport dereference as deref from cpython.exc cimport PyErr_Clear +from cpython cimport array, Py_buffer, PyObject_CheckBuffer +from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_WRITABLE from types import ModuleType as _ModuleType import os as _os import sys as _sys -import imp as _imp import traceback as _traceback from functools import partial as _partial import warnings as _warnings @@ -29,6 +31,8 @@ import threading as _threading import socket as _socket import random as _random import collections as _collections +import array +import asyncio _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR @@ -113,17 +117,6 @@ cdef public VoidPromise * call_server_method(PyObject * _server, char * _method_ return NULL -cdef public C_Capability.Client * call_py_restorer(PyObject * _restorer, C_DynamicObject.Reader & _reader) except * with gil: - restorer = _restorer - reader = _DynamicObjectReader()._init(_reader, None) - - ret = restorer._restore(reader) - cdef _DynamicCapabilityServer server = ret - cdef _InterfaceSchema schema = ret.schema - - return new C_Capability.Client(helpers.server_to_client(schema.thisptr, server)) - - cdef public convert_array_pyobject(PyArray & arr) with gil: return [arr[i] for i in range(arr.size())] @@ -202,14 +195,16 @@ class KjException(Exception): Type = _make_enum('Type', **{x : x for x in _Type.reverse_mapping.values()}) - def __init__(self, message=None, nature=None, durability=None, wrapper=None): + def __init__(self, message=None, nature=None, durability=None, wrapper=None, type=None): if wrapper is not None: self.wrapper = wrapper self.message = str(wrapper) else: + self.wrapper = None self.message = message self.nature = nature self.durability = durability + self._type = type @property def file(self): @@ -222,7 +217,7 @@ class KjException(Exception): if self.wrapper is not None: return self.wrapper.type else: - return self.type + return self._type @property def description(self): if self.wrapper is not None: @@ -259,6 +254,14 @@ cdef public object get_exception_info(object exc_type, object exc_obj, object ex except: return (b'', 0, b"Couldn't determine python exception") +cdef schema_cpp.ReaderOptions make_reader_opts(traversal_limit_in_words, nesting_limit) with gil: + cdef schema_cpp.ReaderOptions opts + if traversal_limit_in_words is not None: + opts.traversalLimitInWords = traversal_limit_in_words + if nesting_limit is not None: + opts.nestingLimit = nesting_limit + return opts + ctypedef fused _DynamicStructReaderOrBuilder: _DynamicStructReader _DynamicStructBuilder @@ -274,8 +277,8 @@ ctypedef fused PromiseTypes: PromiseFulfillerPair cdef extern from "Python.h": - cdef int PyObject_AsReadBuffer(object, void** b, Py_ssize_t* c) - cdef int PyObject_AsWriteBuffer(object, void** b, Py_ssize_t* c) + cdef int PyObject_GetBuffer(object, Py_buffer *view, int flags) + cdef void PyBuffer_Release(Py_buffer *view) # Templated classes are weird in cython. I couldn't put it in a pxd header for some reason cdef extern from "capnp/list.h" namespace " ::capnp": @@ -301,11 +304,11 @@ cdef extern from "" namespace "std": capnp.AsyncIoContext moveAsyncContext"std::move"(capnp.AsyncIoContext) cdef extern from "" namespace " ::capnp": - StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) - StringTree printStructBuilder" ::capnp::prettyPrint"(DynamicStruct_Builder) - StringTree printRequest" ::capnp::prettyPrint"(Request &) - StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) - StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) + StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) except +reraise_kj_exception + StringTree printStructBuilder" ::capnp::prettyPrint"(DynamicStruct_Builder) except +reraise_kj_exception + StringTree printRequest" ::capnp::prettyPrint"(Request &) except +reraise_kj_exception + StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) except +reraise_kj_exception + StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) except +reraise_kj_exception cdef class _NodeReader: cdef C_Node.Reader thisptr @@ -534,6 +537,17 @@ cdef class _DynamicListBuilder: """ return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) + cpdef init(self, index, size): + """A method for initializing an element in a list + + :type index: int + :param index: The index of the element in the list + + :type size: int + :param size: Size of the element to be initialized. + """ + return to_python_builder(self.thisptr.init(index, size), self._parent) + def __str__(self): return printListBuilder(self.thisptr).flatten().cStr() @@ -542,9 +556,9 @@ cdef class _DynamicListBuilder: return '' % strListBuilder(self.thisptr).cStr() cdef class _List_NestedNode_Reader: - cdef List[C_Node.NestedNode].Reader thisptr + cdef C_Node.NestedNode.Reader.ListNestedNodeReader thisptr cdef _init(self, List[C_Node.NestedNode].Reader other): - self.thisptr = other + self.thisptr = other return self def __getitem__(self, index): @@ -650,13 +664,19 @@ cdef C_DynamicValue.Reader _extract_dynamic_server(object value): cdef C_DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): return C_DynamicValue.Reader(value.thisptr) +cdef C_DynamicValue.Reader _extract_any_pointer(_DynamicObjectReader value): + return C_DynamicValue.Reader(value.thisptr) + +cdef C_DynamicValue.Reader _extract_any_pointer_builder(_DynamicObjectBuilder value): + return C_DynamicValue.Reader(value.thisptr.asReader()) + cdef _setBytes(_DynamicSetterClasses thisptr, field, value): cdef capnp.StringPtr temp_string = capnp.StringPtr(value, len(value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.set(field, temp) cdef _setBaseString(_DynamicSetterClasses thisptr, field, value): - encoded_value = value.encode() + encoded_value = value.encode('utf-8') cdef capnp.StringPtr temp_string = capnp.StringPtr(encoded_value, len(encoded_value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.set(field, temp) @@ -667,7 +687,7 @@ cdef _setBytesField(DynamicStruct_Builder thisptr, _StructSchemaField field, val thisptr.setByField(field.thisptr, temp) cdef _setBaseStringField(DynamicStruct_Builder thisptr, _StructSchemaField field, value): - encoded_value = value.encode() + encoded_value = value.encode('utf-8') cdef capnp.StringPtr temp_string = capnp.StringPtr(encoded_value, len(encoded_value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) thisptr.setByField(field.thisptr, temp) @@ -695,6 +715,9 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): elif value_type is list: builder = to_python_builder(thisptr.init(field, len(value)), parent) _from_list(builder, value) + elif value_type is tuple: + builder = to_python_builder(thisptr.init(field, len(value)), parent) + _from_tuple(builder, value) elif value_type is dict: if _DynamicSetterClasses is DynamicStruct_Builder: builder = to_python_builder(thisptr.get(field), parent) @@ -715,6 +738,10 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) @@ -757,6 +784,10 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField thisptr.setByField(field.thisptr, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.setByField(field.thisptr, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) @@ -799,6 +830,10 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) + elif value_type is _DynamicObjectReader: + thisptr.set(field, _extract_any_pointer(value)) + elif value_type is _DynamicObjectBuilder: + thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException("Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'".format(field, str(value), str(type(value)))) @@ -862,10 +897,15 @@ cdef _to_dict(msg, bint verbose, bint ordered): return msg + cdef _from_list(_DynamicListBuilder msg, list d): - cdef size_t count = 0 - for i in range(len(d)): - msg._set(i, d[i]) + for i, x in enumerate(d): + msg._set(i, x) + + +cdef _from_tuple(_DynamicListBuilder msg, tuple d): + for i, x in enumerate(d): + msg._set(i, x) cdef class _DynamicEnum: @@ -907,6 +947,10 @@ cdef class _DynamicEnum: elif op == 5: # >= return left >= right + def __hash__(_DynamicEnum self): + return hash(self._as_str()) + + cdef class _DynamicEnumField: cdef _init(self, proto): self.thisptr = proto @@ -984,16 +1028,16 @@ cdef class _DynamicStructReader: return self cpdef _get(self, field): - return to_python_reader(self.thisptr.get(field), self._parent) + return to_python_reader(self.thisptr.get(field), self) def __getattr__(self, field): try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _get_by_field(self, _StructSchemaField field): - return to_python_reader(self.thisptr.getByField(field.thisptr), self._parent) + return to_python_reader(self.thisptr.getByField(field.thisptr), self) cpdef _has(self, field): return self.thisptr.has(field) @@ -1041,7 +1085,7 @@ cdef class _DynamicStructReader: return self._schema def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def __str__(self): return printStructReader(self.thisptr).flatten().cStr() @@ -1104,7 +1148,7 @@ cdef class _DynamicStructBuilder: if not self.is_root: raise KjException("You can only call write() on the message's root struct.") if self._is_written: - _warnings.warn("This message has already been written once. Be very careful that you're not setting Text/Struct/List fields more than once, since that will cause memory leaks (both in memory and in the serialized data). You can disable this warning by setting the `_is_written` field of this object to False after every write.") + _warnings.warn("This message has already been written once. Be very careful that you're not setting Text/Struct/List fields more than once, since that will cause memory leaks (both in memory and in the serialized data). You can disable this warning by calling the `clear_write_flag` method of this object after every write.") def write(self, file): """Writes the struct's containing message to the given file object in unpacked binary format. @@ -1157,6 +1201,20 @@ cdef class _DynamicStructBuilder: self._is_written = True return ret + cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception: + """Returns the struct's containing message as a Python list of Python bytes objects. + + This avoids making copies. + + NB: This is not currently supported on PyPy. + + :rtype: list + """ + self._check_write() + cdef _MessageBuilder builder = self._parent + segments = builder.get_segments_for_output() + return segments + cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception: cdef _MessageBuilder builder = self._parent array = helpers.messageToPackedBytes(deref(builder.thisptr), word_count) @@ -1190,7 +1248,7 @@ cdef class _DynamicStructBuilder: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _set(self, field, value): _setDynamicField(self.thisptr, field, value, self._parent) @@ -1202,7 +1260,7 @@ cdef class _DynamicStructBuilder: try: self._set(field, value) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef _has(self, field): return self.thisptr.has(field) @@ -1359,7 +1417,7 @@ cdef class _DynamicStructBuilder: return self._schema def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def __str__(self): return printStructBuilder(self.thisptr).flatten().cStr() @@ -1387,6 +1445,13 @@ cdef class _DynamicStructBuilder: size = self.thisptr.totalSize() return _MessageSize(size.wordCount, size.capCount) + def clear_write_flag(self): + """A method used to clear the _is_written flag. + + This allows you to write the struct more than once without seeing any warnings. + """ + self._is_written = False + def __reduce_ex__(self, proto): return _struct_reducer, (self.schema.node.id, self.to_bytes()) @@ -1411,7 +1476,7 @@ cdef class _DynamicStructPipeline: if type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init((self.thisptr.get(field)).asCapability(), self._parent) elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) + return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline((self.thisptr.get(field)).asStruct()), self._parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -1421,7 +1486,7 @@ cdef class _DynamicStructPipeline: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] property schema: """A property that returns the _StructSchema object matching this reader""" @@ -1429,7 +1494,7 @@ cdef class _DynamicStructPipeline: return _StructSchema()._init(self.thisptr.getSchema()) def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) # def __str__(self): # return printStructReader(self.thisptr).flatten().cStr() @@ -1570,7 +1635,7 @@ cdef class _EventLoop: self._init() cdef _init(self) except +reraise_kj_exception: - self.thisptr = new capnp.AsyncIoContext(moveAsyncContext(capnp.setupAsyncIo())) + self.thisptr = new capnp.AsyncIoContext(capnp.setupAsyncIo()) def __dealloc__(self): del self.thisptr #TODO:MEMORY: fix problems with Promises still being around @@ -1579,16 +1644,20 @@ cdef class _EventLoop: del self.thisptr self.thisptr = NULL + cdef TwoWayPipe makeTwoWayPipe(self): + return deref(deref(self.thisptr).provider).newTwoWayPipe() + cdef Own[AsyncIoStream] wrapSocketFd(self, int fd): return deref(deref(self.thisptr).lowLevelProvider).wrapSocketFd(fd) -cdef _EventLoop C_DEFAULT_EVENT_LOOP = _EventLoop() +cdef _EventLoop C_DEFAULT_EVENT_LOOP _C_DEFAULT_EVENT_LOOP_LOCAL = None _THREAD_LOCAL_EVENT_LOOPS = [] cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): 'Optimization for not having to deal with threadlocal event loops unless we need to' + global C_DEFAULT_EVENT_LOOP if C_DEFAULT_EVENT_LOOP is not None: return C_DEFAULT_EVENT_LOOP elif _C_DEFAULT_EVENT_LOOP_LOCAL is not None: @@ -1598,8 +1667,9 @@ cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): else: _C_DEFAULT_EVENT_LOOP_LOCAL.loop = _EventLoop() return _C_DEFAULT_EVENT_LOOP_LOCAL.loop - - raise KjException("You don't have any EventLoops running. Please make sure to add one") + else: + C_DEFAULT_EVENT_LOOP = _EventLoop() + return C_DEFAULT_EVENT_LOOP cdef class _Timer: cdef capnp.Timer * thisptr @@ -1609,7 +1679,7 @@ cdef class _Timer: return self cpdef after_delay(self, time) except +reraise_kj_exception: - return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Duration(time))) + return _VoidPromise()._init(self.thisptr.afterDelay(capnp.Nanoseconds(time))) def getTimer(): return _Timer()._init(helpers.getTimer(C_DEFAULT_EVENT_LOOP_GETTER().thisptr)) @@ -1660,6 +1730,10 @@ def wait_forever(): cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() helpers.waitNeverDone(deref(loop.thisptr).waitScope) +def poll_once(): + cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() + helpers.pollWaitScope(deref(loop.thisptr).waitScope) + cdef class _CallContext: cdef CallContext * thisptr @@ -1735,7 +1809,7 @@ cdef class Promise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: @@ -1798,7 +1872,7 @@ cdef class _VoidPromise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: @@ -1851,11 +1925,24 @@ cdef class _RemotePromise: def __dealloc__(self): del self.thisptr - cpdef wait(self) except +reraise_kj_exception: + cpdef _wait(self) except +reraise_kj_exception: + return _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent) + + def wait(self): if self.is_consumed: raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') - ret = _Response()._init_childptr(helpers.waitRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope), self._parent) + ret = self._wait() + self.is_consumed = True + return ret + + async def a_wait(self): + if self.is_consumed: + raise KjException('Promise was already used in a consuming operation. You can no longer use this Promise object') + + while not helpers.pollRemote(self.thisptr, deref(self._event_loop.thisptr).waitScope): + await asyncio.sleep(0.01) + ret = self._wait() self.is_consumed = True return ret @@ -1871,7 +1958,7 @@ cdef class _RemotePromise: argspec = None try: - argspec = _inspect.getargspec(func) + argspec = _inspect.getfullargspec(func) except: pass if argspec: @@ -1880,9 +1967,6 @@ cdef class _RemotePromise: if args_length - defaults_length != 1: raise KjException('Function passed to `then` call must take exactly one argument') - Py_INCREF(func) - Py_INCREF(error_func) - cdef Promise new_promise = Promise()._init(helpers.then(deref(self.thisptr), func, error_func), self) return Promise()._init(new_promise.thisptr.attach(capnp.makePyRefCounter(func), capnp.makePyRefCounter(error_func)), new_promise) @@ -1891,7 +1975,7 @@ cdef class _RemotePromise: if type == capnp.TYPE_CAPABILITY: return _DynamicCapabilityClient()._init((self.thisptr.get(field)).asCapability(), self._parent) elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline(moveStructPipeline((self.thisptr.get(field)).asStruct())), self._parent) + return _DynamicStructPipeline()._init(new C_DynamicStruct.Pipeline((self.thisptr.get(field)).asStruct()), self._parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -1901,7 +1985,7 @@ cdef class _RemotePromise: try: return self._get(field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] property schema: """A property that returns the _StructSchema object matching this reader""" @@ -1909,7 +1993,7 @@ cdef class _RemotePromise: return _StructSchema()._init(self.thisptr.getSchema()) def __dir__(self): - return list(self.schema.fieldnames) + return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) def to_dict(self, verbose=False, ordered=False): return _to_dict(self, verbose, ordered) @@ -2004,7 +2088,7 @@ cdef class _DynamicCapabilityServer: try: return getattr(self.server, field) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cdef class _DynamicCapabilityClient: cdef C_DynamicCapability.Client thisptr @@ -2039,7 +2123,7 @@ cdef class _DynamicCapabilityClient: return _find_field_order(params.struct) cdef _set_fields(self, Request * request, name, args, kwargs): - if args is not None: + if args is not None and len(args) > 0: arg_names = self._find_method_args(name) if len(args) > len(arg_names): raise KjException('Too many arguments passed to `%s`. Expected %d and got %d' % (name, len(arg_names), len(args))) @@ -2086,7 +2170,7 @@ cdef class _DynamicCapabilityClient: raise AttributeError('Method named %s not found' % name) return _partial(self._send, name) except KjException as e: - raise e._to_python() + raise e._to_python(), None, _sys.exc_info()[2] cpdef upcast(self, schema) except +reraise_kj_exception: cdef _InterfaceSchema s @@ -2113,7 +2197,7 @@ cdef class _DynamicCapabilityClient: return self._cached_schema def __dir__(self): - return list(self.schema.method_names_inherited) + return list(set(self.schema.method_names_inherited) + tuple(dir(self.__class__))) cdef class _CapabilityClient: cdef C_Capability.Client * thisptr @@ -2135,123 +2219,91 @@ cdef class _CapabilityClient: s = schema return _DynamicCapabilityClient()._init(self.thisptr.castAs(s.thisptr), self._parent) -cdef class _Restorer: - cdef PyRestorer * thisptr - cdef public object restore, _parent - - def __init__(self, restore, parent=None): - self.thisptr = new PyRestorer(self) - self.restore = restore - self._parent = parent - - def __dealloc__(self): - del self.thisptr - - def _restore(self, obj): - return self.restore(obj) - cdef class _TwoPartyVatNetwork: cdef Own[C_TwoPartyVatNetwork] thisptr cdef _AsyncIoStream stream - cdef _init(self, _AsyncIoStream stream, Side side): + cdef _init(self, _AsyncIoStream stream, Side side, schema_cpp.ReaderOptions opts): self.stream = stream - self.thisptr = makeTwoPartyVatNetwork(deref(stream.thisptr), side) + self.thisptr = makeTwoPartyVatNetwork(deref(stream.thisptr), side, opts) + return self + + cdef _init_pipe(self, _TwoWayPipe pipe, Side side, schema_cpp.ReaderOptions opts): + self.thisptr = makeTwoPartyVatNetwork(deref(pipe._pipe.ends[0]), side, opts) return self cpdef on_disconnect(self) except +reraise_kj_exception: return _VoidPromise()._init(deref(self.thisptr).onDisconnect(), self) -cdef _Restorer _convert_restorer(restorer): - if isinstance(restorer, _RestorerImpl): - return _Restorer(restorer._restore, restorer) - elif type(restorer) is _Restorer: - return restorer - elif hasattr(restorer, 'restore'): - return _Restorer(restorer.restore, restorer) - elif callable(restorer): - return _Restorer(restorer) - else: - raise KjException("Restorer object ({}) isn't able to be used as a restore".format(str(restorer))) - cdef class TwoPartyClient: cdef RpcSystem * thisptr cdef public _TwoPartyVatNetwork _network cdef public object _orig_stream - cdef public _Restorer _restorer cdef public _AsyncIoStream _stream + cdef public _TwoWayPipe _pipe - def __init__(self, socket, restorer=None): + def __init__(self, socket=None, traversal_limit_in_words=None, nesting_limit=None): if isinstance(socket, basestring): socket = self._connect(socket) - self._orig_stream = socket - self._stream = _FdAsyncIoStream(socket.fileno()) - self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT) - if restorer is None: - self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) - self._restorer = None - else: - self._restorer = _convert_restorer(restorer) - self.thisptr = new RpcSystem(makeRpcClientWithRestorer(deref(self._network.thisptr), deref(self._restorer.thisptr))) + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - Py_INCREF(self._restorer) - Py_INCREF(self._orig_stream) - Py_INCREF(self._stream) + self._orig_stream = socket + if self._orig_stream: + self._stream = _FdAsyncIoStream(socket.fileno()) + self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.CLIENT, opts) + else: + # Initialize TwoWayPipe, to use pipe() acquire other end of the pipe using read() and write() methods + self._pipe = _TwoWayPipe() + self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.CLIENT, opts) + + self.thisptr = new RpcSystem(makeRpcClient(deref(self._network.thisptr))) + if self._orig_stream: + Py_INCREF(self._orig_stream) + Py_INCREF(self._stream) + else: + Py_INCREF(self._pipe) Py_INCREF(self._network) # TODO:MEMORY: attach this to onDrained, also figure out what's leaking + async def read(self, bufsize): + cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper( + self._pipe._pipe.ends[1].get(), + &self._pipe._event_loop.thisptr.waitScope, + bufsize + ) + while not reader.poll(): + await asyncio.sleep(0.01) + + cdef array.array read_buffer = array.array('b', []) + array.resize(read_buffer, reader.read_size()) + memcpy(read_buffer.data.as_voidptr, reader.read_buffer(), reader.read_size()) + del reader + return read_buffer + + def write(self, data): + cdef array.array write_buffer = array.array('b', data) + deref(self._pipe._pipe.ends[1]).write( + write_buffer.data.as_voidptr, + len(data) + ).wait(self._pipe._event_loop.thisptr.waitScope) + def __dealloc__(self): del self.thisptr cpdef _connect(self, host_string): - host, port = host_string.split(':') - - sock = _socket.create_connection((host, port)) - - # Set TCP_NODELAY on socket to disable Nagle's algorithm. This is not - # neccessary, but it speeds things up. - sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) - return sock - - cpdef restore(self, objectId) except +reraise_kj_exception: - cdef _MessageBuilder builder - cdef _MessageReader reader - cdef _DynamicObjectBuilder object_builder - cdef _DynamicObjectReader object_reader - - if objectId is None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr)), self) - elif type(objectId) is _DynamicObjectBuilder: - object_builder = objectId - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(object_builder.thisptr)), self) - elif type(objectId) is _DynamicObjectReader: - object_reader = objectId - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), object_reader.thisptr), self) + if host_string.startswith('unix:'): + path = host_string[5:] + sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) + sock.connect(path) else: - if not hasattr(objectId, 'is_root'): - raise KjException("objectId was not a valid Cap'n Proto struct") - if not objectId.is_root: - raise KjException("objectId must be the root of a Cap'n Proto message, ie. addressbook_capnp.Person.new_message()") + host, port = host_string.split(':') - try: - builder = objectId._parent - except: - reader = objectId._parent + sock = _socket.create_connection((host, port)) - if builder is not None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(builder.thisptr)), self) - elif reader is not None: - return _CapabilityClient()._init(helpers.restoreHelper(deref(self.thisptr), deref(reader.thisptr)), self) - else: - raise KjException("objectId unexpectedly was not convertible to the proper type") - - cpdef ez_restore(self, textId) except +reraise_kj_exception: - # ez-rpc from the C++ API uses Text under the hood - ref = _MallocMessageBuilder().get_root_as_any() - # objectId is an AnyPointer, so we have a special method for setting it to text - ref.set_as_text(textId) - - return self.restore(ref) + # Set TCP_NODELAY on socket to disable Nagle's algorithm. This is not + # neccessary, but it speeds things up. + sock.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1) + return sock cpdef bootstrap(self) except +reraise_kj_exception: return _CapabilityClient()._init(helpers.bootstrapHelper(deref(self.thisptr)), self) @@ -2263,62 +2315,86 @@ cdef class TwoPartyServer: cdef RpcSystem * thisptr cdef public _TwoPartyVatNetwork _network cdef public object _orig_stream, _server_socket, _disconnect_promise - cdef public _Restorer _restorer cdef public _AsyncIoStream _stream + cdef public _TwoWayPipe _pipe cdef object _port cdef public object port_promise, _bootstrap cdef capnp.TaskSet * _task_set cdef capnp.ErrorHandler _error_handler - def __init__(self, socket, restorer=None, server_socket=None, bootstrap=None): - if not restorer and not bootstrap: - raise KjException("You must provide either a bootstrap interface or a restorer (deperecated) to a server constructor.") + def __init__(self, socket=None, server_socket=None, bootstrap=None, + traversal_limit_in_words=None, nesting_limit=None): + if not bootstrap: + raise KjException("You must provide a bootstrap interface to a server constructor.") cdef _InterfaceSchema schema - self._restorer = None + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._bootstrap = None if isinstance(socket, basestring): - self._connect(socket, restorer, bootstrap) - else: - self._orig_stream = socket + self._connect(socket, bootstrap) + return + + self._orig_stream = socket + if self._orig_stream: self._stream = _FdAsyncIoStream(socket.fileno()) - self._server_socket = server_socket - self._port = 0 - self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER) + self._network = _TwoPartyVatNetwork()._init(self._stream, capnp.SERVER, opts) + else: + # Initialize TwoWayPipe, to use pipe() acquire other end of the pipe using read() and write() methods + self._pipe = _TwoWayPipe() + self._network = _TwoPartyVatNetwork()._init_pipe(self._pipe, capnp.SERVER, opts) - if bootstrap: - self._bootstrap = bootstrap - schema = bootstrap.schema - self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) - elif restorer: - self._restorer = _convert_restorer(restorer) - self.thisptr = new RpcSystem(makeRpcServer(deref(self._network.thisptr), deref(self._restorer.thisptr))) + self._server_socket = server_socket + self._port = 0 - Py_INCREF(self._orig_stream) - Py_INCREF(self._stream) - Py_INCREF(self._restorer) - Py_INCREF(self._bootstrap) - Py_INCREF(self._network) - self._disconnect_promise = self.on_disconnect().then(self._decref) + if bootstrap: + self._bootstrap = bootstrap + schema = bootstrap.schema + self.thisptr = new RpcSystem(makeRpcServerBootstrap(deref(self._network.thisptr), helpers.server_to_client(schema.thisptr, bootstrap))) - cpdef _connect(self, host_string, restorer, bootstrap): + Py_INCREF(self._orig_stream) + Py_INCREF(self._stream) + Py_INCREF(self._pipe) + Py_INCREF(self._bootstrap) + Py_INCREF(self._network) + self._disconnect_promise = self.on_disconnect().then(self._decref) + + async def read(self, bufsize): + cdef AsyncIoStreamReadHelper *reader = new AsyncIoStreamReadHelper( + self._pipe._pipe.ends[1].get(), + &self._pipe._event_loop.thisptr.waitScope, + bufsize + ) + while not reader.poll(): + await asyncio.sleep(0.01) + + cdef array.array read_buffer = array.array('b', []) + array.resize(read_buffer, reader.read_size()) + memcpy(read_buffer.data.as_voidptr, reader.read_buffer(), reader.read_size()) + del reader + return read_buffer + + async def write(self, data): + cdef array.array write_buffer = array.array('b', data) + deref(self._pipe._pipe.ends[1]).write( + write_buffer.data.as_voidptr, + len(data) + ).wait(self._pipe._event_loop.thisptr.waitScope) + + cpdef _connect(self, host_string, bootstrap): cdef _InterfaceSchema schema cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() cdef capnp.StringPtr temp_string = capnp.StringPtr(host_string, len(host_string)) self._task_set = new capnp.TaskSet(self._error_handler) - if restorer: - self._restorer = _convert_restorer(restorer) - self.port_promise = Promise()._init(helpers.connectServerRestorer(deref(self._task_set), deref(self._restorer.thisptr), loop.thisptr, temp_string)) - else: - self._bootstrap = bootstrap - Py_INCREF(self._bootstrap) - schema = bootstrap.schema - self.port_promise = Promise()._init(helpers.connectServer(deref(self._task_set), helpers.server_to_client(schema.thisptr, bootstrap), loop.thisptr, temp_string)) + + self._bootstrap = bootstrap + Py_INCREF(self._bootstrap) + schema = bootstrap.schema + self.port_promise = Promise()._init(helpers.connectServer(deref(self._task_set), helpers.server_to_client(schema.thisptr, bootstrap), loop.thisptr, temp_string)) def _decref(self): Py_DECREF(self._bootstrap) - Py_DECREF(self._restorer) + Py_INCREF(self._pipe) Py_DECREF(self._orig_stream) Py_DECREF(self._stream) Py_DECREF(self._network) @@ -2330,12 +2406,20 @@ cdef class TwoPartyServer: cpdef on_disconnect(self) except +reraise_kj_exception: return _VoidPromise()._init(deref(self._network.thisptr).onDisconnect()) + async def poll_forever(self): + while True: + poll_once() + await asyncio.sleep(0.01) + cpdef run_forever(self): if self.port_promise is None: raise KjException("You must pass a string as the socket parameter in __init__ to use this function") wait_forever() + cpdef bootstrap(self) except +reraise_kj_exception: + return _CapabilityClient()._init(helpers.bootstrapHelperServer(deref(self.thisptr)), self) + property port: def __get__(self): if self._port is None: @@ -2347,6 +2431,18 @@ cdef class TwoPartyServer: cdef class _AsyncIoStream: cdef Own[AsyncIoStream] thisptr +cdef class _TwoWayPipe: + cdef _EventLoop _event_loop + cdef TwoWayPipe _pipe + + def __init__(self): + self._init() + + cpdef _init(self) except +reraise_kj_exception: + self._event_loop = C_DEFAULT_EVENT_LOOP_GETTER() + # Create two way pipe using AsyncIoContext + self._pipe = self._event_loop.makeTwoWayPipe() + cdef class _FdAsyncIoStream(_AsyncIoStream): cdef _EventLoop _event_loop @@ -2391,11 +2487,6 @@ cdef class _Schema: cpdef as_enum(self): return _EnumSchema()._init(self.thisptr.asEnum()) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - cpdef get_proto(self): return _NodeReader().init(self.thisptr.getProto()) @@ -2478,11 +2569,6 @@ cdef class _StructSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - def __richcmp__(_StructSchema self, _StructSchema other, mode): if mode == 2: return self.thisptr == other.thisptr @@ -2539,6 +2625,12 @@ cdef class _InterfaceMethod: # TODO(soon): make sure this is memory safe return _StructSchema()._init(self.thisptr.getParamType()) + property result_type: + """The type of this method's result struct""" + def __get__(self): + # TODO(soon): make sure this is memory safe + return _StructSchema()._init(self.thisptr.getResultType()) + cdef class _InterfaceSchema: cdef _init(self, C_InterfaceSchema other): self.thisptr = other @@ -2610,11 +2702,6 @@ cdef class _InterfaceSchema: def __get__(self): return _DynamicStructReader()._init(self.thisptr.getProto(), self) - cpdef get_dependency(self, id): - '.. warning:: This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream' - _warnings.warn('This method is deprecated and will be removed in the 0.6 release. You can access the fields directly from the schema now, so this method is superfluous and deprecated upstream', UserWarning) - return _Schema()._init(self.thisptr.getDependency(id)) - def __repr__(self): return '' % self.node.displayName @@ -2787,20 +2874,13 @@ cdef _new_message(self, kwargs, num_first_segment_words): msg.from_dict(kwargs) return msg -class _RestorerImpl(object): - pass - class _StructModuleWhich(object): pass class _StructModule(object): def __init__(self, schema, name): - def _restore(self, obj): - return self.restore(obj.as_struct(self.schema)) self.schema = schema - self.Restorer = type(name + '.Restorer', (_RestorerImpl,), {'schema':schema, '_restore':_restore}) - # Add enums for union fields for field, raw_field in zip(schema.node.struct.fields, schema.fields_list): if field.which() == 'group': @@ -2830,9 +2910,9 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: :class:`_DynamicStructReader`""" - reader = _StreamFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit) + reader = _StreamFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) - def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None): + def read_multiple(self, file, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): """Returns an iterable, that when traversed will return Readers for messages. :type file: file @@ -2844,8 +2924,11 @@ class _StructModule(object): :type nesting_limit: int :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + :type skip_copy: bool + :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is never read completely. Skip this only if you know what you're doing. + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultipleMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultipleMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) return reader def read_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object read from file. @@ -2860,9 +2943,9 @@ class _StructModule(object): :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. :rtype: :class:`_DynamicStructReader`""" - reader = _PackedFdMessageReader(file.fileno(), traversal_limit_in_words, nesting_limit) + reader = _PackedFdMessageReader(file, traversal_limit_in_words, nesting_limit) return reader.get_root(self.schema) - def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None): + def read_multiple_packed(self, file, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): """Returns an iterable, that when traversed will return Readers for messages. :type file: file @@ -2874,8 +2957,11 @@ class _StructModule(object): :type nesting_limit: int :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + :type skip_copy: bool + :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is never read completely. Skip this only if you know what you're doing. + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) + reader = _MultiplePackedMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) return reader def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns an iterable, that when traversed will return Readers for messages. @@ -2931,6 +3017,17 @@ class _StructModule(object): else: message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit) return message.get_root(self.schema) + def from_segments(self, segments, traversal_limit_in_words = None, nesting_limit = None): + """Returns a Reader for a list of segment bytes. + + This avoids making copies. + + NB: This is not currently supported on PyPy. + + :rtype: list + """ + message = _SegmentArrayMessageReader(segments, traversal_limit_in_words, nesting_limit) + return message.get_root(self.schema) def from_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): """Returns a Reader for the packed object in buf. @@ -2988,10 +3085,6 @@ class _EnumModule(object): setattr(self, name, val) cdef class _StringArrayPtr: - cdef StringPtr * thisptr - cdef object parent - cdef size_t size - def __cinit__(self, size_t size, parent): self.size = size self.thisptr = malloc(sizeof(StringPtr) * size) @@ -3009,10 +3102,6 @@ cdef class SchemaParser: Do not use this class unless you're sure you know what you're doing. Use the convenience method :func:`load` instead. """ - cdef C_SchemaParser * thisptr - cdef public dict modules_by_id - cdef list _all_imports - cdef _StringArrayPtr _last_import_array def __cinit__(self): self.thisptr = new C_SchemaParser() @@ -3038,6 +3127,7 @@ cdef class SchemaParser: self._last_import_array = importArray ret = _ParsedSchema() + # TODO (HaaTa): Convert to parseFromDirectory() as per deprecation note ret._init_child(self.thisptr.parseDiskFile(displayName, diskPath, importArray.asArrayPtr())) return ret @@ -3123,6 +3213,9 @@ cdef class SchemaParser: if not _os.path.isfile(file_name): raise IOError("File not found: " + file_name) + if not file_name.endswith('.capnp'): + raise ValueError("File does not end with .capnp, {}".format(file_name)) + if display_name is None: display_name = _os.path.basename(file_name) @@ -3131,7 +3224,13 @@ cdef class SchemaParser: module._parser = parser - fileSchema = parser._parse_disk_file(display_name, file_name, imports) + # Some systems (Windows running pytest) add non-directories to the sys.path used for imports + # Filter these out so kj doesn't implode when searching paths + filtered_imports = [] + for imp in imports: + if _os.path.isdir(imp): + filtered_imports.append(imp) + fileSchema = parser._parse_disk_file(display_name, file_name, filtered_imports) _load(fileSchema, module) abs_path = _os.path.abspath(file_name) @@ -3227,6 +3326,18 @@ cdef class _MessageBuilder: self.thisptr.setRootDynamicStruct((<_DynamicStructReader>value).thisptr) return self.get_root(value.schema) + cpdef get_segments_for_output(self) except +reraise_kj_exception: + segments = self.thisptr.getSegmentsForOutput() + res = [] + cdef const char* ptr + cdef bytes segment_bytes + for i in range(0, segments.size()): + segment = segments[i] + ptr = segment.begin() + segment_bytes = ptr[:8*segment.size()] + res.append(segment_bytes) + return res + cpdef new_orphan(self, schema) except +reraise_kj_exception: """A method for instantiating Cap'n Proto orphans @@ -3322,21 +3433,17 @@ cdef class _StreamFdMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_message_to_fd` and :class:`_MessageBuilder`, but in one class:: f = open('out.txt') - message = _StreamFdMessageReader(f.fileno()) + message = _StreamFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name :Parameters: - fd (`int`) - A file descriptor """ - def __init__(self, int fd, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - - self.thisptr = new schema_cpp.StreamFdMessageReader(fd, opts) + self._parent = file + self.thisptr = new schema_cpp.StreamFdMessageReader(file.fileno(), opts) def __dealloc__(self): del self.thisptr @@ -3348,7 +3455,7 @@ cdef class _PackedMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name @@ -3358,15 +3465,9 @@ cdef class _PackedMessageReader(_MessageReader): pass cdef _init(self, schema_cpp.BufferedInputStream & stream, traversal_limit_in_words = None, nesting_limit = None, parent = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = parent - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.PackedMessageReader(stream, opts) return self @@ -3376,28 +3477,24 @@ cdef class _PackedMessageReader(_MessageReader): cdef class _PackedMessageReaderBytes(_MessageReader): cdef schema_cpp.ArrayInputStream * stream + cdef Py_buffer view def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = buf - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit + if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) - - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) self.thisptr = new schema_cpp.PackedMessageReader(deref(self.stream), opts) def __dealloc__(self): del self.thisptr del self.stream + PyBuffer_Release(&self.view) cdef class _InputMessageReader(_MessageReader): """Read a Cap'n Proto message from a file descriptor in a packed manner @@ -3405,7 +3502,7 @@ cdef class _InputMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name @@ -3415,15 +3512,9 @@ cdef class _InputMessageReader(_MessageReader): pass cdef _init(self, schema_cpp.BufferedInputStream & stream, traversal_limit_in_words = None, nesting_limit = None, parent = None): - cdef schema_cpp.ReaderOptions opts + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) self._parent = parent - - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - self.thisptr = new schema_cpp.InputStreamMessageReader(stream, opts) return self @@ -3437,21 +3528,17 @@ cdef class _PackedFdMessageReader(_MessageReader): You use this class to for reading message(s) from a file. It's analagous to the inverse of :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: f = open('out.txt') - message = _PackedFdMessageReader(f.fileno()) + message = _PackedFdMessageReader(f) person = message.get_root(addressbook.Person) print person.name :Parameters: - fd (`int`) - A file descriptor """ - def __init__(self, int fd, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts + def __init__(self, file, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - - self.thisptr = new schema_cpp.PackedFdMessageReader(fd, opts) + self._parent = file + self.thisptr = new schema_cpp.PackedFdMessageReader(file.fileno(), opts) def __dealloc__(self): del self.thisptr @@ -3460,15 +3547,18 @@ cdef class _PackedFdMessageReader(_MessageReader): cdef class _MultipleMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef cbool skip_copy - cdef public object traversal_limit_in_words, nesting_limit, schema + cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, int fd, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): + self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit + self.skip_copy = skip_copy - self.stream = new schema_cpp.FdInputStream(fd) + self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): @@ -3478,7 +3568,10 @@ cdef class _MultipleMessageReader: def __next__(self): try: reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root(self.schema) + ret = reader.get_root(self.schema) + if not self.skip_copy: + ret = ret.as_builder().as_reader() + return ret except KjException as e: if 'EOF' in str(e): raise StopIteration @@ -3491,15 +3584,18 @@ cdef class _MultipleMessageReader: cdef class _MultiplePackedMessageReader: cdef schema_cpp.FdInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef cbool skip_copy - cdef public object traversal_limit_in_words, nesting_limit, schema + cdef public object traversal_limit_in_words, nesting_limit, schema, file - def __init__(self, int fd, schema, traversal_limit_in_words = None, nesting_limit = None): + def __init__(self, file, schema, traversal_limit_in_words = None, nesting_limit = None, skip_copy = False): + self.file = file self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit + self.skip_copy = skip_copy - self.stream = new schema_cpp.FdInputStream(fd) + self.stream = new schema_cpp.FdInputStream(file.fileno()) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): @@ -3509,7 +3605,10 @@ cdef class _MultiplePackedMessageReader: def __next__(self): try: reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root(self.schema) + ret = reader.get_root(self.schema) + if not self.skip_copy: + ret = ret.as_builder().as_reader() + return ret except KjException as e: if 'EOF' in str(e): raise StopIteration @@ -3520,31 +3619,43 @@ cdef class _MultiplePackedMessageReader: return self cdef class _MultipleBytesMessageReader: - cdef schema_cpp.ArrayInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - - cdef public object traversal_limit_in_words, nesting_limit, schema, buf + cdef Py_ssize_t offset, sz + cdef const char *ptr + cdef object _object_to_pin + cdef public object traversal_limit_in_words, nesting_limit, schema def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.offset = 0 self.schema = schema self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) + self.sz = len(buf) + if isinstance(buf, bytes): + self.ptr = buf + if (self.ptr) % 8 != 0: + aligned = _AlignedBuffer(buf) + self.ptr = aligned.buf + self._object_to_pin = aligned + else: + self._object_to_pin = buf + self.ptr = buf + elif PyObject_CheckBuffer(buf): + view = _BufferView(buf) + self.ptr = view.buf + self._object_to_pin = view + else: + raise TypeError('expected buffer-like object in FlatArrayMessageReader') - self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - del self.buffered_stream - del self.stream def __next__(self): + cdef _FlatArrayMessageReaderAligned reader + if self.offset == self.sz: + raise StopIteration try: - reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + reader = _FlatArrayMessageReaderAligned() + reader._init(self._object_to_pin, self.ptr + self.offset, self.sz - self.offset, self.traversal_limit_in_words, self.nesting_limit) + self.offset += reader.msg_size return reader.get_root(self.schema) except KjException as e: if 'EOF' in str(e): @@ -3558,6 +3669,7 @@ cdef class _MultipleBytesMessageReader: cdef class _MultipleBytesPackedMessageReader: cdef schema_cpp.ArrayInputStream * stream cdef schema_cpp.BufferedInputStream * buffered_stream + cdef Py_buffer view cdef public object traversal_limit_in_words, nesting_limit, schema, buf @@ -3566,15 +3678,15 @@ cdef class _MultipleBytesPackedMessageReader: self.traversal_limit_in_words = traversal_limit_in_words self.nesting_limit = nesting_limit - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(buf, &ptr, &sz) + if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): + PyBuffer_Release(&self.view) del self.buffered_stream del self.stream @@ -3595,65 +3707,157 @@ cdef class _MultipleBytesPackedMessageReader: cdef class _AlignedBuffer: cdef char * buf cdef bint allocated + cdef Py_buffer view # other should also have a length that's a multiple of 8 def __init__(self, other): - cdef const void *ptr - cdef Py_ssize_t sz - PyObject_AsReadBuffer(other, &ptr, &sz) + if PyObject_GetBuffer(other, &self.view, PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") other_len = len(other) # malloc is defined as being word aligned # we don't care about adding NULL terminating character self.buf = malloc(other_len) - memcpy(self.buf, ptr, other_len) + memcpy(self.buf, self.view.buf, other_len) self.allocated = True def __dealloc__(self): if self.allocated: free(self.buf) + PyBuffer_Release(&self.view) + @cython.internal -cdef class _FlatArrayMessageReader(_MessageReader): +cdef class _BufferView: + cdef Py_buffer view + cdef char * buf + + def __init__(self, other): + cdef int ret = PyObject_GetBuffer(other, &self.view, PyBUF_SIMPLE) + if ret < 0: + raise ValueError("Invalid buffer passed to BufferView") + self.buf = self.view.buf + + def __dealloc__(self): + PyBuffer_Release(&self.view) + +@cython.internal +cdef class _FlatArrayMessageReaderAligned(_MessageReader): + """ + Creates a reader based on a contiguous block of memory + + For performance consideration it's assumed that the provided buffer is already aligned. This + allows us to align a set of adjacent messages with a single align operation. + """ cdef object _object_to_pin - def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): - cdef schema_cpp.ReaderOptions opts - cdef _AlignedBuffer aligned + cdef Py_ssize_t msg_size + def __init__(self): + self.msg_size = 0 - if traversal_limit_in_words is not None: - opts.traversalLimitInWords = traversal_limit_in_words - if nesting_limit is not None: - opts.nestingLimit = nesting_limit - sz = len(buf) - if sz % 8 != 0: - raise ValueError("input length must be a multiple of eight bytes") + cdef _init(self, buf, const char *ptr, Py_ssize_t sz, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) + cdef schema_cpp.FlatArrayMessageReader * flat_reader - cdef char * ptr = buf - if (ptr) % 8 != 0: - aligned = _AlignedBuffer(buf) - ptr = aligned.buf - self._object_to_pin = aligned - else: - self._object_to_pin = buf + self._object_to_pin = buf - self.thisptr = new schema_cpp.FlatArrayMessageReader(schema_cpp.WordArrayPtr(ptr, sz//8)) + flat_reader = new schema_cpp.FlatArrayMessageReader( + schema_cpp.WordArrayPtr(ptr, sz//8), + opts) + self.thisptr = flat_reader + self.msg_size = flat_reader.getEnd() - ptr + return self def __dealloc__(self): del self.thisptr +@cython.internal +cdef class _FlatArrayMessageReader(_MessageReader): + cdef object _object_to_pin + def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) + cdef _AlignedBuffer aligned + + sz = len(buf) + if sz % 8 != 0: + raise ValueError("input length must be a multiple of eight bytes") + + cdef char * ptr + if isinstance(buf, bytes): + ptr = buf + if (ptr) % 8 != 0: + aligned = _AlignedBuffer(buf) + ptr = aligned.buf + self._object_to_pin = aligned + else: + self._object_to_pin = buf + elif PyObject_CheckBuffer(buf): + view = _BufferView(buf) + ptr = view.buf + self._object_to_pin = view + else: + raise TypeError('expected buffer-like object in FlatArrayMessageReader') + + self.thisptr = new schema_cpp.FlatArrayMessageReader( + schema_cpp.WordArrayPtr(ptr, sz//8), + opts) + + def __dealloc__(self): + del self.thisptr + + +@cython.internal +cdef class _SegmentArrayMessageReader(_MessageReader): + + cdef object _objects_to_pin + cdef schema_cpp.ConstWordArrayPtr* _seg_ptrs + cdef Py_buffer* views + + def __init__(self, segments, traversal_limit_in_words = None, nesting_limit = None): + cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) + # take a Python array of bytes and constructs a ConstWordArrayArrayPtr + num_segments = len(segments) + cdef schema_cpp.ConstWordArrayPtr seg_ptr + self._seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) + self.views = malloc(num_segments * sizeof(Py_buffer)) + self._objects_to_pin = [] + for i in range(0, num_segments): + if PyObject_GetBuffer(segments[i], &self.views[i], PyBUF_SIMPLE) != 0: + raise KjException("could not get read buffer") + + if (self.views[i].buf) % 8 != 0: + aligned = _AlignedBuffer(segments[i]) + self.views[i].buf = aligned.buf + self._objects_to_pin.append(aligned) + else: + self._objects_to_pin.append(segments[i]) + seg_ptr = schema_cpp.ConstWordArrayPtr(self.views[i].buf, self.views[i].len//8) + self._seg_ptrs[i] = seg_ptr + self.thisptr = new schema_cpp.SegmentArrayMessageReader( + schema_cpp.ConstWordArrayArrayPtr(self._seg_ptrs, num_segments), + opts) + + def __dealloc__(self): + free(self._seg_ptrs) + free(self.views) + del self.thisptr + + @cython.internal cdef class _FlatMessageBuilder(_MessageBuilder): cdef object _object_to_pin + cdef Py_buffer view def __init__(self, buf): - cdef void *ptr - cdef Py_ssize_t sz - PyObject_AsWriteBuffer(buf, &ptr, &sz) - if sz % 8 != 0: + if PyObject_GetBuffer(buf, &self.view, PyBUF_WRITABLE) != 0: + raise KjException("expected variable length string object") + if self.view.len % 8 != 0: raise KjException("input length must be a multiple of eight bytes") self._object_to_pin = buf - self.thisptr = new schema_cpp.FlatMessageBuilder(schema_cpp.WordArrayPtr(ptr, sz//8)) + self.thisptr = new schema_cpp.FlatMessageBuilder(schema_cpp.WordArrayPtr(self.view.buf, self.view.len//8)) + + def __dealloc__(self): + PyBuffer_Release(&self.view) def _message_to_packed_bytes(_MessageBuilder message): r, w = _os.pipe() @@ -3683,7 +3887,7 @@ def _write_message_to_fd(int fd, _MessageBuilder message): _write_message_to_fd(f.fileno(), message) ... f = open('out.txt') - _StreamFdMessageReader(f.fileno()) + _StreamFdMessageReader(f) :type fd: int :param fd: A file descriptor @@ -3709,7 +3913,7 @@ def _write_packed_message_to_fd(int fd, _MessageBuilder message): _write_packed_message_to_fd(f.fileno(), message) ... f = open('out.txt') - _PackedFdMessageReader(f.fileno()) + _PackedFdMessageReader(f) :type fd: int :param fd: A file descriptor @@ -3723,6 +3927,13 @@ def _write_packed_message_to_fd(int fd, _MessageBuilder message): _global_schema_parser = None +def cleanup_global_schema_parser(): + """Unloads all of the schema from the current context""" + global _global_schema_parser + if _global_schema_parser: + del _global_schema_parser + _global_schema_parser = None + def load(file_name, display_name=None, imports=[]): """Load a Cap'n Proto schema from a file diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx index df45582..e0ed81c 100644 --- a/capnp/templates/module.pyx +++ b/capnp/templates/module.pyx @@ -1,6 +1,5 @@ # addressbook_fast.pyx # distutils: language = c++ -# distutils: extra_compile_args = --std=c++11 # distutils: include_dirs = {{include_dir}} # distutils: libraries = capnpc capnp capnp-rpc # distutils: sources = {{file.filename}}.cpp @@ -90,7 +89,7 @@ cpdef _set_{{field.name}}(self, value): if type(value) is bytes: temp_string = StringPtr(value, len(value)) else: - encoded_value = value.encode() + encoded_value = value.encode('utf-8') temp_string = StringPtr(encoded_value, len(encoded_value)) self.thisptr_child.set{{field.c_name}}(temp_string) {% elif 'data' == field['type'] -%} @@ -99,7 +98,7 @@ cpdef _set_{{field.name}}(self, value): if type(value) is bytes: temp_string = StringPtr(value, len(value)) else: - encoded_value = value.encode() + encoded_value = value.encode('utf-8') temp_string = StringPtr(encoded_value, len(encoded_value)) self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](temp_string.begin(), temp_string.size())) {% else -%} @@ -133,40 +132,6 @@ cdef _from_list(_DynamicListBuilder msg, list d): msg._set(count, val) count += 1 -cdef DynamicValue.Reader to_dynamic_value(value): - cdef DynamicValue.Reader temp - cdef StringPtr temp_string - value_type = type(value) - - if value_type is int or value_type is long: - if value < 0: - temp = DynamicValue.Reader(value) - else: - temp = DynamicValue.Reader(value) - elif value_type is float: - temp = DynamicValue.Reader(value) - elif value_type is bool: - temp = DynamicValue.Reader(value) - elif value_type is bytes: - temp_string = StringPtr(value, len(value)) - temp = DynamicValue.Reader(temp_string) - elif isinstance(value, basestring): - encoded_value = value.encode() - temp_string = StringPtr(encoded_value, len(encoded_value)) - temp = DynamicValue.Reader(temp_string) - elif value is None: - temp = DynamicValue.Reader(VOID) - elif value_type is _DynamicStructBuilder: - temp = _extract_dynamic_struct_builder(value) - elif value_type is _DynamicStructReader: - temp = _extract_dynamic_struct_reader(value) - elif value_type is _DynamicEnum: - temp = _extract_dynamic_enum(value) - else: - raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value)))) - - return temp - cdef extern from "{{file.filename}}.h": {%- for node in code.nodes %} diff --git a/docs/conf.py b/docs/conf.py index 08eed2d..948267a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,6 @@ +''' +Docs configuration +''' # -*- coding: utf-8 -*- # # capnp documentation build configuration file, created by @@ -11,17 +14,19 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os, string +import string +# import sys, os +import capnp # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. @@ -34,7 +39,7 @@ templates_path = ['_templates'] source_suffix = '.rst' # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' @@ -48,47 +53,46 @@ copyright = u'2013, Author' # built documents. # # The short X.Y version. -import capnp vs = capnp.__version__ # The short X.Y version. -version = vs.rstrip(string.letters) +version = vs.rstrip(string.ascii_letters) # The full version, including alpha/beta/rc tags. release = vs # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- @@ -100,26 +104,26 @@ html_theme = 'nature' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, @@ -128,44 +132,44 @@ html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'capnpdoc' @@ -173,43 +177,43 @@ htmlhelp_basename = 'capnpdoc' # -- Options for LaTeX output -------------------------------------------------- -latex_elements = { # The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', +# 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', +# 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. -#'preamble': '', +# 'preamble': '', +latex_elements = { } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'capnp.tex', u'capnp Documentation', - u'Author', 'manual'), + ('index', 'capnp.tex', u'capnp Documentation', + u'Author', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -222,7 +226,7 @@ man_pages = [ ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -231,19 +235,19 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'capnp', u'capnp Documentation', - u'Author', 'capnp', 'One line description of project.', - 'Miscellaneous'), + ('index', 'capnp', u'capnp Documentation', + u'Author', 'capnp', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # -- Options for Epub output --------------------------------------------------- @@ -256,36 +260,36 @@ epub_copyright = u'2013, Author' # The language of the text. It defaults to the language option # or en if the language is not set. -#epub_language = '' +# epub_language = '' # The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' +# epub_scheme = '' # The unique identifier of the text. This can be a ISBN number # or the project homepage. -#epub_identifier = '' +# epub_identifier = '' # A unique identification for the text. -#epub_uid = '' +# epub_uid = '' # A tuple containing the cover image and cover page html template filenames. -#epub_cover = () +# epub_cover = () # HTML files that should be inserted before the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_pre_files = [] +# epub_pre_files = [] # HTML files shat should be inserted after the pages created by sphinx. # The format is a list of tuples containing the path and title. -#epub_post_files = [] +# epub_post_files = [] # A list of files that should not be packed into the epub file. -#epub_exclude_files = [] +# epub_exclude_files = [] # The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 +# epub_tocdepth = 3 # Allow duplicate toc entries. -#epub_tocdup = True +# epub_tocdup = True intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/quickstart.rst b/docs/quickstart.rst index aa89b89..3b7a1a8 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -304,6 +304,29 @@ There are also packed versions:: alice2 = addressbook_capnp.Person.from_bytes_packed(alice.to_bytes_packed()) + +Byte Segments +~~~~~~~~~~~~~ + +.. note:: This feature is not supported in PyPy at the moment, pending investigation. + +Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way). + +For compatibility on the Python side, use the ``to_segments()`` and ``from_segments()`` functions:: + + segments = alice.to_segments() + +This returns a list of segments, each a byte buffer. Each segment can be, e.g., turned into a ZeroMQ message frame. The list of segments can also be turned back into an object:: + + alice = addressbook_capnp.Person.from_segments(segments) + +For more information, please refer to the following links: + +- `Advice on minimizing copies from Cap'n Proto `_ (from the author of Cap'n Proto) +- `Advice on using Cap'n Proto over ZeroMQ `_ (from the author of Cap'n Proto) +- `Discussion about sending and reassembling Cap'n Proto message segments in C++ `_ (from the Cap'n Proto mailing list; includes sample code) + + RPC ---------- diff --git a/examples/addressbook.py b/examples/addressbook.py index c1ca793..2bd869a 100755 --- a/examples/addressbook.py +++ b/examples/addressbook.py @@ -1,6 +1,7 @@ +#!/usr/bin/env python3 + from __future__ import print_function -import os -import capnp +import capnp # noqa: F401 import addressbook_capnp diff --git a/examples/async_calculator_client.py b/examples/async_calculator_client.py new file mode 100755 index 0000000..90f29f9 --- /dev/null +++ b/examples/async_calculator_client.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 + +from __future__ import print_function +import argparse +import asyncio +import socket +import capnp + +import calculator_capnp + + +class PowerFunction(calculator_capnp.Calculator.Function.Server): + + '''An implementation of the Function interface wrapping pow(). Note that + we're implementing this on the client side and will pass a reference to + the server. The server will then be able to make calls back to the client.''' + + def call(self, params, **kwargs): + '''Note the **kwargs. This is very necessary to include, since + protocols can add parameters over time. Also, by default, a _context + variable is passed to all server methods, but you can also return + results directly as python objects, and they'll be added to the + results struct in the correct order''' + + return pow(params[0], params[1]) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Calculator server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Pass "calculator" to ez_restore (there's also a `restore` function that + # takes a struct or AnyPointer as an argument), and then cast the returned + # capability to it's proper type. This casting is due to capabilities not + # having a reference to their schema + calculator = client.bootstrap().cast_as(calculator_capnp.Calculator) + + '''Make a request that just evaluates the literal value 123. + + What's interesting here is that evaluate() returns a "Value", which is + another interface and therefore points back to an object living on the + server. We then have to call read() on that object to read it. + However, even though we are making two RPC's, this block executes in + *one* network round trip because of promise pipelining: we do not wait + for the first call to complete before we send the second call to the + server.''' + + print('Evaluating a literal... ', end="") + + # Make the request. Note we are using the shorter function form (instead + # of evaluate_request), and we are passing a dictionary that represents a + # struct and its member to evaluate + eval_promise = calculator.evaluate({"literal": 123}) + + # This is equivalent to: + ''' + request = calculator.evaluate_request() + request.expression.literal = 123 + + # Send it, which returns a promise for the result (without blocking). + eval_promise = request.send() + ''' + + # Using the promise, create a pipelined request to call read() on the + # returned object. Note that here we are using the shortened method call + # syntax read(), which is mostly just sugar for read_request().send() + read_promise = eval_promise.value.read() + + # Now that we've sent all the requests, wait for the response. Until this + # point, we haven't waited at all! + response = await read_promise.a_wait() + assert response.value == 123 + + print("PASS") + + '''Make a request to evaluate 123 + 45 - 67. + + The Calculator interface requires that we first call getOperator() to + get the addition and subtraction functions, then call evaluate() to use + them. But, once again, we can get both functions, call evaluate(), and + then read() the result -- four RPCs -- in the time of *one* network + round trip, because of promise pipelining.''' + + print("Using add and subtract... ", end='') + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "subtract" function from the server. + subtract = calculator.getOperator(op='subtract').func + + # Build the request to evaluate 123 + 45 - 67. Note the form is 'evaluate' + # + '_request', where 'evaluate' is the name of the method we want to call + request = calculator.evaluate_request() + subtract_call = request.expression.init('call') + subtract_call.function = subtract + subtract_params = subtract_call.init('params', 2) + subtract_params[1].literal = 67.0 + + add_call = subtract_params[0].init('call') + add_call.function = add + add_params = add_call.init('params', 2) + add_params[0].literal = 123 + add_params[1].literal = 45 + + # Send the evaluate() request, read() the result, and wait for read() to finish. + eval_promise = request.send() + read_promise = eval_promise.value.read() + + response = await read_promise.a_wait() + assert response.value == 101 + + print("PASS") + + ''' + Note: a one liner version of building the previous request (I highly + recommend not doing it this way for such a complicated structure, but I + just wanted to demonstrate it is possible to set all of the fields with a + dictionary): + + eval_promise = calculator.evaluate( +{'call': {'function': subtract, + 'params': [{'call': {'function': add, + 'params': [{'literal': 123}, + {'literal': 45}]}}, + {'literal': 67.0}]}}) + ''' + + '''Make a request to evaluate 4 * 6, then use the result in two more + requests that add 3 and 5. + + Since evaluate() returns its result wrapped in a `Value`, we can pass + that `Value` back to the server in subsequent requests before the first + `evaluate()` has actually returned. Thus, this example again does only + one network round trip.''' + + print("Pipelining eval() calls... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "multiply" function from the server. + multiply = calculator.getOperator(op='multiply').func + + # Build the request to evaluate 4 * 6 + request = calculator.evaluate_request() + + multiply_call = request.expression.init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[0].literal = 4 + multiply_params[1].literal = 6 + + multiply_result = request.send().value + + # Use the result in two calls that add 3 and add 5. + + add_3_request = calculator.evaluate_request() + add_3_call = add_3_request.expression.init("call") + add_3_call.function = add + add_3_params = add_3_call.init("params", 2) + add_3_params[0].previousResult = multiply_result + add_3_params[1].literal = 3 + add_3_promise = add_3_request.send().value.read() + + add_5_request = calculator.evaluate_request() + add_5_call = add_5_request.expression.init("call") + add_5_call.function = add + add_5_params = add_5_call.init("params", 2) + add_5_params[0].previousResult = multiply_result + add_5_params[1].literal = 5 + add_5_promise = add_5_request.send().value.read() + + # Now wait for the results. + assert (await add_3_promise.a_wait()).value == 27 + assert (await add_5_promise.a_wait()).value == 29 + + print("PASS") + + '''Our calculator interface supports defining functions. Here we use it + to define two functions and then make calls to them as follows: + + f(x, y) = x * 100 + y + g(x) = f(x, x + 1) * 2; + f(12, 34) + g(21) + + Once again, the whole thing takes only one network round trip.''' + + print("Defining functions... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + # Get the "multiply" function from the server. + multiply = calculator.getOperator(op='multiply').func + + # Define f. + request = calculator.defFunction_request() + request.paramCount = 2 + + # Build the function body. + add_call = request.body.init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[1].parameter = 1 # y + + multiply_call = add_params[0].init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[0].parameter = 0 # x + multiply_params[1].literal = 100 + + f = request.send().func + + # Define g. + request = calculator.defFunction_request() + request.paramCount = 1 + + # Build the function body. + multiply_call = request.body.init("call") + multiply_call.function = multiply + multiply_params = multiply_call.init("params", 2) + multiply_params[1].literal = 2 + + f_call = multiply_params[0].init("call") + f_call.function = f + f_params = f_call.init("params", 2) + f_params[0].parameter = 0 + + add_call = f_params[1].init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[0].parameter = 0 + add_params[1].literal = 1 + + g = request.send().func + + # OK, we've defined all our functions. Now create our eval requests. + + # f(12, 34) + f_eval_request = calculator.evaluate_request() + f_call = f_eval_request.expression.init("call") + f_call.function = f + f_params = f_call.init("params", 2) + f_params[0].literal = 12 + f_params[1].literal = 34 + f_eval_promise = f_eval_request.send().value.read() + + # g(21) + g_eval_request = calculator.evaluate_request() + g_call = g_eval_request.expression.init("call") + g_call.function = g + g_call.init('params', 1)[0].literal = 21 + g_eval_promise = g_eval_request.send().value.read() + + # Wait for the results. + assert (await f_eval_promise.a_wait()).value == 1234 + assert (await g_eval_promise.a_wait()).value == 4244 + + print("PASS") + + '''Make a request that will call back to a function defined locally. + + Specifically, we will compute 2^(4 + 5). However, exponent is not + defined by the Calculator server. So, we'll implement the Function + interface locally and pass it to the server for it to use when + evaluating the expression. + + This example requires two network round trips to complete, because the + server calls back to the client once before finishing. In this + particular case, this could potentially be optimized by using a tail + call on the server side -- see CallContext::tailCall(). However, to + keep the example simpler, we haven't implemented this optimization in + the sample server.''' + + print("Using a callback... ", end="") + + # Get the "add" function from the server. + add = calculator.getOperator(op='add').func + + # Build the eval request for 2^(4+5). + request = calculator.evaluate_request() + + pow_call = request.expression.init("call") + pow_call.function = PowerFunction() + pow_params = pow_call.init("params", 2) + pow_params[0].literal = 2 + + add_call = pow_params[1].init("call") + add_call.function = add + add_params = add_call.init("params", 2) + add_params[0].literal = 4 + add_params[1].literal = 5 + + # Send the request and wait. + response = await request.send().value.read().a_wait() + assert response.value == 512 + + print("PASS") + +if __name__ == '__main__': + asyncio.run(main(parse_args().host)) diff --git a/examples/async_calculator_server.py b/examples/async_calculator_server.py new file mode 100755 index 0000000..8324902 --- /dev/null +++ b/examples/async_calculator_server.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 + +from __future__ import print_function +import argparse +import asyncio +import socket +import capnp + +import calculator_capnp + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + await client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +def read_value(value): + '''Helper function to asynchronously call read() on a Calculator::Value and + return a promise for the result. (In the future, the generated code might + include something like this automatically.)''' + + return value.read().then(lambda result: result.value) + + +def evaluate_impl(expression, params=None): + '''Implementation of CalculatorImpl::evaluate(), also shared by + FunctionImpl::call(). In the latter case, `params` are the parameter + values passed to the function; in the former case, `params` is just an + empty list.''' + + which = expression.which() + + if which == 'literal': + return capnp.Promise(expression.literal) + elif which == 'previousResult': + return read_value(expression.previousResult) + elif which == 'parameter': + assert expression.parameter < len(params) + return capnp.Promise(params[expression.parameter]) + elif which == 'call': + call = expression.call + func = call.function + + # Evaluate each parameter. + paramPromises = [evaluate_impl(param, params) for param in call.params] + + joinedParams = capnp.join_promises(paramPromises) + # When the parameters are complete, call the function. + ret = (joinedParams + .then(lambda vals: func.call(vals)) + .then(lambda result: result.value)) + + return ret + else: + raise ValueError("Unknown expression type: " + which) + + +class ValueImpl(calculator_capnp.Calculator.Value.Server): + + "Simple implementation of the Calculator.Value Cap'n Proto interface." + + def __init__(self, value): + self.value = value + + def read(self, **kwargs): + return self.value + + +class FunctionImpl(calculator_capnp.Calculator.Function.Server): + + '''Implementation of the Calculator.Function Cap'n Proto interface, where the + function is defined by a Calculator.Expression.''' + + def __init__(self, paramCount, body): + self.paramCount = paramCount + self.body = body.as_builder() + + def call(self, params, _context, **kwargs): + '''Note that we're returning a Promise object here, and bypassing the + helper functionality that normally sets the results struct from the + returned object. Instead, we set _context.results directly inside of + another promise''' + + assert len(params) == self.paramCount + # using setattr because '=' is not allowed inside of lambdas + return evaluate_impl(self.body, params).then(lambda value: setattr(_context.results, 'value', value)) + + +class OperatorImpl(calculator_capnp.Calculator.Function.Server): + + '''Implementation of the Calculator.Function Cap'n Proto interface, wrapping + basic binary arithmetic operators.''' + + def __init__(self, op): + self.op = op + + def call(self, params, **kwargs): + assert len(params) == 2 + + op = self.op + + if op == 'add': + return params[0] + params[1] + elif op == 'subtract': + return params[0] - params[1] + elif op == 'multiply': + return params[0] * params[1] + elif op == 'divide': + return params[0] / params[1] + else: + raise ValueError('Unknown operator') + + +class CalculatorImpl(calculator_capnp.Calculator.Server): + + "Implementation of the Calculator Cap'n Proto interface." + + def evaluate(self, expression, _context, **kwargs): + return evaluate_impl(expression).then(lambda value: setattr(_context.results, 'value', ValueImpl(value))) + + def defFunction(self, paramCount, body, _context, **kwargs): + return FunctionImpl(paramCount, body) + + def getOperator(self, op, **kwargs): + return OperatorImpl(op) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=CalculatorImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ) + except Exception: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + family=socket.AF_INET6 + ) + + async with server: + await server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/async_client.py b/examples/async_client.py new file mode 100755 index 0000000..63c3e42 --- /dev/null +++ b/examples/async_client.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import asyncio +import argparse +import time +import capnp +import socket + +import thread_capnp + +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + +if __name__ == '__main__': + asyncio.run(main(parse_args().host)) diff --git a/examples/async_reconnecting_ssl_client.py b/examples/async_reconnecting_ssl_client.py new file mode 100755 index 0000000..a4967bf --- /dev/null +++ b/examples/async_reconnecting_ssl_client.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import asyncio +import argparse +import os +import time +import socket +import ssl + +import capnp + +import thread_capnp + +this_dir = os.path.dirname(os.path.abspath(__file__)) +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + try: + # Must be a wait_for in order to give watch_connection a slot + # to try again + data = await asyncio.wait_for(reader.read(4096), timeout=1.0) + except asyncio.TimeoutError: + continue + client.write(data) + + +async def mywriter(client, writer): + while True: + try: + # Must be a wait_for in order to give watch_connection a slot + # to try again + data = await asyncio.wait_for(client.read(4096), timeout=1.0) + writer.write(data.tobytes()) + except asyncio.TimeoutError: + continue + + +async def watch_connection(cap): + while True: + try: + await asyncio.wait_for(cap.alive().a_wait(), timeout=5) + await asyncio.sleep(1) + except asyncio.TimeoutError: + print("Watch timeout!") + asyncio.get_running_loop().stop() + return False + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert')) + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + ) + except OSError: + print("Try IPv6") + try: + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + family=socket.AF_INET6 + ) + except OSError: + return False + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Start watcher to restart socket connection if it is lost + overalltasks = [] + watcher = [watch_connection(cap)] + overalltasks.append(asyncio.gather(*watcher, return_exceptions=True)) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + overalltasks.append(asyncio.gather(*coroutines, return_exceptions=True)) + + # Start background task for subscriber + tasks = [background(cap)] + overalltasks.append(asyncio.gather(*tasks, return_exceptions=True)) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + + for task in overalltasks: + task.cancel() + + return True + +if __name__ == '__main__': + # Using asyncio.run hits an asyncio ssl bug + # https://bugs.python.org/issue36709 + # asyncio.run(main(parse_args().host), loop=loop, debug=True) + retry = True + while retry: + loop = asyncio.new_event_loop() + try: + retry = not loop.run_until_complete(main(parse_args().host)) + except RuntimeError: + # If an IO is hung, the event loop will be stopped + # and will throw RuntimeError exception + continue + if retry: + time.sleep(1) + print("Retrying...") + +# How this works +# - There are two retry mechanisms +# 1. Connection retry +# 2. alive RPC verification +# - The connection retry just loops the connection (IPv4+IPv6 until there is a connection or Ctrl+C) +# - The alive RPC verification attempts a very basic rpc call with a timeout +# * If there is a timeout, stop the current event loop +# * Use the RuntimeError exception to force a reconnect +# * myreader and mywriter must also be wrapped in wait_for in order for the events to get triggered correctly diff --git a/examples/async_server.py b/examples/async_server.py new file mode 100755 index 0000000..e6fa225 --- /dev/null +++ b/examples/async_server.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import argparse +import capnp + +import thread_capnp +import asyncio +import socket + + +class ExampleImpl(thread_capnp.Example.Server): + + "Implementation of the Example threading Cap'n Proto interface." + + def subscribeStatus(self, subscriber, **kwargs): + return capnp.getTimer().after_delay(10**9) \ + .then(lambda: subscriber.status(True)) \ + .then(lambda _: self.subscribeStatus(subscriber)) + + def longRunning(self, **kwargs): + return capnp.getTimer().after_delay(1 * 10**9) + + +async def myreader(server, reader): + while True: + data = await reader.read(4096) + # Close connection if 0 bytes read + if len(data) == 0: + server.close() + await server.write(data) + + +async def mywriter(server, writer): + while True: + data = await server.read(4096) + writer.write(data.tobytes()) + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=ExampleImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ) + except Exception: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + family=socket.AF_INET6 + ) + + async with server: + await server.serve_forever() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py new file mode 100755 index 0000000..802044e --- /dev/null +++ b/examples/async_ssl_client.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import asyncio +import argparse +import os +import time +import capnp +import socket +import ssl + +import thread_capnp + +this_dir = os.path.dirname(os.path.abspath(__file__)) +capnp.remove_event_loop() +capnp.create_event_loop(threaded=True) + + +def parse_args(): + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ +at the given address and does some RPCs') + parser.add_argument("host", help="HOST:PORT") + + return parser.parse_args() + + +class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): + '''An implementation of the StatusSubscriber interface''' + + def status(self, value, **kwargs): + print('status: {}'.format(time.time())) + + +async def myreader(client, reader): + while True: + data = await reader.read(4096) + client.write(data) + + +async def mywriter(client, writer): + while True: + data = await client.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def background(cap): + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + await promise.a_wait() + + +async def main(host): + host = host.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert')) + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + ) + except Exception: + print("Try IPv6") + reader, writer = await asyncio.open_connection( + addr, port, + ssl=ctx, + family=socket.AF_INET6 + ) + + # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode) + client = capnp.TwoPartyClient() + cap = client.bootstrap().cast_as(thread_capnp.Example) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(client, reader), mywriter(client, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + # Start background task for subscriber + tasks = [background(cap)] + asyncio.gather(*tasks, return_exceptions=True) + + # Run blocking tasks + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + await cap.longRunning().a_wait() + print('main: {}'.format(time.time())) + +if __name__ == '__main__': + # Using asyncio.run hits an asyncio ssl bug + # https://bugs.python.org/issue36709 + # asyncio.run(main(parse_args().host), loop=loop, debug=True) + loop = asyncio.get_event_loop() + loop.run_until_complete(main(parse_args().host)) diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py new file mode 100755 index 0000000..06a7d28 --- /dev/null +++ b/examples/async_ssl_server.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +from __future__ import print_function + +import argparse +import os +import capnp + +import thread_capnp +import asyncio +import socket +import ssl + + +this_dir = os.path.dirname(os.path.abspath(__file__)) + + +class ExampleImpl(thread_capnp.Example.Server): + + "Implementation of the Example threading Cap'n Proto interface." + + def subscribeStatus(self, subscriber, **kwargs): + return capnp.getTimer().after_delay(10**9) \ + .then(lambda: subscriber.status(True)) \ + .then(lambda _: self.subscribeStatus(subscriber)) + + def longRunning(self, **kwargs): + return capnp.getTimer().after_delay(1 * 10**9) + + def alive(self, **kwargs): + return True + + +async def myreader(server, reader): + while True: + data = await reader.read(4096) + # Close connection if 0 bytes read + if len(data) == 0: + server.close() + await server.write(data) + + +async def mywriter(server, writer): + while True: + data = await server.read(4096) + writer.write(data.tobytes()) + await writer.drain() + + +async def myserver(reader, writer): + # Start TwoPartyServer using TwoWayPipe (only requires bootstrap) + server = capnp.TwoPartyServer(bootstrap=ExampleImpl()) + + # Assemble reader and writer tasks, run in the background + coroutines = [myreader(server, reader), mywriter(server, writer)] + asyncio.gather(*coroutines, return_exceptions=True) + + await server.poll_forever() + + +def parse_args(): + parser = argparse.ArgumentParser(usage='''Runs the server bound to the\ +given address/port ADDRESS. ''') + + parser.add_argument("address", help="ADDRESS:PORT") + + return parser.parse_args() + + +async def main(): + address = parse_args().address + host = address.split(':') + addr = host[0] + port = host[1] + + # Setup SSL context + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + ctx.load_cert_chain(os.path.join(this_dir, 'selfsigned.cert'), os.path.join(this_dir, 'selfsigned.key')) + + # Handle both IPv4 and IPv6 cases + try: + print("Try IPv4") + server = await asyncio.start_server( + myserver, + addr, port, + ssl=ctx, + ) + except Exception: + print("Try IPv6") + server = await asyncio.start_server( + myserver, + addr, port, + ssl=ctx, + family=socket.AF_INET6, + ) + + async with server: + await server.serve_forever() + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/examples/calculator_client.py b/examples/calculator_client.py index de246c3..85694da 100755 --- a/examples/calculator_client.py +++ b/examples/calculator_client.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import argparse -import socket import capnp import calculator_capnp @@ -302,5 +301,6 @@ def main(host): print("PASS") + if __name__ == '__main__': main(parse_args().host) diff --git a/examples/calculator_server.py b/examples/calculator_server.py index f072f58..38ca7ef 100755 --- a/examples/calculator_server.py +++ b/examples/calculator_server.py @@ -1,9 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function import argparse -import socket -import random import capnp import calculator_capnp @@ -136,5 +134,6 @@ def main(): server = capnp.TwoPartyServer(address, bootstrap=CalculatorImpl()) server.run_forever() + if __name__ == '__main__': main() diff --git a/examples/selfsigned.cert b/examples/selfsigned.cert new file mode 100644 index 0000000..399026c --- /dev/null +++ b/examples/selfsigned.cert @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE----- +MIICpDCCAYwCCQDR+CRWUUUdKDANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwHhcNMTkwOTE3MDczODI2WhcNNDcwMjAyMDczODI2WjAUMRIwEAYD +VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg ++X+utjujNcq/zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXs +czvUTWDoPhvvbxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9 +UNKzf4VQVoa0n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapL +GO2kiPzp36388n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjS +Fcml6J06CITpYeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEON +vWZiM7Ul+9mOPejUMPwlAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAICkFOw0Dd1U +r60rgVpUiHMoFNuBP3ikZfBQ+KXOtfTIYbxbi+iKvvPlB1NFA7Qy3FqqN4sUncvp +QQLxdm+KClM+hvAogng/SyEJJW169vuQbqn5s/1iKCOtGFkI18thCr3rwsI6vTaR +0TmTPtjSQKl5PqcS8kQJTED+CnQhqOAv7C68Bpg+x2dSD9VCq81cPeDbfnK6gico +29qJYUm4RCXMicrzvEwNObx06TQKJb/pWjpl1NAmpFvcz+2MYPL/QTfH/cS5lhgx +KCe4/kDO0HCueOi2MqBFaO2B0kZxanMoZ2KZe2b/Bp1CTJzXCXNYWQj6QbLzZxaD +fOp0J8wAo1U= +-----END CERTIFICATE----- diff --git a/examples/selfsigned.key b/examples/selfsigned.key new file mode 100644 index 0000000..296d799 --- /dev/null +++ b/examples/selfsigned.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCg+X+utjujNcq/ +zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXsczvUTWDoPhvv +bxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9UNKzf4VQVoa0 +n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapLGO2kiPzp3638 +8n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjSFcml6J06CITp +YeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEONvWZiM7Ul+9mO +PejUMPwlAgMBAAECggEAD6Vwlai8zzZRSKc7Vf98LI3dDRkRVS3XLf/uSluNlo7e +o9Iyz8fOypA8GT2NwGKNyvfAXvhObQRsbq9bvXhvhJLRKde2m5x7vovZ3mztOj63 +f/kwHSjC5hksgjxC8NFtGBadBDlm2dIvMasxk7bbr4tn36orbr0NPGMTm0C/Md8B +bSUzuc/mT+6KWfW9g4svqebSbKvC7tGuAu3/RfL1cmbuuvtJJA+EPfRjgCtOiFSk +8NLE0KLUYySf6M3MAHMeSwQhVr1xyYUkiOqQoxMC7CpaplNqaB2rrOe2nEjLpXgx +80WLFbB22HNEkBSgX5zz4FmrLchwaI79f3PiGQ2DWQKBgQDSIsl1t13eR8ubsGU5 +Z097U8/eylyXJC+ZU1/TbWgaPNHlMf26EncSJwlmq7pNJ0Rk7p/edPMWKu9tOgQE +iG0QeKxvxcbc1LzVmfLKXY8hW4DOhiBTiRNNb+YWuYmKYnHXM8hrKthNXBFfRrwb +Pb+mid9FcK4GcIDkWbgXEahmwwKBgQDEG9sB7Ee4fDiQi1oVsPkMwxnxsiNpSRRG +9CmG/xIvL81vSaONPmg1f5q5/Wqkd6s3QmMs4rE5U8+kgRvuw/MnN/UxQBH5/LAn +T3hbo7qlIONOOpgQswg7wIQnKP31spNU2FthA1ACazRBsIyH+hPmgeqvtvFhL4Qy +6q5tfhFy9wKBgGYZtu82aCqPkdOU0qogk1Ll9zNV+cUKNQJ3qzDMkO9mq8mED7cw +L6CnTP8Q45WHRckQ1Ka/Bjm4JNtafAdDzlJZf9dTLnuv9gyHH5vJ97iKgDxYmS5d +hP50J0TVY4nUqWGZ7IB9sdlsqZg0g0NtLkiZ5t0TkcrZMRdCrJqw3rUHAoGBAJNk +wEmEti8Rpk31fsK43aba6KABHJ5gX84oayHcimVOz1/qf/ODyT0UaE2MC2AL1XLW +AcZVp5AHzxO8OitNuW5rn2zh0+EJK7iQAU0XFQxRWKaOYYaDmReXzXvFUoMdMaDe +cGfM3pDC1Gbe8/CrY9OnJ6XjoS5DUWAXhPwkeabnAoGBAMmI9xcyfcPjO6XKKymZ +z+6bwvFaey9Acy5vkxLrEcRqH8pRR09CltWzR4vhpJwHABWPHVGqdtvlLw1/lUrj +xr4RXvYXK28cK9Tdak0c3+HPSsozxrsX6AQzcG27ymo5s0KYaVuxWh98iZU7Eb52 +YfCx3eOSIHPH0ay7KBVAQocG +-----END PRIVATE KEY----- diff --git a/examples/thread.capnp b/examples/thread.capnp index ae32b8d..8caf56f 100644 --- a/examples/thread.capnp +++ b/examples/thread.capnp @@ -8,4 +8,5 @@ interface Example { longRunning @0 () -> (value: Bool); subscribeStatus @1 (subscriber: StatusSubscriber); + alive @2 () -> (value: Bool); } diff --git a/examples/thread_client.py b/examples/thread_client.py index 7201dd0..2b23f2d 100755 --- a/examples/thread_client.py +++ b/examples/thread_client.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function @@ -14,11 +14,11 @@ capnp.create_event_loop(threaded=True) def parse_args(): - parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ + parser = argparse.ArgumentParser(usage='Connects to the Example thread server \ at the given address and does some RPCs') - parser.add_argument("host", help="HOST:PORT") + parser.add_argument("host", help="HOST:PORT") - return parser.parse_args() + return parser.parse_args() class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): @@ -30,29 +30,30 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): def start_status_thread(host): - client = capnp.TwoPartyClient(host) - cap = client.ez_restore('example').cast_as(thread_capnp.Example) + client = capnp.TwoPartyClient(host) + cap = client.bootstrap().cast_as(thread_capnp.Example) - subscriber = StatusSubscriber() - promise = cap.subscribeStatus(subscriber) - promise.wait() + subscriber = StatusSubscriber() + promise = cap.subscribeStatus(subscriber) + promise.wait() def main(host): - client = capnp.TwoPartyClient(host) - cap = client.ez_restore('example').cast_as(thread_capnp.Example) + client = capnp.TwoPartyClient(host) + cap = client.bootstrap().cast_as(thread_capnp.Example) - status_thread = threading.Thread(target=start_status_thread, args=(host,)) - status_thread.daemon = True - status_thread.start() + status_thread = threading.Thread(target=start_status_thread, args=(host,)) + status_thread.daemon = True + status_thread.start() + + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) + cap.longRunning().wait() + print('main: {}'.format(time.time())) - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) - cap.longRunning().wait() - print('main: {}'.format(time.time())) if __name__ == '__main__': main(parse_args().host) diff --git a/examples/thread_server.py b/examples/thread_server.py index 04b1e2d..6a2b52f 100755 --- a/examples/thread_server.py +++ b/examples/thread_server.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from __future__ import print_function @@ -18,7 +18,7 @@ class ExampleImpl(thread_capnp.Example.Server): .then(lambda _: self.subscribeStatus(subscriber)) def longRunning(self, **kwargs): - return capnp.getTimer().after_delay(3 * 10**9) + return capnp.getTimer().after_delay(1 * 10**9) def parse_args(): @@ -31,19 +31,12 @@ given address/port ADDRESS may be '*' to bind to all local addresses.\ return parser.parse_args() -impl = ExampleImpl() - - -def restore(ref): - assert ref.as_text() == 'example' - return impl - - def main(): address = parse_args().address - server = capnp.TwoPartyServer(address, restore) + server = capnp.TwoPartyServer(address, bootstrap=ExampleImpl()) server.run_forever() + if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt index 8712fde..99f2720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -jinja2 >= 2.7.3 -cython == 0.21.2 -setuptools >= 0.8 +jinja2 +cython +setuptools pytest tox diff --git a/scripts/capnp-json.py b/scripts/capnp-json.py index f41a05e..cc385d7 100755 --- a/scripts/capnp-json.py +++ b/scripts/capnp-json.py @@ -18,7 +18,7 @@ def encode(schema_file, struct_name, **kwargs): schema = capnp.load(schema_file) struct_schema = getattr(schema, struct_name) - + struct_dict = json.load(sys.stdin) struct = struct_schema.from_dict(struct_dict) @@ -29,7 +29,7 @@ def decode(schema_file, struct_name, defaults): struct_schema = getattr(schema, struct_name) struct = struct_schema.read(sys.stdin) - + json.dump(struct.to_dict(defaults), sys.stdout) def main(): @@ -41,4 +41,5 @@ def main(): globals()[command](**kwargs) # hacky way to get defined functions, and call function with name=command -main() \ No newline at end of file + +main() diff --git a/scripts/capnp_test_pycapnp.py b/scripts/capnp_test_pycapnp.py index 5a02600..7f6b046 100755 --- a/scripts/capnp_test_pycapnp.py +++ b/scripts/capnp_test_pycapnp.py @@ -1,12 +1,13 @@ #!/usr/bin/env python from __future__ import print_function -import capnp import os +import sys + +import capnp capnp.add_import_hook([os.getcwd(), "/usr/local/include/"]) # change this to be auto-detected? -import test_capnp +import test_capnp # noqa: E402 -import sys def decode(name): class_name = name[0].upper() + name[1:] @@ -18,6 +19,7 @@ def encode(name): message = getattr(test_capnp, class_name).from_dict(val.to_dict()) print(message.to_bytes()) + if sys.argv[1] == 'decode': decode(sys.argv[2]) else: diff --git a/setup.py b/setup.py index d97a4f4..db57b62 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,41 @@ #!/usr/bin/env python +''' +pycapnp-async distutils setup.py +''' + from __future__ import print_function -use_cython = False - -from distutils.core import setup import os +import struct import sys -from buildutils import test_build, fetch_libcapnp, build_libcapnp, info + +from distutils.command.clean import clean as _clean from distutils.errors import CompileError from distutils.extension import Extension +from distutils.spawn import find_executable + +from setuptools import setup, find_packages, Extension + +from buildutils import test_build, fetch_libcapnp, build_libcapnp, info _this_dir = os.path.dirname(__file__) MAJOR = 0 -MINOR = 5 -MICRO = 6 +MINOR = 7 +MICRO = 0 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) # Write version info def write_version_py(filename=None): + ''' + Generate pycapnp-async version + ''' cnt = """\ version = '%s' short_version = '%s' +# flake8: noqa E402 F401 from .lib.capnp import _CAPNP_VERSION_MAJOR as LIBCAPNP_VERSION_MAJOR from .lib.capnp import _CAPNP_VERSION_MINOR as LIBCAPNP_VERSION_MINOR from .lib.capnp import _CAPNP_VERSION_MICRO as LIBCAPNP_VERSION_MICRO @@ -39,21 +51,25 @@ from .lib.capnp import _CAPNP_VERSION as LIBCAPNP_VERSION finally: a.close() + write_version_py() # Try to convert README using pandoc try: import pypandoc - long_description = pypandoc.convert('README.md', 'rst') - changelog = pypandoc.convert('CHANGELOG.md', 'rst') + long_description = pypandoc.convert_file('README.md', 'rst') + changelog = pypandoc.convert_file('CHANGELOG.md', 'rst') changelog = '\nChangelog\n=============\n' + changelog long_description += changelog except (IOError, ImportError): + if sys.argv and sys.argv[-1] == 'sdist': + raise long_description = '' -# Clean command, invoked with `python setup.py clean` -from distutils.command.clean import clean as _clean class clean(_clean): + ''' + Clean command, invoked with `python setup.py clean` + ''' def run(self): _clean.run(self) for x in [ 'capnp/lib/capnp.cpp', 'capnp/lib/capnp.h', 'capnp/version.py' ]: @@ -63,10 +79,6 @@ class clean(_clean): except OSError: pass -# set use_cython if lib/capnp.cpp is not detected -capnp_compiled_file = os.path.join(os.path.dirname(__file__), 'capnp', 'lib', 'capnp.cpp') -if not os.path.isfile(capnp_compiled_file): - use_cython = True # hack to parse commandline arguments force_bundled_libcapnp = "--force-bundled-libcapnp" in sys.argv @@ -78,62 +90,103 @@ if force_system_libcapnp: force_cython = "--force-cython" in sys.argv if force_cython: sys.argv.remove("--force-cython") - use_cython = True + # Always use cython, ignoring option +libcapnp_url = None +try: + libcapnp_url_index = sys.argv.index("--libcapnp-url") + libcapnp_url = sys.argv[libcapnp_url_index + 1] + sys.argv.remove("--libcapnp-url") + sys.argv.remove(libcapnp_url) +except Exception: + pass -if use_cython: - from Cython.Distutils import build_ext as build_ext_c -else: - from distutils.command.build_ext import build_ext as build_ext_c +from Cython.Distutils import build_ext as build_ext_c class build_libcapnp_ext(build_ext_c): + ''' + Build capnproto library + ''' def build_extension(self, ext): build_ext_c.build_extension(self, ext) def run(self): - build_failed = False - try: - test_build() - except CompileError: - build_failed = True + if force_bundled_libcapnp: + need_build = True + elif force_system_libcapnp: + need_build = False + else: + # Try to use capnp executable to find include and lib path + capnp_executable = find_executable("capnp") + if capnp_executable: + self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')] + self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')] - if build_failed and force_system_libcapnp: - raise RuntimeError("libcapnp C++ library not detected and --force-system-libcapnp was used") - if build_failed or force_bundled_libcapnp: - if build_failed: - info("*WARNING* no libcapnp detected. Will download and build it from source now. If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. Downloading and building libcapnp may take a while.") + # Try to autodetect presence of library. Requires compile/run + # step so only works for host (non-cross) compliation + try: + test_build(include_dirs=self.include_dirs, library_dirs=self.library_dirs) + need_build = False + except CompileError: + need_build = True + + if need_build: + info( + "*WARNING* no libcapnp detected or rebuild forced. " + "Will download and build it from source now. " + "If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. " + "Downloading and building libcapnp may take a while." + ) bundle_dir = os.path.join(_this_dir, "bundled") if not os.path.exists(bundle_dir): os.mkdir(bundle_dir) - build_dir = os.path.join(_this_dir, "build") + build_dir = os.path.join(_this_dir, "build{}".format(8 * struct.calcsize("P"))) if not os.path.exists(build_dir): os.mkdir(build_dir) - fetch_libcapnp(bundle_dir) - build_libcapnp(bundle_dir, build_dir) + # Check if we've already built capnproto + capnp_bin = os.path.join(build_dir, 'bin', 'capnp') + if os.name == 'nt': + capnp_bin = os.path.join(build_dir, 'bin', 'capnp.exe') + + if not os.path.exists(capnp_bin): + # Not built, fetch and build + fetch_libcapnp(bundle_dir, libcapnp_url) + build_libcapnp(bundle_dir, build_dir) + else: + info("capnproto already built at {}".format(build_dir)) self.include_dirs += [os.path.join(build_dir, 'include')] self.library_dirs += [os.path.join(build_dir, 'lib')] return build_ext_c.run(self) -if use_cython: - from Cython.Build import cythonize - import Cython - extensions = cythonize('capnp/lib/*.pyx') -else: - extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"], - include_dirs=["."], - language='c++', - extra_compile_args=['--std=c++11'], - libraries=['capnpc', 'capnp-rpc', 'capnp', 'kj-async', 'kj'])] +extra_compile_args = ['--std=c++14'] +extra_link_args = [] +if os.name == 'nt': + extra_compile_args = ['/std:c++14', '/MD'] + extra_link_args = ['/MANIFEST'] + +import Cython.Build +import Cython # noqa: F401 +extensions = [Extension( + '*', ['capnp/lib/*.pyx'], + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + language='c++', +)] setup( - name="pycapnp", + name="pycapnp-async", packages=["capnp"], version=VERSION, - package_data={'capnp': ['*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*']}, - ext_modules=extensions, - cmdclass = { + package_data={ + 'capnp': [ + '*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', + 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*' + ] + }, + ext_modules=Cython.Build.cythonize(extensions), + cmdclass={ 'clean': clean, 'build_ext': build_libcapnp_ext }, @@ -145,12 +198,12 @@ setup( description="A cython wrapping of the C++ Cap'n Proto library", long_description=long_description, license='BSD', - author="Jason Paryani", - author_email="pypi-contact@jparyani.com", - url = 'https://github.com/jparyani/pycapnp', - download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, - keywords = ['capnp', 'capnproto', "Cap'n Proto"], - classifiers = [ + author="Jacob Alexander", + author_email="haata@kiibohd.com", + url='https://github.com/haata/pycapnp-async', + download_url='https://github.com/haata/pycapnp-async/archive/v%s.zip' % VERSION, + keywords=['capnp', 'capnproto', "Cap'n Proto"], + classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', @@ -158,12 +211,7 @@ setup( 'Operating System :: POSIX', 'Programming Language :: C++', 'Programming Language :: Cython', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Communications'], ) diff --git a/test/addressbook.capnp b/test/addressbook.capnp index b50b68b..576396c 100644 --- a/test/addressbook.capnp +++ b/test/addressbook.capnp @@ -37,3 +37,6 @@ struct AddressBook { people @0 :List(Person); } +struct NestedList { + list @0 :List(List(Int32)); +} diff --git a/test/all-types.binary b/test/all-types.binary index ea39763..3381caa 100644 Binary files a/test/all-types.binary and b/test/all-types.binary differ diff --git a/test/all-types.packed b/test/all-types.packed index 8627833..acb6086 100644 Binary files a/test/all-types.packed and b/test/all-types.packed differ diff --git a/test/all-types.txt b/test/all-types.txt index 079ff8d..a85df4c 100644 --- a/test/all-types.txt +++ b/test/all-types.txt @@ -25,7 +25,7 @@ uInt64Field = 345678901234567890, float32Field = -1.25e-10, float64Field = 345, - textField = "baz", + textField = "☃", dataField = "qux", structField = ( voidField = void, diff --git a/test/test_capability.capnp b/test/test_capability.capnp index 87a2e17..6770a3a 100644 --- a/test/test_capability.capnp +++ b/test/test_capability.capnp @@ -76,4 +76,20 @@ interface TestTailCallee { interface TestTailCaller { foo @0 (i :Int32, callee :TestTailCallee) -> TestTailCallee.TailResult; -} \ No newline at end of file +} + +interface TestPassedCap { + foo @0 (cap :TestInterface) -> (x: Text); +} + +interface TestStructArg { + bar @0 BarParams -> (c: Text); +} +struct BarParams { + a @0 :Text; + b @1 :Int32; +} + +interface TestGeneric(MyObject) { + foo @0 (a :MyObject) -> (b: Text); +} diff --git a/test/test_capability.py b/test/test_capability.py index 7a21739..5ec1fdb 100644 --- a/test/test_capability.py +++ b/test/test_capability.py @@ -1,8 +1,7 @@ import pytest -import capnp -import os import time +import capnp import test_capability_capnp as capability class Server(capability.TestInterface.Server): @@ -160,6 +159,7 @@ class BadPipelineServer(capability.TestPipeline.Server): _results = _context.results _results.s = response.x + '_foo' _results.outBox.cap = Server(100) + def _error(error): raise Exception('test was a success') @@ -186,7 +186,7 @@ def test_pipeline_exception(): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -194,7 +194,7 @@ def test_pipeline_exception(): def test_casting(): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) @@ -343,3 +343,43 @@ def test_inheritance(): response = remote.wait() assert response.x == '26' + + +class TestPassedCap(capability.TestPassedCap.Server): + def foo(self, cap, _context, **kwargs): + def set_result(res): + _context.results.x = res.x + return cap.foo(5).then(set_result) + + +def test_null_cap(): + client = capability.TestPassedCap._new_client(TestPassedCap()) + assert client.foo(Server()).wait().x == '26' + + with pytest.raises(capnp.KjException): + client.foo().wait() + + +class TestStructArg(capability.TestStructArg.Server): + def bar(self, a, b, **kwargs): + return a + str(b) + + +def test_struct_args(): + client = capability.TestStructArg._new_client(TestStructArg()) + assert client.bar(a='test', b=1).wait().c == 'test1' + with pytest.raises(capnp.KjException): + assert client.bar('test', 1).wait().c == 'test1' + + +class TestGeneric(capability.TestGeneric.Server): + def foo(self, a, **kwargs): + return a.as_text() + 'test' + + +def test_generic(): + client = capability.TestGeneric._new_client(TestGeneric()) + + obj = capnp._MallocMessageBuilder().get_root_as_any() + obj.set_as_text("anypointer_") + assert client.foo(obj).wait().b == 'anypointer_test' diff --git a/test/test_capability_context.py b/test/test_capability_context.py index 4f2c111..e4d6fb6 100644 --- a/test/test_capability_context.py +++ b/test/test_capability_context.py @@ -1,12 +1,16 @@ -import pytest -import capnp import os +import pytest + +import capnp this_dir = os.path.dirname(__file__) +# flake8: noqa: E501 + @pytest.fixture def capability(): - return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) + capnp.cleanup_global_schema_parser() + return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) class Server: def __init__(self, val=1): @@ -115,7 +119,15 @@ def test_simple_client_context(capability): with pytest.raises(Exception): remote = client.foo(baz=5) +@pytest.mark.xfail def test_pipeline_context(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f87c1ac6e40 7f87c17c3250 7f87c17be260 7f87c17c49f0 7f87c17c0f50 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409 + ''' client = capability.TestPipeline._new_client(PipelineServer()) foo_client = capability.TestInterface._new_client(Server()) @@ -150,6 +162,7 @@ class BadPipelineServer: def _then(response): context.results.s = response.x + '_foo' context.results.outBox.cap = capability().TestInterface._new_server(Server(100)) + def _error(error): raise Exception('test was a success') @@ -176,7 +189,7 @@ def test_pipeline_exception_context(capability): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -184,7 +197,7 @@ def test_pipeline_exception_context(capability): def test_casting_context(capability): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) @@ -219,7 +232,15 @@ class TailCallee: results.t = context.params.t results.c = capability().TestCallOrder._new_server(TailCallOrder()) +@pytest.mark.xfail def test_tail_call(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:75: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f87c17c5540 7f87c17c51b0 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c388ace7 + ''' callee_server = TailCallee() caller_server = TailCaller() diff --git a/test/test_capability_old.py b/test/test_capability_old.py index beaa5ca..cdcf46a 100644 --- a/test/test_capability_old.py +++ b/test/test_capability_old.py @@ -1,12 +1,15 @@ -import pytest -import capnp import os +import pytest + +import capnp this_dir = os.path.dirname(__file__) +# flake8: noqa: E501 + @pytest.fixture def capability(): - return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) + return capnp.load(os.path.join(this_dir, 'test_capability.capnp')) class Server: def __init__(self, val=1): @@ -116,7 +119,15 @@ def test_simple_client(capability): with pytest.raises(Exception): remote = client.foo(baz=5) +@pytest.mark.xfail def test_pipeline(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f680f7fce40 7f680f4f9250 7f680f4f4260 7f680f4fa9f0 7f680f4f6f50 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409 + ''' client = capability.TestPipeline._new_client(PipelineServer()) foo_client = capability.TestInterface._new_client(Server()) @@ -154,6 +165,7 @@ class BadPipelineServer: _results = _context.results _results.s = response.x + '_foo' _results.outBox.cap = capability().TestInterface._new_server(Server(100)) + def _error(error): raise Exception('test was a success') @@ -180,7 +192,7 @@ def test_pipeline_exception(capability): pipelinePromise = outCap.foo(i=10) with pytest.raises(Exception): - loop.wait(pipelinePromise) + pipelinePromise.wait() with pytest.raises(Exception): remote.wait() @@ -188,7 +200,7 @@ def test_pipeline_exception(capability): def test_casting(capability): client = capability.TestExtends._new_client(Server()) client2 = client.upcast(capability.TestInterface) - client3 = client2.cast_as(capability.TestInterface) + _ = client2.cast_as(capability.TestInterface) with pytest.raises(Exception): client.upcast(capability.TestPipeline) @@ -223,7 +235,15 @@ class TailCallee: results.t = t results.c = capability().TestCallOrder._new_server(TailCallOrder()) +@pytest.mark.xfail def test_tail_call(capability): + ''' + E capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:104: failed: :Fixture "capability" called directly. Fixtures are not meant to be called directly, + E but are created automatically when test functions request them as parameters. + E See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and + E https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code. + E stack: 7f680f4fb540 7f680f4fb1b0 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409 7f68115eb767 7f68115ece7e 7f68115c0ce7 + ''' callee_server = TailCallee() caller_server = TailCaller() diff --git a/test/test_examples.py b/test/test_examples.py new file mode 100644 index 0000000..1167418 --- /dev/null +++ b/test/test_examples.py @@ -0,0 +1,75 @@ +import os +import socket +import subprocess +import sys +import time + +examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') + + +def run_subprocesses(address, server, client): + cmd = [sys.executable, os.path.join(examples_dir, server), address] + server = subprocess.Popen(cmd) + retries = 30 + addr, port = address.split(':') + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" + cmd = [sys.executable, os.path.join(examples_dir, client), address] + client = subprocess.Popen(cmd) + + ret = client.wait(timeout=30) + server.kill() + assert ret == 0 + + +def test_async_calculator_example(): + address = 'localhost:36432' + server = 'async_calculator_server.py' + client = 'async_calculator_client.py' + run_subprocesses(address, server, client) + + +def test_thread_example(): + address = 'localhost:36433' + server = 'thread_server.py' + client = 'thread_client.py' + run_subprocesses(address, server, client) + + +def test_addressbook_example(): + proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')]) + ret = proc.wait() + assert ret == 0 + + +def test_async_example(): + address = 'localhost:36434' + server = 'async_server.py' + client = 'async_client.py' + run_subprocesses(address, server, client) + + +def test_ssl_async_example(): + address = 'localhost:36435' + server = 'async_ssl_server.py' + client = 'async_ssl_client.py' + run_subprocesses(address, server, client) + + +def test_ssl_reconnecting_async_example(): + address = 'localhost:36436' + server = 'async_ssl_server.py' + client = 'async_reconnecting_ssl_client.py' + run_subprocesses(address, server, client) diff --git a/test/test_large_read.capnp b/test/test_large_read.capnp new file mode 100644 index 0000000..45675ea --- /dev/null +++ b/test/test_large_read.capnp @@ -0,0 +1,13 @@ +@0x86dbb3b256f5d2af; + +struct Row { + values @0 :List(Int32); +} + +struct MultiArray { + rows @0 :List(Row); +} + +struct Msg { + data @0 :List(UInt8); +} diff --git a/test/test_large_read.py b/test/test_large_read.py new file mode 100644 index 0000000..d9e76c1 --- /dev/null +++ b/test/test_large_read.py @@ -0,0 +1,84 @@ +import os +import platform +import tempfile + +import pytest + +import capnp + +this_dir = os.path.dirname(__file__) + + +@pytest.fixture +def test_capnp(): + return capnp.load(os.path.join(this_dir, 'test_large_read.capnp')) + + +def test_large_read(test_capnp): + f = tempfile.TemporaryFile() + + array = test_capnp.MultiArray.new_message() + + row = array.init('rows', 1)[0] + values = row.init('values', 10000) + for i in range(len(values)): + values[i] = i + + array.write_packed(f) + f.seek(0) + + array = test_capnp.MultiArray.read_packed(f) + del f + assert array.rows[0].values[9000] == 9000 + +def test_large_read_multiple(test_capnp): + f = tempfile.TemporaryFile() + msg1 = test_capnp.Msg.new_message() + msg1.data = [0x41] * 8192 + msg1.write(f) + msg2 = test_capnp.Msg.new_message() + msg2.write(f) + f.seek(0) + + for m in test_capnp.Msg.read_multiple(f): + pass + +def get_two_adjacent_messages(test_capnp): + msg1 = test_capnp.Msg.new_message() + msg1.data = [0x41] * 8192 + m1 = msg1.to_bytes() + msg2 = test_capnp.Msg.new_message() + m2 = msg2.to_bytes() + + return m1 + m2 + +def test_large_read_multiple_bytes(test_capnp): + data = get_two_adjacent_messages(test_capnp) + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp)[:-1] + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp) + b' ' + for m in test_capnp.Msg.read_multiple_bytes(data): + pass + +@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="PyPy memoryview support is limited") +def test_large_read_mutltiple_bytes_memoryview(test_capnp): + data = get_two_adjacent_messages(test_capnp) + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp)[:-1] + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass + + with pytest.raises(capnp.KjException): + data = get_two_adjacent_messages(test_capnp) + b' ' + for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)): + pass diff --git a/test/test_load.py b/test/test_load.py index 7f6d6af..174d793 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -7,15 +7,15 @@ this_dir = os.path.dirname(__file__) @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) @pytest.fixture def foo(): - return capnp.load(os.path.join(this_dir, 'foo.capnp')) + return capnp.load(os.path.join(this_dir, 'foo.capnp')) @pytest.fixture def bar(): - return capnp.load(os.path.join(this_dir, 'bar.capnp')) + return capnp.load(os.path.join(this_dir, 'bar.capnp')) def test_basic_load(): capnp.load(os.path.join(this_dir, 'addressbook.capnp')) @@ -56,17 +56,23 @@ def test_failed_import(): bar.foo = foo def test_defualt_import_hook(): - import addressbook_capnp + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + + import addressbook_capnp # noqa: F401 def test_dash_import(): - import addressbook_with_dashes_capnp + import addressbook_with_dashes_capnp # noqa: F401 def test_spaces_import(): - import addressbook_with_spaces_capnp + import addressbook_with_spaces_capnp # noqa: F401 def test_add_import_hook(): capnp.add_import_hook([this_dir]) + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + import addressbook_capnp addressbook_capnp.AddressBook.new_message() @@ -75,6 +81,9 @@ def test_multiple_add_import_hook(): capnp.add_import_hook() capnp.add_import_hook([this_dir]) + # Make sure any previous imports of addressbook_capnp are gone + capnp.cleanup_global_schema_parser() + import addressbook_capnp addressbook_capnp.AddressBook.new_message() @@ -86,4 +95,4 @@ def test_remove_import_hook(): del sys.modules['addressbook_capnp'] # hack to deal with it being imported already with pytest.raises(ImportError): - import addressbook_capnp + import addressbook_capnp # noqa: F401 diff --git a/test/test_object.py b/test/test_object.py index f87e124..3fee49a 100644 --- a/test/test_object.py +++ b/test/test_object.py @@ -7,7 +7,7 @@ this_dir = os.path.dirname(__file__) @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) def test_object_basic(addressbook): diff --git a/test/test_regression.py b/test/test_regression.py index 388bcc5..acddcee 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -1,13 +1,22 @@ +# -*- coding: utf-8 -*- + import pytest import capnp import os import math +import sys this_dir = os.path.dirname(__file__) +if sys.version_info[0] < 3: + EXPECT_BYTES = True +else: + EXPECT_BYTES = False + + @pytest.fixture def addressbook(): - return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) + return capnp.load(os.path.join(this_dir, 'addressbook.capnp')) def test_addressbook_message_classes(addressbook): def writeAddressBook(fd): @@ -39,7 +48,7 @@ def test_addressbook_message_classes(addressbook): def printAddressBook(fd): - message = capnp._PackedFdMessageReader(f.fileno()) + message = capnp._PackedFdMessageReader(f) addressBook = message.get_root(addressbook.AddressBook) people = addressBook.people @@ -62,7 +71,7 @@ def test_addressbook_message_classes(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') writeAddressBook(f.fileno()) @@ -121,7 +130,7 @@ def test_addressbook(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') @@ -183,7 +192,7 @@ def test_addressbook_resizable(addressbook): assert bobPhones[0].type == 'home' assert bobPhones[1].number == "555-7654" assert bobPhones[1].type == 'work' - assert bob.employment.unemployed == None + assert bob.employment.unemployed is None f = open('example', 'w') @@ -253,7 +262,7 @@ def test_addressbook_explicit_fields(addressbook): assert bobPhones[1]._get_by_field(phone_fields['number']) == "555-7654" assert bobPhones[1]._get_by_field(phone_fields['type']) == 'work' employment = bob._get_by_field(person_fields['employment']) - employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) == None + employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) is None f = open('example', 'w') @@ -300,7 +309,7 @@ def init_all_types(builder): subBuilder.uInt64Field = 345678901234567890 subBuilder.float32Field = -1.25e-10 subBuilder.float64Field = 345 - subBuilder.textField = "baz" + subBuilder.textField = "☃" subBuilder.dataField = b"qux" subSubBuilder = subBuilder.structField subSubBuilder.textField = "nested" @@ -362,8 +371,8 @@ def check_list(reader, expected): assert reader[i] == v def check_all_types(reader): - assert reader.voidField == None - assert reader.boolField == True + assert reader.voidField is None + assert reader.boolField assert reader.int8Field == -123 assert reader.int16Field == -12345 assert reader.int32Field == -12345678 @@ -378,8 +387,8 @@ def check_all_types(reader): assert reader.dataField == b"bar" subReader = reader.structField - assert subReader.voidField == None - assert subReader.boolField == True + assert subReader.voidField is None + assert subReader.boolField assert subReader.int8Field == -12 assert subReader.int16Field == 3456 assert subReader.int32Field == -78901234 @@ -390,7 +399,15 @@ def check_all_types(reader): assert subReader.uInt64Field == 345678901234567890 assert_almost(subReader.float32Field, -1.25e-10) assert subReader.float64Field == 345 - assert subReader.textField == "baz" + + assert subReader.textField == "☃" + # This assertion highlights the encoding we expect to see here, since + # otherwise this appears a bit magical... + if EXPECT_BYTES: + assert len(subReader.textField) == 3 + else: + assert len(subReader.textField) == 1 + assert subReader.dataField == b"qux" subSubReader = subReader.structField @@ -398,6 +415,12 @@ def check_all_types(reader): assert subSubReader.structField.textField == "really nested" assert subReader.enumField == "baz" + # Check that enums are hashable and can be used as keys in dicts + # interchangably with their string version. + assert hash(subReader.enumField) == hash('baz') + assert {subReader.enumField: 17}.get(subReader.enumField) == 17 + assert {subReader.enumField: 17}.get('baz') == 17 + assert {'baz': 17}.get(subReader.enumField) == 17 check_list(subReader.voidList, [None, None, None]) check_list(subReader.boolList, [False, True, False, True, True]) @@ -463,26 +486,26 @@ def check_all_types(reader): def test_build(all_types): root = all_types.TestAllTypes.new_message() init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_build_first_segment_size(all_types): root = all_types.TestAllTypes.new_message(1) init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText - root = all_types.TestAllTypes.new_message(1024*1024) + root = all_types.TestAllTypes.new_message(1024 * 1024) init_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_binary_read(all_types): - f = open(os.path.join(this_dir, 'all-types.binary'), 'r') + f = open(os.path.join(this_dir, 'all-types.binary'), 'r', encoding='utf8') root = all_types.TestAllTypes.read(f) check_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText # Test set_root(). @@ -495,11 +518,11 @@ def test_binary_read(all_types): check_all_types(builder2.get_root(all_types.TestAllTypes)) def test_packed_read(all_types): - f = open(os.path.join(this_dir, 'all-types.packed'), 'r') + f = open(os.path.join(this_dir, 'all-types.packed'), 'r', encoding='utf8') root = all_types.TestAllTypes.read_packed(f) check_all_types(root) - expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read() + expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read() assert str(root) + '\n' == expectedText def test_binary_write(all_types): diff --git a/test/test_response.capnp b/test/test_response.capnp new file mode 100644 index 0000000..268bc08 --- /dev/null +++ b/test/test_response.capnp @@ -0,0 +1,13 @@ +@0x84249be5c3bff005; + +interface Foo { + foo @0 () -> (val :UInt32); +} + +struct Bar { + foo @0 :Foo; +} + +interface Baz { + grault @0 () -> (bar: Bar); +} diff --git a/test/test_response.py b/test/test_response.py new file mode 100644 index 0000000..ae3eb40 --- /dev/null +++ b/test/test_response.py @@ -0,0 +1,35 @@ +import test_response_capnp + +class FooServer(test_response_capnp.Foo.Server): + def __init__(self, val=1): + self.val = val + + def foo(self, **kwargs): + return 1 + +class BazServer(test_response_capnp.Baz.Server): + def __init__(self, val=1): + self.val = val + + def grault(self, **kwargs): + return {"foo": FooServer()} + +def test_response_reference(): + baz = test_response_capnp.Baz._new_client(BazServer()) + + bar = baz.grault().wait().bar + + foo = bar.foo + # This used to cause an exception about invalid pointers because the response got garbage collected + assert foo.foo().wait().val == 1 + +def test_response_reference2(): + baz = test_response_capnp.Baz._new_client(BazServer()) + + bar = baz.grault().wait().bar + + # This always worked since it saved the intermediate response object + response = baz.grault().wait() + bar = response.bar + foo = bar.foo + assert foo.foo().wait().val == 1 diff --git a/test/test_rpc.py b/test/test_rpc.py index d288d04..ba1c5e8 100644 --- a/test/test_rpc.py +++ b/test/test_rpc.py @@ -1,6 +1,9 @@ +''' +rpc test +''' + import pytest import capnp -import os import socket import test_capability_capnp @@ -8,89 +11,32 @@ import test_capability_capnp class Server(test_capability_capnp.TestInterface.Server): - def __init__(self, val=1): + def __init__(self, val=100): self.val = val def foo(self, i, j, **kwargs): return str(i * 5 + self.val) -def restore_func(ref_id): - return Server(100) +def test_simple_rpc_with_options(): + read, write = socket.socketpair() - -class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): - - def restore(self, ref_id): - assert ref_id.tag == 'testInterface' - return Server(100) - - -def test_simple_rpc(): - read, write = socket.socketpair(socket.AF_UNIX) - - restorer = SimpleRestorer() - server = capnp.TwoPartyServer(write, restorer) - client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - -def test_simple_rpc_restore_func(): - read, write = socket.socketpair(socket.AF_UNIX) - - server = capnp.TwoPartyServer(write, restore_func) - client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - -def text_restore_func(objectId): - text = objectId.as_text() - assert text == 'testInterface' - return Server(100) - - -def test_ez_rpc(): - read, write = socket.socketpair(socket.AF_UNIX) - - server = capnp.TwoPartyServer(write, text_restore_func) - client = capnp.TwoPartyClient(read) - - cap = client.ez_restore('testInterface') - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = remote.wait() - - assert response.x == '125' - - cap = client.restore(test_capability_capnp.TestSturdyRefObjectId.new_message()) - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) + _ = capnp.TwoPartyServer(write, bootstrap=Server()) + # This traversal limit is too low to receive the response in, so we expect + # an exception during the call. + client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) with pytest.raises(capnp.KjException): - response = remote.wait() + cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) + + remote = cap.foo(i=5) + _ = remote.wait() + def test_simple_rpc_bootstrap(): - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() - server = capnp.TwoPartyServer(write, bootstrap=Server(100)) + _ = capnp.TwoPartyServer(write, bootstrap=Server(100)) client = capnp.TwoPartyClient(read) cap = client.bootstrap() diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py index 2eccf99..50ba4e6 100644 --- a/test/test_rpc_calculator.py +++ b/test/test_rpc_calculator.py @@ -1,21 +1,83 @@ -import capnp -import os -import socket import gc - +import os +import pytest +import socket +import subprocess import sys # add examples dir to sys.path -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'examples')) -import calculator_client -import calculator_server +import time + +import capnp + +examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') +sys.path.append(examples_dir) + +import calculator_client # noqa: E402 +import calculator_server # noqa: E402 def test_calculator(): - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() - server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) + _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) calculator_client.main(read) +def run_subprocesses(address): + cmd = [sys.executable, os.path.join(examples_dir, 'calculator_server.py'), address] + server = subprocess.Popen(cmd) + retries = 30 + if 'unix' in address: + addr = address.split(':')[1] + while True: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + result = sock.connect_ex(addr) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" + else: + addr, port = address.split(':') + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + result = sock.connect_ex((addr, int(port))) + if result == 0: + break + # Give the server some small amount of time to start listening + time.sleep(0.1) + retries -= 1 + if retries == 0: + assert False, "Timed out waiting for server to start" + cmd = [sys.executable, os.path.join(examples_dir, 'calculator_client.py'), address] + client = subprocess.Popen(cmd) + + ret = client.wait() + server.kill() + assert ret == 0 + + +def test_calculator_tcp(): + address = 'localhost:36431' + run_subprocesses(address) + + +@pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows") +def test_calculator_unix(): + path = '/tmp/pycapnp-test' + try: + os.unlink(path) + except OSError: + pass + + address = 'unix:' + path + run_subprocesses(address) + def test_calculator_gc(): def new_evaluate_impl(old_evaluate_impl): def call(*args, **kwargs): @@ -23,13 +85,13 @@ def test_calculator_gc(): return old_evaluate_impl(*args, **kwargs) return call - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() # inject a gc.collect to the beginning of every evaluate_impl call evaluate_impl_orig = calculator_server.evaluate_impl calculator_server.evaluate_impl = new_evaluate_impl(evaluate_impl_orig) - server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) + _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl()) calculator_client.main(read) calculator_server.evaluate_impl = evaluate_impl_orig diff --git a/test/test_serialization.py b/test/test_serialization.py index a1893e7..1f26d15 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -5,6 +5,8 @@ import platform import test_regression import tempfile import pickle +import mmap +import sys this_dir = os.path.dirname(__file__) @@ -40,6 +42,50 @@ def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.from_bytes(message_bytes) test_regression.check_all_types(msg) +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="TODO: Investigate why this works on CPython but fails on PyPy." +) +def test_roundtrip_segments(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + segments = msg.to_segments() + msg = all_types.TestAllTypes.from_segments(segments) + test_regression.check_all_types(msg) + +@pytest.mark.skipif(sys.version_info[0] < 3, reason="mmap doesn't implement the buffer interface under python 2.") +def test_roundtrip_bytes_mmap(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + with tempfile.TemporaryFile() as f: + msg.write(f) + length = f.tell() + + f.seek(0) + memory = mmap.mmap(f.fileno(), length) + + msg = all_types.TestAllTypes.from_bytes(memory) + test_regression.check_all_types(msg) + +@pytest.mark.skipif(sys.version_info[0] < 3, reason="memoryview is a builtin on Python 3") +def test_roundtrip_bytes_buffer(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + b = msg.to_bytes() + v = memoryview(b) + msg = all_types.TestAllTypes.from_bytes(v) + test_regression.check_all_types(msg) + +def test_roundtrip_bytes_fail(all_types): + with pytest.raises(TypeError): + all_types.TestAllTypes.from_bytes(42) + +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test." +) def test_roundtrip_bytes_packed(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -106,7 +152,10 @@ def test_roundtrip_bytes_multiple_packed(all_types): i += 1 assert i == 3 -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now.") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now." +) def test_roundtrip_dict(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -142,3 +191,36 @@ def test_pickle(all_types): msg2 = pickle.loads(data) test_regression.check_all_types(msg2) + +def test_from_bytes_traversal_limit(all_types): + size = 1024 + bld = all_types.TestAllTypes.new_message() + bld.init("structList", size) + data = bld.to_bytes() + + msg = all_types.TestAllTypes.from_bytes(data) + with pytest.raises(capnp.KjException): + for i in range(0, size): + msg.structList[i].uInt8Field == 0 + + msg = all_types.TestAllTypes.from_bytes(data, + traversal_limit_in_words=2**62) + for i in range(0, size): + assert msg.structList[i].uInt8Field == 0 + + +def test_from_bytes_packed_traversal_limit(all_types): + size = 1024 + bld = all_types.TestAllTypes.new_message() + bld.init("structList", size) + data = bld.to_bytes_packed() + + msg = all_types.TestAllTypes.from_bytes_packed(data) + with pytest.raises(capnp.KjException): + for i in range(0, size): + msg.structList[i].uInt8Field == 0 + + msg = all_types.TestAllTypes.from_bytes_packed(data, + traversal_limit_in_words=2**62) + for i in range(0, size): + assert msg.structList[i].uInt8Field == 0 diff --git a/test/test_struct.py b/test/test_struct.py index f0c3b45..7b23b5a 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -77,7 +77,10 @@ def test_which_reader(addressbook): addresses.which -@pytest.mark.skipif(capnp.version.LIBCAPNP_VERSION < 5000, reason="Using ints as enums requires v0.5.0+ of the C++ capnp library") +@pytest.mark.skipif( + capnp.version.LIBCAPNP_VERSION < 5000, + reason="Using ints as enums requires v0.5.0+ of the C++ capnp library" +) def test_enum(addressbook): addresses = addressbook.AddressBook.new_message() people = addresses.init('people', 2) @@ -107,6 +110,22 @@ def test_builder_set(addressbook): person.foo = 'test' +def test_builder_set_from_list(all_types): + msg = all_types.TestAllTypes.new_message() + + msg.int32List = [0, 1, 2] + + assert list(msg.int32List) == [0, 1, 2] + + +def test_builder_set_from_tuple(all_types): + msg = all_types.TestAllTypes.new_message() + + msg.int32List = (0, 1, 2) + + assert list(msg.int32List) == [0, 1, 2] + + def test_null_str(all_types): msg = all_types.TestAllTypes.new_message() @@ -172,13 +191,8 @@ def test_set_dict_union(addressbook): assert person.employment.employer.name == 'foo' -try: - basestring # attempt to evaluate basestring - def isstr(s): - return isinstance(s, basestring) -except NameError: - def isstr(s): - return isinstance(s, str) +def isstr(s): + return isinstance(s, str) def test_to_dict_enum(addressbook): @@ -211,10 +225,28 @@ def test_to_dict_verbose(addressbook): def test_to_dict_ordered(addressbook): - person = addressbook.Person.new_message(**{'name': 'Alice', 'phones': [{'type': 'mobile', 'number': '555-1212'}], 'id': 123, 'employment': {'school': 'MIT'}, 'email': 'alice@example.com'}) + person = addressbook.Person.new_message(**{ + 'name': 'Alice', + 'phones': [{'type': 'mobile', 'number': '555-1212'}], + 'id': 123, + 'employment': {'school': 'MIT'}, 'email': 'alice@example.com' + }) if sys.version_info >= (2, 7): assert list(person.to_dict(ordered=True).keys()) == ['id', 'name', 'email', 'phones', 'employment'] else: with pytest.raises(Exception): person.to_dict(ordered=True) + +def test_nested_list(addressbook): + struct = addressbook.NestedList.new_message() + struct.init('list', 2) + + struct.list.init(0, 1) + struct.list.init(1, 2) + + struct.list[0][0] = 1 + struct.list[1][0] = 2 + struct.list[1][1] = 3 + + assert struct.to_dict()["list"] == [[1], [2, 3]] diff --git a/test/test_threads.py b/test/test_threads.py index 8604ecf..bd7ae71 100644 --- a/test/test_threads.py +++ b/test/test_threads.py @@ -1,20 +1,38 @@ -import capnp -import pytest -import test_capability_capnp +''' +thread test +''' + +import platform import socket import threading -import platform -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +import pytest + +import capnp +import test_capability_capnp + +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_making_event_loop(): + ''' + Event loop test + ''' capnp.remove_event_loop(True) capnp.create_event_loop() capnp.remove_event_loop() capnp.create_event_loop() -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_making_threaded_event_loop(): + ''' + Threaded event loop test + ''' capnp.remove_event_loop(True) capnp.create_event_loop(True) @@ -23,31 +41,34 @@ def test_making_threaded_event_loop(): class Server(test_capability_capnp.TestInterface.Server): - - def __init__(self, val=1): + ''' + Server + ''' + def __init__(self, val=100): self.val = val def foo(self, i, j, **kwargs): + ''' + foo + ''' return str(i * 5 + self.val) -class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer): - - def restore(self, ref_id): - assert ref_id.tag == 'testInterface' - return Server(100) - - -@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy") +@pytest.mark.skipif( + platform.python_implementation() == 'PyPy', + reason="pycapnp's GIL handling isn't working properly at the moment for PyPy" +) def test_using_threads(): + ''' + Thread test + ''' capnp.remove_event_loop(True) capnp.create_event_loop(True) - read, write = socket.socketpair(socket.AF_UNIX) + read, write = socket.socketpair() def run_server(): - restorer = SimpleRestorer() - server = capnp.TwoPartyServer(write, restorer) + _ = capnp.TwoPartyServer(write, bootstrap=Server()) capnp.wait_forever() server_thread = threading.Thread(target=run_server) @@ -55,10 +76,7 @@ def test_using_threads(): server_thread.start() client = capnp.TwoPartyClient(read) - - ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface') - cap = client.restore(ref) - cap = cap.cast_as(test_capability_capnp.TestInterface) + cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) remote = cap.foo(i=5) response = remote.wait() diff --git a/tox.ini b/tox.ini index 54b3276..75af016 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py32,py33 +envlist = py27,py34,py35,py36 [testenv] deps=