Merge branch 'develop'

.github/workflows/pythonpackage.yml

@@ -0,0 +1,40 @@
+name: Python Test Packaging
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 4
+      fail-fast: false
+      matrix:
+        # Some asyncio commands require 3.7+
+        # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work
+        python-version: [3.7]
+        os: [ubuntu-latest, macOS-latest, windows-latest]
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Build pycapnp and install
+      run: |
+        python setup.py build
+        pip install .
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude benchmark
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude benchmark
+    - name: Test with pytest
+      run: |
+        pip install pytest
+        pytest

.gitignore

@@ -41,3 +41,9 @@ capnp/*.cpp
 capnp/version.py
 MANIFEST
 docs/_build
+
+capnp/lib/capnp.cpp
+capnp/lib/capnp.h
+bundled/
+example
+*.iml

.travis.yml

@@ -1,10 +1,13 @@
+# Use older, non-container infrastructure to allow sudo
+sudo: required
+
 language: python
 
 python:
-  - 2.6
   - 2.7
-  - 3.3
   - 3.4
+  - 3.5
+  - 3.6
   - pypy
 
 env:

CHANGELOG.md

@@ -1,3 +1,68 @@
+## v0.6.4 (2019-01-31)
+- Fix bugs in `read_multiple_bytes` (thanks to @tsh56)
+- Remove end-of-life Python versions 2.6, 3.2, and 3.3. Add CI tests for 3.6
+- Expose SchemaParser in Cython header
+
+## v0.6.3 (2018-01-14)
+- Bump bundled capnp version to v0.6.1 (thanks to @E8Yuval)
+- Fix a memleak in RemotePromise (thanks to @E8Yuval)
+
+## v0.6.2 (2017-11-30)
+- Add support for buffers/memoryviews in `from_bytes` (thanks to @aldanor)
+
+## v0.6.1 (2017-07-27)
+- Fixed upload to PyPi (forgot to cythonize)
+
+## v0.6.0 (2017-07-27)
+- Update bundled capnp version to v0.6.0 and fix related problems (thanks to @benmoran)
+- Fix memleak with KjException (thanks to @tsh56)
+
+## v0.5.12 (2017-04-18)
+- Bump bundled capnp version to v0.5.3.1
+
+## v0.5.11 (2017-04-10)
+- Make enums hashable (thanks to @madeleine-empirical)
+- Rework logic on when to build bundled libcapnp. Fixes cross-compilation (thanks to @benizl)
+- Add traversal_limit_in_words and nesting_limit to RPC classes (thanks to @asilversempirical)
+- Include class attributes in __dir__. This allows for code completion of class methods (thanks to @chaoflow )
+- Allow setting lists with python tuples (thanks to @chaoflow)
+- Fix traversal_limit_in_words and nesting_limit being ignored by `from_bytes` (thanks to @plesner)
+
+## v0.5.10 (2016-11-28)
+- Fix bug that prevented event loop from actually being lazy initialized
+- Fix possible recursive loop in KjException
+- Add `clear_write_flag` method to builder classes
+
+## v0.5.9 (2016-07-07)
+- Make the event loop be lazy initialized
+- Add support for segment (de)serialization (thanks to @gcv). See to_segments/from_segments methods.
+- Fix response objects not referencing parents correctly
+- Add test for large reads
+
+## v0.5.8 (2016-05-27)
+- Fix build problem with Cython v0.24
+- Include the changelog in the manifest (should fix install problems if pandoc is present)
+- Include the traceback in exceptions
+- Make sure to encode to utf-8, not the default encoding (thanks to @novas0x2a)
+- Add --libcapnp-url option in installer to allow installing arbitrary libcapnp versions
+- Support mmap objects for reading with from_bytes (thanks to @bpiwowar)
+- Change read_multiple and read_multiple_packed to copy by default
+- Fix mistakenly discarding the file parameter on reads
+- Add reraise_kj_exception to the prettyPrint functions. (thanks to @kdienes)
+- Fix KjException init (missing wrapper). (thanks to @E8-Storage)
+- Add `result_type` to InterfaceMethodSchema
+
+
+## v0.5.7 (2015-06-16)
+- Update bundled libcapnp to v0.5.2
+- Add warnings for using old restorer methods. You should use `bootstrap` instead
+- Fix warning from PyEventPort
+- Handle AnyPointers better as arguments to RPC functions
+- Add support for using keyword arguments with a named struct in an RPC
+- Add bootstrap method to TwoPartyServer
+- Add `init` method to lists
+- Add support for unix sockets in RPC
+
 ## v0.5.6 (2015-04-13)
 - Fix a serious bug in TwoPartyServer that was preventing it from working when passed a string address.
 - Fix bugs that were exposed by defining KJDEBUG (thanks @davidcarne for finding this)

DEPLOY.md

@@ -0,0 +1,44 @@
+# Deployment instructions for PyPi
+
+This file is meant for maintainers of pycapnp, and documents the process for uploading to PyPI.
+
+## Pre-requisites
+
+```
+pip install pypandoc cython
+```
+
+## Run tests
+
+I typically sanity check by running the tests once again locally, but as long as Travis is green, you're probably fine.
+
+## Add a commit that bumps the version
+
+Bump the version in setup.py, and add descriptions of all the changes to CHANGELOG.md (see 19e1b189caa786c7f572e679d6bb94aadfbdb5e0 for an example commit).
+
+## Run the build and upload
+
+Run the following command to clean up old artifacts, run the build, and then upload the result to PyPI
+
+```
+rm -rf bundled/ capnp/version.py capnp/lib/capnp.{h,cpp} build; python setup.py build && python setup.py sdist upload -r PyPI
+```
+
+## Test the PyPI release
+
+I manually test the PyPI release after it's been uploaded. I have a few virtualenvs that I manually run the following command in (run this from the pycapnp directory since it runs the tests at the end):
+```
+yes | pip uninstall pycapnp; pip install pycapnp && py.test test
+```
+
+I usually test the following configurations:
+- Python 2.7 with and without cython installed
+- Python 3.6 with and without cython installed
+
+This step could probably benefit greatly from some automation. Perhaps even Travis could handle it, but I'm not sure how best to trigger Travis from a PyPI release.
+
+## Tag the github release
+
+Tag the release on the develop branch (not the master branch). Sadly, I've stopped using git-flow, and at this point it might be worth moving back to using just master, but that would take some amount of work and I worry that it would break open PRs. Definitely worth considering if development picks back up.
+
+Version numbers roughly follow semver, although I try to loosely follow upstream Cap'n Proto C++ versions as well. So when pycapnp officially starts using v0.7.0 of the C++ library, pycapnp's version should be bumped to v0.7.0 as well.

MANIFEST.in

@@ -1,3 +1,4 @@
 include README.md
+include CHANGELOG.md
 include requirements.txt
 include buildutils/*

README.md

@@ -1,53 +1,96 @@
-# pycapnp
+# pycapnp-async
+
+[![Actions Status](https://github.com/haata/pycapnp-async/workflows/Python%20Test%20Packaging/badge.svg)](https://github.com/haata/pycapnp-async/actions)
 
-More thorough docs are available at [http://jparyani.github.io/pycapnp/](http://jparyani.github.io/pycapnp/).
 
 ## Requirements
 
-pycapnp's distribution has no requirements beyond a C++11 compatible compiler. GCC 4.8+ or Clang 3.3+ should work fine.
+* C++14 supported compiler
+  - gcc 6.1+ (5+ may work)
+  - clang 6 (3.4+ may work)
+  - Visual Studio 2017+
+* cmake (needed for bundled capnproto)
+  - ninja (macOS + Linux)
+  - Visual Studio 2017+
+
+* capnproto-0.7.0
+  - Not necessary if using bundled capnproto
+
+32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake.
+
+pycapnp has additional development dependencies, including cython and pytest. See requirements.txt for them all.
 
-pycapnp has additional development dependencies, including cython and py.test. See requirements.txt for them all.
 
 ## Building and installation
 
-Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-4.8 pip install pycapnp`.
+Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-8.2 pip install pycapnp`.
 
 Or you can clone the repo like so:
 
-    git clone https://github.com/jparyani/pycapnp.git
+```bash
-    pip install --install-option '--force-cython' ./pycapnp
+git clone https://github.com/haata/pycapnp-async.git
+cd pycapnp-async
+pip install .
+```
 
-Note: for OSX, if using clang from Xcode 5, you may need to set `CFLAGS` like so:
+If you wish to install using the latest upstream C++ Cap'n Proto:
+
+```bash
+pip install \
+    --install-option "--libcapnp-url" \
+    --install-option "https://github.com/sandstorm-io/capnproto/archive/master.tar.gz" \
+    --install-option "--force-bundled-libcapnp" .
+```
+
+To force bundled python:
+
+```bash
+pip install --install-option "--force-bundled-libcapnp" .
+```
 
-    CFLAGS='-stdlib=libc++' pip install pycapnp
 
 ## Python Versions
 
-Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported.
+Python 3.7+ is supported.
+Earlier versions of Python have asyncio bugs that might be possible to work around, but may require significant work (3.5 and 3.6).
 
-One oddity to note is that `Text` type fields will be treated as byte strings under Python 2, and unicode strings under Python 3. `Data` fields will always be treated as byte strings.
 
 ## Development
 
-This project uses [git-flow](http://jeffkreeftmeijer.com/2010/why-arent-you-using-git-flow/). Essentially, just make sure you do your changes in the `develop` branch. You can run the tests by installing pytest with `pip install pytest`, and then run `py.test` from the `test` directory.
+Git flow has been abandoned, use master.
+
+To test, use a pipenv (or install requirements.txt and run pytest manually).
+```bash
+pip install pipenv
+pipenv install
+pipenv run pytest
+```
+
 
 ### Binary Packages
 
-In order to build binary packages from this source code, you must specify the `--disable-cython` option:
-
 Building a dumb binary distribution:
 
-    python setup.py bdist_dumb --disable-cython
+```bash
+python setup.py bdist_dumb
+```
 
 Building a Python wheel distributiion:
 
-    python setup.py bdist_wheel --disable-cython
+```bash
+python setup.py bdist_wheel
+```
 
-If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with:
 
-    python setup.py build --force-cython
+### Pypi Upload Instructions
+
+Only necessary if uploading release to pypi.org.
+
+TODO
+
 
 ## Documentation/Example
+
 There is some basic documentation [here](http://jparyani.github.io/pycapnp/).
 
 The examples directory has one example that shows off pycapnp quite nicely. Here it is, reproduced:
@@ -158,17 +201,3 @@ if __name__ == '__main__':
     server(write_end)
     client(read_end)
 ```
-
-## Common Problems
-
-If you get an error on installation like:
-
-    ...
-    gcc-4.8: error: capnp/capnp.c: No such file or directory
-
-    gcc-4.8: fatal error: no input files
-
-Then you have too old a version of setuptools. Run `pip install -U setuptools` then try again.
-
-
-[![Build Status](https://travis-ci.org/jparyani/pycapnp.png?branch=develop)](https://travis-ci.org/jparyani/pycapnp)

benchmark/bin/run_all.py

@@ -6,10 +6,13 @@ import sys
 import os
 import json
 import argparse
+import time
+
+_this_dir = os.path.dirname(__file__)
 
 def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l capnproto -l protobuf", action='append', default=['pycapnp', 'pyproto', 'pyproto_cpp'])
+    parser.add_argument('-l', "--langs", help="Add languages to test, ie: -l pyproto -l pyproto_cpp", action='append', default=['pycapnp'])
     parser.add_argument("-r", "--reuse", help="If this flag is passed, re-use tests will be run", action='store_true')
     parser.add_argument("-c", "--compression", help="If this flag is passed, compression tests will be run", action='store_true')
     parser.add_argument("-i", "--scale_iters", help="Scaling factor to multiply the default iters by", type=float, default=1.0)
@@ -26,26 +29,24 @@ def run_one(prefix, name, mode, iters, faster, compression):
     if compression != 'none':
         res_type += '_' + compression
 
-    command = ["time", "-p", prefix+"-"+name, mode, reuse, compression, str(iters)]
+    command = [os.path.join(_this_dir, prefix+"-"+name), mode, reuse, compression, str(iters)]
+    start = time.time()
+    print('running: ' + ' '.join(command), file=sys.stderr)
     p = Popen(command, stdout=PIPE, stderr=PIPE)
-    res = p.communicate()[1]
+    res = p.wait()
+    end = time.time()
 
     data = {}
 
     if p.returncode != 0:
-        sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + res + '\n')
+        sys.stderr.write(' '.join(command) + ' failed to run with errors: \n' + p.stderr.read() + '\n')
         sys.stderr.flush()
-    else:
-        res = res.strip()
-
-        for line in res.split('\n'):
-            vals = line.split()
-            data[vals[0]] = float(vals[1])
 
     data['type'] = res_type
     data['mode'] = mode
     data['name'] = name
     data['iters'] = iters
+    data['time'] = end - start
 
     return data
 

buildutils/__init__.py

@@ -2,6 +2,7 @@
 
 Largely adapted from h5py
 """
+# flake8: noqa F401 F403
 
 from .msg import *
 from .config import *

buildutils/build.py

@@ -2,29 +2,79 @@
 
 import subprocess
 import os
-import tempfile
+import shutil
+import struct
+import sys
 
-def build_libcapnp(bundle_dir, build_dir, verbose=False):
+def build_libcapnp(bundle_dir, build_dir):
+    '''
+    Build capnproto
+    '''
     bundle_dir = os.path.abspath(bundle_dir)
     capnp_dir = os.path.join(bundle_dir, 'capnproto-c++')
     build_dir = os.path.abspath(build_dir)
+    tmp_dir = os.path.join(capnp_dir, 'build{}'.format(8 * struct.calcsize("P")))
+
+    # Clean the tmp build directory every time
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    os.mkdir(tmp_dir)
 
-  with tempfile.TemporaryFile() as f:
-    stdout = f
-    if verbose:
-      stdout = None
     cxxflags = os.environ.get('CXXFLAGS', None)
-    os.environ['CXXFLAGS'] = (cxxflags or '') + ' -fPIC -O2 -DNDEBUG'
+    os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG'
-    conf = subprocess.Popen(['./configure', '--disable-shared', '--prefix', build_dir], cwd=capnp_dir, stdout=stdout)
+
+    # Enable ninja for compilation if available
+    build_type = []
+    if shutil.which('ninja'):
+        build_type = ['-G', 'Ninja']
+
+    # Determine python shell architecture for Windows
+    python_arch = 8 * struct.calcsize("P")
+    build_arch = []
+    build_flags = []
+    if os.name == 'nt':
+        if python_arch == 64:
+            build_arch_flag = "x64"
+        elif python_arch == 32:
+            build_arch_flag = "Win32"
+        else:
+            raise RuntimeError('Unknown windows build arch')
+        build_arch = ['-A', build_arch_flag]
+        build_flags = ['--config', 'Release']
+        print('Building module for {}'.format(python_arch))
+
+    if not shutil.which('cmake'):
+        raise RuntimeError('Could not find cmake in your path!')
+
+    args = [
+        'cmake',
+        '-DCMAKE_POSITION_INDEPENDENT_CODE=1',
+        '-DBUILD_TESTING=OFF',
+        '-DBUILD_SHARED_LIBS=OFF',
+        '-DCMAKE_INSTALL_PREFIX:PATH={}'.format(build_dir),
+        capnp_dir,
+    ]
+    args.extend(build_type)
+    args.extend(build_arch)
+    conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout)
     returncode = conf.wait()
     if returncode != 0:
-      raise RuntimeError('Configure failed')
+        raise RuntimeError('CMake failed {}'.format(returncode))
 
-    make = subprocess.Popen(['make', '-j4', 'install'], cwd=capnp_dir, stdout=stdout)
+    # Run build through cmake
-    returncode = make.wait()
+    args = [
+        'cmake',
+        '--build',
+        '.',
+        '--target',
+        'install',
+    ]
+    args.extend(build_flags)
+    build = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout)
+    returncode = build.wait()
     if cxxflags is None:
         del os.environ['CXXFLAGS']
     else:
         os.environ['CXXFLAGS'] = cxxflags
     if returncode != 0:
-      raise RuntimeError('Make failed')
+        raise RuntimeError('capnproto compilation failed: {}'.format(returncode))

buildutils/bundle.py

@@ -1,12 +1,11 @@
 """utilities for fetching build dependencies."""
 
-#-----------------------------------------------------------------------------
+#
 #  Copyright (C) PyZMQ Developers
 #  Distributed under the terms of the Modified BSD License.
 #
 #  This bundling code is largely adapted from pyzmq-static's get.sh by
 #  Brandon Craig-Rhodes, which is itself BSD licensed.
-#-----------------------------------------------------------------------------
 #
 # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq
 # for original project.
@@ -14,40 +13,31 @@
 
 import os
 import shutil
-import stat
-import sys
 import tarfile
-from glob import glob
-from subprocess import Popen, PIPE
 
-try:
-    # py2
-    from urllib2 import urlopen
-except ImportError:
-    # py3
 from urllib.request import urlopen
-
+from .msg import info
-from .msg import fatal, debug, info, warn
 
 pjoin = os.path.join
 
-#-----------------------------------------------------------------------------
+#
 # Constants
-#-----------------------------------------------------------------------------
+#
 
-bundled_version = (0,5,1,2)
+bundled_version = (0, 7, 0)
-libcapnp = "capnproto-c++-%i.%i.%i.%i.tar.gz" % (bundled_version)
+libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version)
-libcapnp_url = "https://capnproto.org/" + libcapnp
+libcapnp_url = "https://capnproto.org/" + libcapnp_name
 
 HERE = os.path.dirname(__file__)
 ROOT = os.path.dirname(HERE)
 
-#-----------------------------------------------------------------------------
+#
 # Utilities
-#-----------------------------------------------------------------------------
+#
 
 
 def untgz(archive):
+    """Remove .tar.gz"""
     return archive.replace('.tar.gz', '')
 
 def localpath(*args):
@@ -69,97 +59,29 @@ def fetch_archive(savedir, url, fname, force=False):
         f.write(req.read())
     return dest
 
-#-----------------------------------------------------------------------------
+#
 # libcapnp
-#-----------------------------------------------------------------------------
+#
 
-def fetch_libcapnp(savedir):
+def fetch_libcapnp(savedir, url=None):
     """download and extract libcapnp"""
+    is_preconfigured = False
+    if url is None:
+        url = libcapnp_url
+        is_preconfigured = True
     dest = pjoin(savedir, 'capnproto-c++')
     if os.path.exists(dest):
         info("already have %s" % dest)
         return
-    fname = fetch_archive(savedir, libcapnp_url, libcapnp)
+    fname = fetch_archive(savedir, url, libcapnp_name)
     tf = tarfile.open(fname)
     with_version = pjoin(savedir, tf.firstmember.path)
     tf.extractall(savedir)
     tf.close()
     # remove version suffix:
+    if is_preconfigured:
         shutil.move(with_version, dest)
-
-def stage_platform_hpp(capnproot):
-    """stage platform.hpp into libcapnp sources
-
-    Tries ./configure first (except on Windows),
-    then falls back on included platform.hpp previously generated.
-    """
-
-    platform_hpp = pjoin(capnproot, 'src', 'platform.hpp')
-    if os.path.exists(platform_hpp):
-        info("already have platform.hpp")
-        return
-    if os.name == 'nt':
-        # stage msvc platform header
-        platform_dir = pjoin(capnproot, 'builds', 'msvc')
     else:
-        info("attempting ./configure to generate platform.hpp")
+        cpp_dir = os.path.join(with_version, 'c++')
-
+        shutil.move(cpp_dir, dest)
-        p = Popen('./configure', cwd=capnproot, shell=True,
-            stdout=PIPE, stderr=PIPE,
-        )
-        o,e = p.communicate()
-        if p.returncode:
-            warn("failed to configure libcapnp:\n%s" % e)
-            if sys.platform == 'darwin':
-                platform_dir = pjoin(HERE, 'include_darwin')
-            elif sys.platform.startswith('freebsd'):
-                platform_dir = pjoin(HERE, 'include_freebsd')
-            elif sys.platform.startswith('linux-armv'):
-                platform_dir = pjoin(HERE, 'include_linux-armv')
-            else:
-                platform_dir = pjoin(HERE, 'include_linux')
-        else:
-            return
-
-    info("staging platform.hpp from: %s" % platform_dir)
-    shutil.copy(pjoin(platform_dir, 'platform.hpp'), platform_hpp)
-
-
-def copy_and_patch_libcapnp(capnp, libcapnp):
-    """copy libcapnp into source dir, and patch it if necessary.
-
-    This command is necessary prior to running a bdist on Linux or OS X.
-    """
-    if sys.platform.startswith('win'):
-        return
-    # copy libcapnp into capnp for bdist
-    local = localpath('capnp',libcapnp)
-    if not capnp and not os.path.exists(local):
-        fatal("Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` "
-        "or copy libcapnp into capnp/ manually prior to running bdist.")
-    try:
-        # resolve real file through symlinks
-        lib = os.path.realpath(pjoin(capnp, 'lib', libcapnp))
-        print ("copying %s -> %s"%(lib, local))
-        shutil.copy(lib, local)
-    except Exception:
-        if not os.path.exists(local):
-            fatal("Could not copy libcapnp into capnp/, which is necessary for bdist. "
-            "Please specify capnp prefix via `setup.py configure --capnp=/path/to/capnp` "
-            "or copy libcapnp into capnp/ manually.")
-
-    if sys.platform == 'darwin':
-        # chmod u+w on the lib,
-        # which can be user-read-only for some reason
-        mode = os.stat(local).st_mode
-        os.chmod(local, mode | stat.S_IWUSR)
-        # patch install_name on darwin, instead of using rpath
-        cmd = ['install_name_tool', '-id', '@loader_path/../%s'%libcapnp, local]
-        try:
-            p = Popen(cmd, stdout=PIPE,stderr=PIPE)
-        except OSError:
-            fatal("install_name_tool not found, cannot patch libcapnp for bundling.")
-        out,err = p.communicate()
-        if p.returncode:
-            fatal("Could not patch bundled libcapnp install_name: %s"%err, p.returncode)
 

buildutils/config.py

@@ -1,5 +1,5 @@
 """Config functions"""
-#-----------------------------------------------------------------------------
+#
 #  Copyright (C) PyZMQ Developers
 #
 #  This file is part of pyzmq, copied and adapted from h5py.
@@ -9,149 +9,12 @@
 #
 #  Distributed under the terms of the New BSD License.  The full license is in
 #  the file COPYING.BSD, distributed as part of this software.
-#-----------------------------------------------------------------------------
+#
 
-import sys
+#
-import os
-import json
-
-try:
-    from configparser import ConfigParser
-except:
-    from ConfigParser import ConfigParser
-
-pjoin = os.path.join
-from .msg import debug, fatal, warn
-
-#-----------------------------------------------------------------------------
 # Utility functions (adapted from h5py: http://h5py.googlecode.com)
-#-----------------------------------------------------------------------------
+#
-
-
-def load_config(name, base='conf'):
-    """Load config dict from JSON"""
-    fname = pjoin(base, name + '.json')
-    if not os.path.exists(fname):
-        return {}
-    try:
-        with open(fname) as f:
-            cfg = json.load(f)
-    except Exception as e:
-        warn("Couldn't load %s: %s" % (fname, e))
-        cfg = {}
-    return cfg
-
-
-def save_config(name, data, base='conf'):
-    """Save config dict to JSON"""
-    if not os.path.exists(base):
-        os.mkdir(base)
-    fname = pjoin(base, name+'.json')
-    with open(fname, 'w') as f:
-        json.dump(data, f, indent=2)
-
 
 def v_str(v_tuple):
     """turn (2,0,1) into '2.0.1'."""
     return ".".join(str(x) for x in v_tuple)
-
-def get_eargs():
-    """ Look for options in environment vars """
-
-    settings = {}
-
-    zmq = os.environ.get("ZMQ_PREFIX", None)
-    if zmq is not None:
-        debug("Found environ var ZMQ_PREFIX=%s" % zmq)
-        settings['zmq_prefix'] = zmq
-
-    return settings
-
-def cfg2dict(cfg):
-    """turn a ConfigParser into a nested dict
-    
-    because ConfigParser objects are dumb.
-    """
-    d = {}
-    for section in cfg.sections():
-        d[section] = dict(cfg.items(section))
-    return d
-
-def get_cfg_args():
-    """ Look for options in setup.cfg """
-
-    if not os.path.exists('setup.cfg'):
-        return {}
-    cfg = ConfigParser()
-    cfg.read('setup.cfg')
-    cfg = cfg2dict(cfg)
-
-    g = cfg.setdefault('global', {})
-    # boolean keys:
-    for key in ['libzmq_extension',
-                'bundle_libzmq_dylib',
-                'no_libzmq_extension',
-                'have_sys_un_h',
-                'skip_check_zmq',
-                ]:
-        if key in g:
-            g[key] = eval(g[key])
-
-    # globals go to top level
-    cfg.update(cfg.pop('global'))
-    return cfg
-
-def config_from_prefix(prefix):
-    """Get config from zmq prefix"""
-    settings = {}
-    if prefix.lower() in ('default', 'auto', ''):
-        settings['zmq_prefix'] = ''
-        settings['libzmq_extension'] = False
-        settings['no_libzmq_extension'] = False
-    elif prefix.lower() in ('bundled', 'extension'):
-        settings['zmq_prefix'] = ''
-        settings['libzmq_extension'] = True
-        settings['no_libzmq_extension'] = False
-    else:
-        settings['zmq_prefix'] = prefix
-        settings['libzmq_extension'] = False
-        settings['no_libzmq_extension'] = True
-    return settings
-
-def merge(into, d):
-    """merge two containers
-    
-    into is updated, d has priority
-    """
-    if isinstance(into, dict):
-        for key in d.keys():
-            if key not in into:
-                into[key] = d[key]
-            else:
-                into[key] = merge(into[key], d[key])
-        return into
-    elif isinstance(into, list):
-        return into + d
-    else:
-        return d
-
-def discover_settings(conf_base=None):
-    """ Discover custom settings for ZMQ path"""
-    settings = {
-        'zmq_prefix': '',
-        'libzmq_extension': False,
-        'no_libzmq_extension': False,
-        'skip_check_zmq': False,
-        'build_ext': {},
-        'bdist_egg': {},
-    }
-    if sys.platform.startswith('win'):
-        settings['have_sys_un_h'] = False
-    
-    if conf_base:
-        # lowest priority
-        merge(settings, load_config('config', conf_base))
-    merge(settings, get_cfg_args())
-    merge(settings, get_eargs())
-    
-    return settings

buildutils/constants.py

@@ -1,83 +0,0 @@
-"""
-script for generating files that involve repetitive updates for zmq constants.
-
-Run this after updating utils/constant_names
-
-Currently generates the following files from templates:
-
-- constant_enums.pxi
-- constants.pxi
-- zmq_constants.h
-
-"""
-
-# Copyright (C) PyZMQ Developers
-# Distributed under the terms of the Modified BSD License.
-
-import os
-import sys
-
-from . import info
-pjoin = os.path.join
-
-root = os.path.abspath(pjoin(os.path.dirname(__file__), os.path.pardir))
-
-sys.path.insert(0, pjoin(root, 'zmq', 'utils'))
-from constant_names import all_names, no_prefix
-
-ifndef_t = """#ifndef {0}
-    #define {0} (_PYZMQ_UNDEFINED)
-#endif
-"""
-
-def cython_enums():
-    """generate `enum: ZMQ_CONST` block for constant_enums.pxi"""
-    lines = []
-    for name in all_names:
-        if no_prefix(name):
-            lines.append('enum: ZMQ_{0} "{0}"'.format(name))
-        else:
-            lines.append('enum: ZMQ_{0}'.format(name))
-            
-    return dict(ZMQ_ENUMS='\n    '.join(lines))
-
-def ifndefs():
-    """generate `#ifndef ZMQ_CONST` block for zmq_constants.h"""
-    lines = ['#define _PYZMQ_UNDEFINED (-9999)']
-    for name in all_names:
-        if not no_prefix(name):
-            name = 'ZMQ_%s' % name
-        lines.append(ifndef_t.format(name))
-    return dict(ZMQ_IFNDEFS='\n'.join(lines))
-
-def constants_pyx():
-    """generate CONST = ZMQ_CONST and __all__ for constants.pxi"""
-    all_lines = []
-    assign_lines = []
-    for name in all_names:
-        if name == "NULL":
-            # avoid conflict with NULL in Cython
-            assign_lines.append("globals()['NULL'] = ZMQ_NULL")
-        else:
-            assign_lines.append('{0} = ZMQ_{0}'.format(name))
-        all_lines.append('  "{0}",'.format(name))
-    return dict(ASSIGNMENTS='\n'.join(assign_lines), ALL='\n'.join(all_lines))
-
-def generate_file(fname, ns_func, dest_dir="."):
-    """generate a constants file from its template"""
-    with open(pjoin(root, 'buildutils', 'templates', '%s' % fname), 'r') as f:
-        tpl = f.read()
-    out = tpl.format(**ns_func())
-    dest = pjoin(dest_dir, fname)
-    info("generating %s from template" % dest)
-    with open(dest, 'w') as f:
-        f.write(out)
-
-def render_constants():
-    """render generated constant files from templates"""
-    generate_file("constant_enums.pxi", cython_enums, pjoin(root, 'zmq', 'backend', 'cython'))
-    generate_file("constants.pxi", constants_pyx, pjoin(root, 'zmq', 'backend', 'cython'))
-    generate_file("zmq_constants.h", ifndefs, pjoin(root, 'zmq', 'utils'))
-
-if __name__ == '__main__':
-    render_constants()

buildutils/detect.py

@@ -1,5 +1,5 @@
 """Detect zmq version"""
-#-----------------------------------------------------------------------------
+#
 #  Copyright (C) PyZMQ Developers
 #
 #  This file is part of pyzmq, copied and adapted from h5py.
@@ -9,7 +9,7 @@
 #
 #  Distributed under the terms of the New BSD License.  The full license is in
 #  the file COPYING.BSD, distributed as part of this software.
-#-----------------------------------------------------------------------------
+#
 #
 # Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq
 # for original project.
@@ -21,7 +21,6 @@ import logging
 import platform
 from distutils import ccompiler
 from distutils.ccompiler import get_default_compiler
-from subprocess import Popen, PIPE
 import tempfile
 
 from .misc import get_compiler, get_output_error
@@ -29,15 +28,15 @@ from .patch import patch_lib_paths
 
 pjoin = os.path.join
 
-#-----------------------------------------------------------------------------
+#
 # Utility functions (adapted from h5py: http://h5py.googlecode.com)
-#-----------------------------------------------------------------------------
+#
 
 def test_compilation(cfile, compiler=None, **compiler_attrs):
     """Test simple compilation with given settings"""
     cc = get_compiler(compiler, **compiler_attrs)
 
-    efile, ext = os.path.splitext(cfile)
+    efile, _ = os.path.splitext(cfile)
 
     cpreargs = lpreargs = []
     if sys.platform == 'darwin':
@@ -57,32 +56,17 @@ def test_compilation(cfile, compiler=None, **compiler_attrs):
             lpreargs = ['-m32']
         else:
             lpreargs = ['-m64']
-    extra = compiler_attrs.get('extra_compile_args', [])
+    extra_compile_args = compiler_attrs.get('extra_compile_args', [])
-    extra += ['--std=c++11']
+    if os.name != 'nt':
+        extra_compile_args += ['--std=c++14']
+    extra_link_args = compiler_attrs.get('extra_link_args', [])
+    if cc.compiler_type == 'msvc':
+        extra_link_args += ['/MANIFEST']
 
-    objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra)
+    objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra_compile_args)
-    cc.link_executable(objs, efile, extra_preargs=lpreargs)
+    cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args)
     return efile
 
-def compile_and_run(basedir, src, compiler=None, **compiler_attrs):
-    if not os.path.exists(basedir):
-        os.makedirs(basedir)
-    cfile = pjoin(basedir, os.path.basename(src))
-    shutil.copy(src, cfile)
-    try:
-        cc = get_compiler(compiler, **compiler_attrs)
-        efile = test_compilation(cfile, compiler=cc)
-        patch_lib_paths(efile, cc.library_dirs)
-        result = Popen(efile, stdout=PIPE, stderr=PIPE)
-        so, se = result.communicate()
-        # for py3k:
-        so = so.decode()
-        se = se.decode()
-    finally:
-        shutil.rmtree(basedir)
-
-    return result.returncode, so, se
-
 
 def detect_version(basedir, compiler=None, **compiler_attrs):
     """Compile, link & execute a test program, in empty directory `basedir`.
@@ -144,7 +128,7 @@ def detect_version(basedir, compiler=None, **compiler_attrs):
     return props
 
 
-def test_build():
+def test_build(**compiler_attrs):
     """do a test build of libcapnp"""
     tmp_dir = tempfile.mkdtemp()
 
@@ -152,7 +136,7 @@ def test_build():
     # info("Configure: Autodetecting Cap'n Proto settings...")
     # info("    Custom Cap'n Proto dir:       %s" % prefix)
     try:
-        detected = detect_version(tmp_dir)
+        detected = detect_version(tmp_dir, None, **compiler_attrs)
     finally:
         erase_dir(tmp_dir)
 
@@ -161,8 +145,9 @@ def test_build():
     return detected
 
 
-def erase_dir(dir):
+def erase_dir(path):
+    """Erase directory"""
     try:
-        shutil.rmtree(dir)
+        shutil.rmtree(path)
     except Exception:
         pass

buildutils/misc.py

@@ -4,7 +4,6 @@
 # Distributed under the terms of the Modified BSD License.
 
 import os
-import sys
 import logging
 from distutils import ccompiler
 from distutils.sysconfig import customize_compiler
@@ -14,13 +13,8 @@ from subprocess import Popen, PIPE
 pjoin = os.path.join
 
 
-if sys.version_info[0] >= 3:
-    u = lambda x: x
-else:
-    u = lambda x: x.decode('utf8', 'replace')
-
-
 def customize_mingw(cc):
+    """customize mingw"""
     # strip -mno-cygwin from mingw32 (Python Issue #12641)
     for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]:
         if '-mno-cygwin' in cmd:
@@ -30,14 +24,18 @@ def customize_mingw(cc):
     if 'msvcr90' in cc.dll_libraries:
         cc.dll_libraries.remove('msvcr90')
 
+def customize_msvc(cc):
+    pass
 
 def get_compiler(compiler, **compiler_attrs):
     """get and customize a compiler"""
     if compiler is None or isinstance(compiler, str):
         cc = ccompiler.new_compiler(compiler=compiler)
-        # customize_compiler(cc)
+        customize_compiler(cc)
         if cc.compiler_type == 'mingw32':
             customize_mingw(cc)
+        elif cc.compiler_type == 'msvc':
+            customize_msvc(cc)
     else:
         cc = compiler
 
@@ -55,11 +53,10 @@ def get_output_error(cmd):
     try:
         result = Popen(cmd, stdout=PIPE, stderr=PIPE)
     except IOError as e:
-        return -1, u(''), u('Failed to run %r: %r' % (cmd, e))
+        return -1, '', 'Failed to run %r: %r' % (cmd, e)
     so, se = result.communicate()
     # unicode:
     so = so.decode('utf8', 'replace')
     se = se.decode('utf8', 'replace')
 
     return result.returncode, so, se
-

buildutils/msg.py

@@ -9,9 +9,9 @@ import os
 import sys
 import logging
 
-#-----------------------------------------------------------------------------
+#
 # Logging (adapted from h5py: http://h5py.googlecode.com)
-#-----------------------------------------------------------------------------
+#
 
 
 logger = logging.getLogger()
@@ -22,18 +22,22 @@ else:
 logger.addHandler(logging.StreamHandler(sys.stderr))
 
 def debug(msg):
+    """Debug"""
     logger.debug(msg)
 
 def info(msg):
+    """Info"""
     logger.info(msg)
 
 def fatal(msg, code=1):
-    logger.error("Fatal: " + msg)
+    """Fatal"""
+    logger.error("Fatal: %s", msg)
     exit(code)
 
 def warn(msg):
-    logger.error("Warning: " + msg)
+    """Warning"""
+    logger.error("Warning: %s", msg)
 
 def line(c='*', width=48):
+    """Horizontal rule"""
     print(c * (width // len(c)))
-

buildutils/patch.py

@@ -20,7 +20,7 @@ LIB_PAT = re.compile(r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), "
 def _get_libs(fname):
     rc, so, se = get_output_error(['otool', '-L', fname])
     if rc:
-        logging.error("otool -L %s failed: %r" % (fname, se))
+        logging.error("otool -L %s failed: %r", fname, se)
         return
     for line in so.splitlines()[1:]:
         m = LIB_PAT.match(line)
@@ -33,6 +33,7 @@ def _find_library(lib, path):
         real_lib = os.path.join(d, lib)
         if os.path.exists(real_lib):
             return real_lib
+    return None
 
 def _install_name_change(fname, lib, real_lib):
     rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname])

buildutils/setup_travis.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-set -exo pipefail
-
-CAPNP_VERSION=0.5.1.2
-
-sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-sudo apt-get -qq update
-sudo apt-get -qq install g++-4.8 libstdc++-4.8-dev
-sudo update-alternatives --quiet --install /usr/bin/gcc  gcc  /usr/bin/gcc-4.8  60 --slave   /usr/bin/g++  g++  /usr/bin/g++-4.8 --slave   /usr/bin/gcov gcov /usr/bin/gcov-4.8
-sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8
-
-if ! [ -z "${BUILD_CAPNP}" ]; then
-  wget https://capnproto.org/capnproto-c++-${CAPNP_VERSION}.tar.gz && tar xzvf capnproto-c++-${CAPNP_VERSION}.tar.gz && cd capnproto-c++-${CAPNP_VERSION} && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd ..
-fi

capnp/__init__.py

@@ -31,8 +31,26 @@ Example Usage::
         for phone in person.phones:
             print(phone.type, ':', phone.number)
 """
+# flake8: noqa F401 F403 F405
 from .version import version as __version__
 from .lib.capnp import *
-from .lib.capnp import _DynamicStructReader, _DynamicStructBuilder, _DynamicResizableListBuilder, _DynamicListReader, _DynamicListBuilder, _DynamicOrphan, _DynamicResizableListBuilder, _MallocMessageBuilder, _PackedFdMessageReader, _StreamFdMessageReader, _write_message_to_fd, _write_packed_message_to_fd, _StructModule, _InterfaceModule, _DynamicCapabilityClient, _CapabilityClient, _EventLoop
+from .lib.capnp import (
+    _CapabilityClient,
+    _DynamicCapabilityClient,
+    _DynamicListBuilder,
+    _DynamicListReader,
+    _DynamicOrphan,
+    _DynamicResizableListBuilder,
+    _DynamicStructBuilder,
+    _DynamicStructReader,
+    _EventLoop,
+    _InterfaceModule,
+    _MallocMessageBuilder,
+    _PackedFdMessageReader,
+    _StreamFdMessageReader,
+    _StructModule,
+    _write_message_to_fd,
+    _write_packed_message_to_fd,
+)
 
 add_import_hook() # enable import hook by default

capnp/helpers/asyncHelper.h

@@ -13,11 +13,13 @@ public:
   virtual bool wait() {
     GILAcquire gil;
     PyObject_CallMethod(py_event_port, const_cast<char *>("wait"), NULL);
+    return true;  // TODO: get the bool result from python
   }
 
   virtual bool poll() {
     GILAcquire gil;
     PyObject_CallMethod(py_event_port, const_cast<char *>("poll"), NULL);
+    return true;  // TODO: get the bool result from python
   }
 
   virtual void setRunnable(bool runnable) {
@@ -37,6 +39,11 @@ void waitNeverDone(kj::WaitScope & scope) {
   kj::NEVER_DONE.wait(scope);
 }
 
+void pollWaitScope(kj::WaitScope & scope) {
+  GILRelease gil;
+  scope.poll();
+}
+
 kj::Timer * getTimer(kj::AsyncIoContext * context) {
   return &context->lowLevelProvider->getTimer();
 }
@@ -55,3 +62,8 @@ capnp::Response< ::capnp::DynamicStruct> * waitRemote(capnp::RemotePromise< ::ca
   GILRelease gil;
   return new capnp::Response< ::capnp::DynamicStruct>(promise->wait(scope));
 }
+
+bool pollRemote(capnp::RemotePromise< ::capnp::DynamicStruct> * promise, kj::WaitScope & scope) {
+  GILRelease gil;
+  return promise->poll(scope);
+}

capnp/helpers/asyncIoHelper.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "kj/async.h"
+#include "kj/async-io.h"
+
+class AsyncIoStreamReadHelper {
+public:
+  AsyncIoStreamReadHelper(kj::AsyncIoStream * _stream, kj::WaitScope * _scope, size_t bufsize) {
+    io_stream = _stream;
+    wait_scope = _scope;
+    ready = false;
+    buffer_read_size = 0;
+    buffer = new unsigned char[bufsize];
+    promise = io_stream->read(buffer, 1, bufsize);
+  }
+
+  ~AsyncIoStreamReadHelper() {
+    delete[] buffer;
+  }
+
+  bool poll() {
+    bool result = promise.poll(*wait_scope);
+    if (result) {
+      ready = true;
+      buffer_read_size = promise.wait(*wait_scope);
+    }
+    return result;
+  }
+
+  size_t read_size() {
+    if (!ready) {
+      return 0;
+    }
+    return buffer_read_size;
+  }
+
+  void * read_buffer() {
+    if (!ready) {
+      return 0;
+    }
+    return buffer;
+  }
+
+private:
+  kj::AsyncIoStream * io_stream;
+  kj::WaitScope * wait_scope;
+  kj::Promise<size_t> promise = nullptr;
+
+  unsigned char *buffer;
+  size_t buffer_read_size;
+
+  bool ready;
+};

capnp/helpers/capabilityHelper.h

@@ -66,6 +66,7 @@ void reraise_kj_exception() {
   catch (kj::Exception& exn) {
     auto obj = wrap_kj_exception_for_reraise(exn);
     PyErr_SetObject((PyObject*)obj->ob_type, obj);
+    Py_DECREF(obj);
   }
   catch (const std::exception& exn) {
     PyErr_SetString(PyExc_RuntimeError, exn.what());

capnp/helpers/checkCompiler.h

@@ -1,11 +1,8 @@
-#ifdef __GNUC__
+#ifdef _MSC_VER
-  #if __clang__
+#pragma comment(lib, "Ws2_32.lib")
-    #if __cplusplus >= 201103L && !__has_include(<initializer_list>)
+#pragma comment(lib, "advapi32.lib")
-      #warning "Your compiler supports C++11 but your C++ standard library does not.  If your system has libc++ installed (as should be the case on e.g. Mac OSX), try adding -stdlib=libc++ to your CFLAGS (ignore the other warning that says to use CXXFLAGS)."
-    #endif
-  #endif
 #endif
 
 #include "capnp/dynamic.h"
 
-static_assert(CAPNP_VERSION >= 5000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.5 and then re-install this python library");
+static_assert(CAPNP_VERSION >= 7000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.7 and then re-install this python library");

capnp/helpers/helpers.pxd

@@ -1,11 +1,13 @@
-from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, PyRestorer, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer
+from capnp.includes.capnp_cpp cimport Maybe, DynamicStruct, Request, Response, PyPromise, VoidPromise, PyPromiseArray, RemotePromise, DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, RpcSystem, MessageBuilder, MessageReader, TwoPartyVatNetwork, AnyPointer, DynamicStruct_Builder, WaitScope, AsyncIoContext, StringPtr, TaskSet, Timer
 
 from capnp.includes.schema_cpp cimport ByteArray
 
-from non_circular cimport reraise_kj_exception
+from non_circular cimport reraise_kj_exception, AsyncIoStreamReadHelper
 
 from cpython.ref cimport PyObject
 
+from libcpp cimport bool
+
 cdef extern from "capnp/helpers/fixMaybe.h":
     EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception
     StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception
@@ -25,14 +27,8 @@ cdef extern from "capnp/helpers/capabilityHelper.h":
     VoidPromise convert_to_voidpromise(PyPromise&)
 
 cdef extern from "capnp/helpers/rpcHelper.h":
-    Capability.Client restoreHelper(RpcSystem&)
-    Capability.Client restoreHelper(RpcSystem&, MessageBuilder&)
-    Capability.Client restoreHelper(RpcSystem&, MessageReader&)
-    Capability.Client restoreHelper(RpcSystem&, AnyPointer.Reader&)
-    Capability.Client restoreHelper(RpcSystem&, AnyPointer.Builder&)
     Capability.Client bootstrapHelper(RpcSystem&)
-    RpcSystem makeRpcClientWithRestorer(TwoPartyVatNetwork&, PyRestorer&)
+    Capability.Client bootstrapHelperServer(RpcSystem&)
-    PyPromise connectServerRestorer(TaskSet &, PyRestorer &, AsyncIoContext *, StringPtr)
     PyPromise connectServer(TaskSet &, Capability.Client, AsyncIoContext *, StringPtr)
 
 cdef extern from "capnp/helpers/serialize.h":
@@ -40,7 +36,9 @@ cdef extern from "capnp/helpers/serialize.h":
 
 cdef extern from "capnp/helpers/asyncHelper.h":
     void waitNeverDone(WaitScope&)
+    void pollWaitScope(WaitScope&)
     Response * waitRemote(RemotePromise *, WaitScope&)
+    bool pollRemote(RemotePromise *, WaitScope&)
     PyObject * waitPyPromise(PyPromise *, WaitScope&)
     void waitVoidPromise(VoidPromise *, WaitScope&)
     Timer * getTimer(AsyncIoContext *) except +reraise_kj_exception

capnp/helpers/non_circular.pxd

@@ -1,4 +1,6 @@
 from cpython.ref cimport PyObject
+from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope
+from libcpp cimport bool
 
 cdef extern from "capnp/helpers/capabilityHelper.h":
     cppclass PythonInterfaceDynamicImpl:
@@ -10,11 +12,16 @@ cdef extern from "capnp/helpers/capabilityHelper.h":
         PyRefCounter(PyObject *)
 
 cdef extern from "capnp/helpers/rpcHelper.h":
-    cdef cppclass PyRestorer:
-        PyRestorer(PyObject *)
     cdef cppclass ErrorHandler:
         pass
 
 cdef extern from "capnp/helpers/asyncHelper.h":
     cdef cppclass PyEventPort:
         PyEventPort(PyObject *)
+
+cdef extern from "capnp/helpers/asyncIoHelper.h":
+    cdef cppclass AsyncIoStreamReadHelper:
+        AsyncIoStreamReadHelper(AsyncIoStream *, WaitScope *, size_t)
+        bool poll()
+        size_t read_size()
+        void* read_buffer()

capnp/helpers/rpcHelper.h

@@ -6,73 +6,6 @@
 #include "Python.h"
 #include "capabilityHelper.h"
 
-extern "C" {
-   capnp::Capability::Client * call_py_restorer(PyObject *, capnp::AnyPointer::Reader &);
-}
-
-class PyRestorer final: public capnp::SturdyRefRestorer<capnp::AnyPointer> {
-public:
-  PyRestorer(PyObject * _py_restorer): py_restorer(_py_restorer) {
-    // We don't need to incref/decref, since this C++ class will be owned by the Python wrapper class, and we'll make sure the python class doesn't refcount to 0 elsewhere.
-    // Py_INCREF(py_restorer);
-  }
-
-  // ~PyRestorer() {
-  //   Py_DECREF(py_restorer);
-  // }
-
-  capnp::Capability::Client restore(capnp::AnyPointer::Reader objectId) override {
-    GILAcquire gil;
-    capnp::Capability::Client * ret = call_py_restorer(py_restorer, objectId);
-    check_py_error();
-    capnp::Capability::Client stack_ret(*ret);
-    delete ret;
-
-    return stack_ret;
-  }
-
-private:
-  PyObject * py_restorer;
-};
-
-capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::MessageBuilder & objectId) {
-    capnp::MallocMessageBuilder hostIdMessage(8);
-    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
-    return client.restore(hostId, objectId.getRoot<capnp::AnyPointer>());
-}
-
-capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::MessageReader & objectId) {
-    capnp::MallocMessageBuilder hostIdMessage(8);
-    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
-    return client.restore(hostId, objectId.getRoot<capnp::AnyPointer>());
-}
-
-capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::AnyPointer::Reader & objectId) {
-    capnp::MallocMessageBuilder hostIdMessage(8);
-    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
-    return client.restore(hostId, objectId);
-}
-
-capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client, capnp::AnyPointer::Builder & objectId) {
-    capnp::MallocMessageBuilder hostIdMessage(8);
-    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
-    return client.restore(hostId, objectId);
-}
-
-capnp::Capability::Client restoreHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
-    capnp::MallocMessageBuilder hostIdMessage(8);
-    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    hostId.setSide(capnp::rpc::twoparty::Side::SERVER);
-
-    capnp::MallocMessageBuilder blankMessage(8);
-    auto objectId = blankMessage.getRoot<capnp::AnyPointer>();
-    return client.restore(hostId, objectId);
-}
-
 capnp::Capability::Client bootstrapHelper(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
     capnp::MallocMessageBuilder hostIdMessage(8);
     auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
@@ -80,63 +13,19 @@ capnp::Capability::Client bootstrapHelper(capnp::RpcSystem<capnp::rpc::twoparty:
     return client.bootstrap(hostId);
 }
 
-template <typename SturdyRefHostId, typename ProvisionId,
+capnp::Capability::Client bootstrapHelperServer(capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId>& client) {
-          typename RecipientId, typename ThirdPartyCapId, typename JoinAnswer>
+    capnp::MallocMessageBuilder hostIdMessage(8);
-capnp::RpcSystem<SturdyRefHostId> makeRpcClientWithRestorer(
+    auto hostId = hostIdMessage.initRoot<capnp::rpc::twoparty::SturdyRefHostId>();
-    capnp::VatNetwork<SturdyRefHostId, ProvisionId, RecipientId, ThirdPartyCapId, JoinAnswer>& network,
+    hostId.setSide(capnp::rpc::twoparty::Side::CLIENT);
-    PyRestorer& restorer) {
+    return client.bootstrap(hostId);
-    using namespace capnp;
-  return RpcSystem<SturdyRefHostId>(network, restorer);
 }
 
-struct ServerContextRestorer {
-  kj::Own<kj::AsyncIoStream> stream;
-  capnp::TwoPartyVatNetwork network;
-  capnp::RpcSystem<capnp::rpc::twoparty::SturdyRefHostId> rpcSystem;
-
-  ServerContextRestorer(kj::Own<kj::AsyncIoStream>&& stream, capnp::SturdyRefRestorer<capnp::AnyPointer>& restorer)
-      : stream(kj::mv(stream)),
-        network(*this->stream, capnp::rpc::twoparty::Side::SERVER),
-        rpcSystem(makeRpcServer(network, restorer)) {}
-};
-
 class ErrorHandler : public kj::TaskSet::ErrorHandler {
   void taskFailed(kj::Exception&& exception) override {
     kj::throwFatalException(kj::mv(exception));
   }
 };
 
-void acceptLoopRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::Own<kj::ConnectionReceiver>&& listener) {
-  auto ptr = listener.get();
-  tasks.add(ptr->accept().then(kj::mvCapture(kj::mv(listener),
-      [&](kj::Own<kj::ConnectionReceiver>&& listener,
-             kj::Own<kj::AsyncIoStream>&& connection) {
-    acceptLoopRestorer(tasks, restorer, kj::mv(listener));
-
-    auto server = kj::heap<ServerContextRestorer>(kj::mv(connection), restorer);
-
-    // Arrange to destroy the server context when all references are gone, or when the
-    // EzRpcServer is destroyed (which will destroy the TaskSet).
-    tasks.add(server->network.onDisconnect().attach(kj::mv(server)));
-  })));
-}
-
-kj::Promise<PyObject *> connectServerRestorer(kj::TaskSet & tasks, PyRestorer & restorer, kj::AsyncIoContext * context, kj::StringPtr bindAddress) {
-    auto paf = kj::newPromiseAndFulfiller<unsigned int>();
-    auto portPromise = paf.promise.fork();
-
-    tasks.add(context->provider->getNetwork().parseAddress(bindAddress)
-        .then(kj::mvCapture(paf.fulfiller,
-          [&](kj::Own<kj::PromiseFulfiller<unsigned int>>&& portFulfiller,
-                 kj::Own<kj::NetworkAddress>&& addr) {
-      auto listener = addr->listen();
-      portFulfiller->fulfill(listener->getPort());
-      acceptLoopRestorer(tasks, restorer, kj::mv(listener));
-    })));
-
-    return portPromise.addBranch().then([&](unsigned int port) { return PyLong_FromUnsignedLong(port); });
-}
-
 
 struct ServerContext {
   kj::Own<kj::AsyncIoStream> stream;

capnp/includes/capnp_cpp.pxd

@@ -1,11 +1,11 @@
 # schema.capnp.cpp.pyx
 # distutils: language = c++
-# distutils: extra_compile_args = --std=c++11
 cdef extern from "capnp/helpers/checkCompiler.h":
     pass
 
-from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader
+from libcpp cimport bool
-from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyRestorer, PyEventPort, ErrorHandler
+from schema_cpp cimport Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions
+from capnp.helpers.non_circular cimport PythonInterfaceDynamicImpl, reraise_kj_exception, PyRefCounter, PyEventPort, ErrorHandler
 from capnp.includes.types cimport *
 
 cdef extern from "capnp/common.h" namespace " ::capnp":
@@ -45,7 +45,8 @@ cdef extern from "kj/exception.h" namespace " ::kj":
 cdef extern from "kj/memory.h" namespace " ::kj":
     cdef cppclass Own[T]:
         T& operator*()
-    Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side)
+        T* get()
+    Own[TwoPartyVatNetwork] makeTwoPartyVatNetwork" ::kj::heap< ::capnp::TwoPartyVatNetwork>"(AsyncIoStream& stream, Side, ReaderOptions)
     Own[PromiseFulfillerPair] copyPromiseFulfillerPair" ::kj::heap< ::kj::PromiseFulfillerPair<void> >"(PromiseFulfillerPair&)
     Own[PyRefCounter] makePyRefCounter" ::kj::heap< PyRefCounter >"(PyObject *)
 
@@ -55,6 +56,7 @@ cdef extern from "kj/async.h" namespace " ::kj":
         Promise(Promise)
         Promise(T)
         T wait(WaitScope)
+        bool poll(WaitScope)
         # ForkedPromise<T> fork()
         # Promise<T> exclusiveJoin(Promise<T>&& other)
         # Promise[T] eagerlyEvaluate()
@@ -101,7 +103,14 @@ ctypedef Promise[PyArray] PyPromiseArray
 
 cdef extern from "kj/time.h" namespace " ::kj":
     cdef cppclass Duration:
-        Duration(int64_t)
+        Duration operator*(int64_t)
+    Duration NANOSECONDS
+    Duration MICROSECONDS
+    Duration MILLISECONDS
+    Duration SECONDS
+    Duration MINUTES
+    Duration HOURS
+    Duration DAYS
     # cdef cppclass TimePoint:
     #     TimePoint(Duration)
     cdef cppclass Timer:
@@ -109,18 +118,26 @@ cdef extern from "kj/time.h" namespace " ::kj":
         # VoidPromise atTime(TimePoint time)
         VoidPromise afterDelay(Duration delay)
 
+cdef inline Duration Nanoseconds(int64_t nanos):
+    return NANOSECONDS * nanos
+
 cdef extern from "kj/async-io.h" namespace " ::kj":
     cdef cppclass AsyncIoStream:
-        pass
+        Promise[size_t] read(void*, size_t, size_t)
+        Promise[void] write(const void*, size_t)
+
     cdef cppclass LowLevelAsyncIoProvider:
         # Own[AsyncInputStream] wrapInputFd(int)
         # Own[AsyncOutputStream] wrapOutputFd(int)
         Own[AsyncIoStream] wrapSocketFd(int)
         Timer& getTimer() except +reraise_kj_exception
+
     cdef cppclass AsyncIoProvider:
-        pass
+        TwoWayPipe newTwoWayPipe()
+
     cdef cppclass WaitScope:
         pass
+
     cdef cppclass AsyncIoContext:
         AsyncIoContext(AsyncIoContext&)
         Own[LowLevelAsyncIoProvider] lowLevelProvider
@@ -130,6 +147,9 @@ cdef extern from "kj/async-io.h" namespace " ::kj":
     cdef cppclass TaskSet:
         TaskSet(ErrorHandler &)
 
+    cdef cppclass TwoWayPipe:
+        Own[AsyncIoStream] ends[2]
+
     AsyncIoContext setupAsyncIo()
 
 cdef extern from "capnp/schema.capnp.h" namespace " ::capnp":
@@ -341,10 +361,9 @@ cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp":
     cdef Side SERVER" ::capnp::rpc::twoparty::Side::SERVER"
 
     cdef cppclass TwoPartyVatNetwork:
-        TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side)
+        TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side, ReaderOptions)
         VoidPromise onDisconnect()
         VoidPromise onDrained()
-    RpcSystem makeRpcServer(TwoPartyVatNetwork&, PyRestorer&)
     RpcSystem makeRpcServerBootstrap"makeRpcServer"(TwoPartyVatNetwork&, Capability.Client)
     RpcSystem makeRpcClient(TwoPartyVatNetwork&)
 
@@ -427,6 +446,7 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp":
             Reader(DynamicStruct.Reader& value)
             Reader(DynamicCapability.Client& value)
             Reader(PythonInterfaceDynamicImpl& value)
+            Reader(AnyPointer.Reader& value)
             Type getType()
             int64_t asInt"as<int64_t>"()
             uint64_t asUint"as<uint64_t>"()

capnp/includes/schema_cpp.pxd

@@ -1,6 +1,5 @@
 # schema.capnp.cpp.pyx
 # distutils: language = c++
-# distutils: extra_compile_args = --std=c++11
 
 from libc.stdint cimport *
 from capnp_cpp cimport DynamicOrphan
@@ -628,6 +627,11 @@ cdef extern from "capnp/schema.capnp.h" namespace " ::capnp::schema":
             void setId(UInt64)
             Value getValue()
             void setValue(Value)
+    cdef cppclass ListNestedNodeReader"capnp::List<capnp::schema::Node::NestedNode>::Reader":
+       ListNestedNodeReader()
+       ListNestedNodeReader(ListNestedNodeReader)
+       Node.NestedNode.Reader operator[](uint)
+       uint size()
 
 cdef extern from "capnp/message.h" namespace " ::capnp":
     cdef cppclass ReaderOptions:
@@ -662,6 +666,8 @@ cdef extern from "capnp/message.h" namespace " ::capnp":
         DynamicStruct_Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema)
         void setRootDynamicStruct'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader)
 
+        ConstWordArrayArrayPtr getSegmentsForOutput'getSegmentsForOutput'()
+
         AnyPointer.Builder getRootAnyPointer'getRoot< ::capnp::AnyPointer>'()
 
         DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema)
@@ -686,6 +692,10 @@ cdef extern from "capnp/message.h" namespace " ::capnp":
         MallocMessageBuilder()
         MallocMessageBuilder(int)
 
+    cdef cppclass SegmentArrayMessageReader(MessageReader):
+        SegmentArrayMessageReader(ConstWordArrayArrayPtr array) except +reraise_kj_exception
+        SegmentArrayMessageReader(ConstWordArrayArrayPtr array, ReaderOptions) except +reraise_kj_exception
+
     cdef cppclass FlatMessageBuilder(MessageBuilder):
         FlatMessageBuilder(WordArrayPtr array)
         FlatMessageBuilder(WordArrayPtr array, ReaderOptions)
@@ -709,6 +719,16 @@ cdef extern from "kj/common.h" namespace " ::kj":
         ByteArrayPtr(byte *, size_t size)
         size_t size()
         byte& operator[](size_t index)
+    cdef cppclass ConstWordArrayPtr " ::kj::ArrayPtr< const ::capnp::word>":
+        ConstWordArrayPtr()
+        ConstWordArrayPtr(word *, size_t size)
+        size_t size()
+        const word* begin()
+    cdef cppclass ConstWordArrayArrayPtr " ::kj::ArrayPtr< const ::kj::ArrayPtr< const ::capnp::word>>":
+        ConstWordArrayArrayPtr()
+        ConstWordArrayArrayPtr(ConstWordArrayPtr*, size_t size)
+        size_t size()
+        ConstWordArrayPtr& operator[](size_t index)
 
 cdef extern from "kj/array.h" namespace " ::kj":
     # Cython can't handle Array[word] as a function argument
@@ -765,6 +785,7 @@ cdef extern from "capnp/serialize.h" namespace " ::capnp":
     cdef cppclass FlatArrayMessageReader(MessageReader):
         FlatArrayMessageReader(WordArrayPtr array) except +reraise_kj_exception
         FlatArrayMessageReader(WordArrayPtr array, ReaderOptions) except +reraise_kj_exception
+        const word* getEnd() const
 
     void writeMessageToFd(int, MessageBuilder&) except +reraise_kj_exception
 

capnp/lib/capnp.pxd

@@ -1,6 +1,8 @@
+# cython: language_level = 2
+
 from capnp.includes cimport capnp_cpp as capnp
 from capnp.includes cimport schema_cpp
-from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, PyRestorer, RpcSystem, makeRpcServer, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder
+from capnp.includes.capnp_cpp cimport Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, PyPromise, VoidPromise, CallContext, RpcSystem, makeRpcServerBootstrap, makeRpcClient, Capability as C_Capability, TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, makeTwoPartyVatNetwork, PromiseFulfillerPair as C_PromiseFulfillerPair, copyPromiseFulfillerPair, newPromiseAndFulfiller, PyArray, DynamicStruct_Builder, TwoWayPipe
 from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode
 from capnp.includes.types cimport *
 from capnp.helpers.non_circular cimport reraise_kj_exception
@@ -12,6 +14,18 @@ cdef class _StructSchemaField:
     cdef object _parent
     cdef _init(self, C_StructSchema.Field other, parent=?)
 
+cdef class _StringArrayPtr:
+    cdef StringPtr * thisptr
+    cdef object parent
+    cdef size_t size
+    cdef ArrayPtr[StringPtr] asArrayPtr(self) except +reraise_kj_exception
+
+cdef class SchemaParser:
+    cdef C_SchemaParser * thisptr
+    cdef public dict modules_by_id
+    cdef list _all_imports
+    cdef _StringArrayPtr _last_import_array
+    cpdef _parse_disk_file(self, displayName, diskPath, imports) except +reraise_kj_exception
 
 cdef class _DynamicOrphan:
     cdef C_DynamicOrphan thisptr
@@ -46,15 +60,16 @@ cdef class _DynamicStructBuilder:
     cdef DynamicStruct_Builder thisptr
     cdef public object _parent
     cdef public bint is_root
-    cdef bint _is_written
+    cdef public bint _is_written
     cdef object _schema
 
     cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?)
 
     cdef _check_write(self)
-    cpdef to_bytes(_DynamicStructBuilder self)
+    cpdef to_bytes(_DynamicStructBuilder self) except +reraise_kj_exception
-    cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count)
+    cpdef to_segments(_DynamicStructBuilder self) except +reraise_kj_exception
-    cpdef to_bytes_packed(_DynamicStructBuilder self)
+    cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) except +reraise_kj_exception
+    cpdef to_bytes_packed(_DynamicStructBuilder self) except +reraise_kj_exception
 
     cpdef _get(self, field)
     cpdef _set(self, field, value)
@@ -88,21 +103,19 @@ cdef class _Schema:
     cpdef as_struct(self)
     cpdef as_interface(self)
     cpdef as_enum(self)
-    cpdef get_dependency(self, id)
     cpdef get_proto(self)
 
 cdef class _InterfaceSchema:
     cdef C_InterfaceSchema thisptr
     cdef object __method_names, __method_names_inherited, __methods, __methods_inherited
     cdef _init(self, C_InterfaceSchema other)
-    cpdef get_dependency(self, id)
 
 cdef class _DynamicEnum:
     cdef capnp.DynamicEnum thisptr
     cdef public object _parent
 
     cdef _init(self, capnp.DynamicEnum other, object parent)
-    cpdef _as_str(self)
+    cpdef _as_str(self) except +reraise_kj_exception
 
 cdef class _DynamicListBuilder:
     cdef C_DynamicList.Builder thisptr
@@ -115,9 +128,12 @@ cdef class _DynamicListBuilder:
     cpdef adopt(self, index, _DynamicOrphan orphan)
     cpdef disown(self, index)
 
+    cpdef init(self, index, size)
+
 cdef to_python_reader(C_DynamicValue.Reader self, object parent)
 cdef to_python_builder(C_DynamicValue.Builder self, object parent)
 cdef _to_dict(msg, bint verbose, bint ordered)
 cdef _from_list(_DynamicListBuilder msg, list d)
+cdef _from_tuple(_DynamicListBuilder msg, tuple d)
 cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent)
 cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent)

capnp/templates/module.pyx

@@ -1,6 +1,5 @@
 # addressbook_fast.pyx
 # distutils: language = c++
-# distutils: extra_compile_args = --std=c++11
 # distutils: include_dirs = {{include_dir}}
 # distutils: libraries = capnpc capnp capnp-rpc
 # distutils: sources = {{file.filename}}.cpp
@@ -90,7 +89,7 @@ cpdef _set_{{field.name}}(self, value):
     if type(value) is bytes:
         temp_string = StringPtr(<char*>value, len(value))
     else:
-        encoded_value = value.encode()
+        encoded_value = value.encode('utf-8')
         temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
     self.thisptr_child.set{{field.c_name}}(temp_string)
     {% elif 'data' == field['type'] -%}
@@ -99,7 +98,7 @@ cpdef _set_{{field.name}}(self, value):
     if type(value) is bytes:
         temp_string = StringPtr(<char*>value, len(value))
     else:
-        encoded_value = value.encode()
+        encoded_value = value.encode('utf-8')
         temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
     self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](<byte *>temp_string.begin(), temp_string.size()))
     {% else -%}
@@ -133,40 +132,6 @@ cdef _from_list(_DynamicListBuilder msg, list d):
         msg._set(count, val)
         count += 1
 
-cdef DynamicValue.Reader to_dynamic_value(value):
-    cdef DynamicValue.Reader temp
-    cdef StringPtr temp_string
-    value_type = type(value)
-
-    if value_type is int or value_type is long:
-        if value < 0:
-           temp = DynamicValue.Reader(<long long>value)
-        else:
-           temp = DynamicValue.Reader(<unsigned long long>value)
-    elif value_type is float:
-        temp = DynamicValue.Reader(<double>value)
-    elif value_type is bool:
-        temp = DynamicValue.Reader(<cbool>value)
-    elif value_type is bytes:
-        temp_string = StringPtr(<char*>value, len(value))
-        temp = DynamicValue.Reader(temp_string)
-    elif isinstance(value, basestring):
-        encoded_value = value.encode()
-        temp_string = StringPtr(<char*>encoded_value, len(encoded_value))
-        temp = DynamicValue.Reader(temp_string)
-    elif value is None:
-        temp = DynamicValue.Reader(VOID)
-    elif value_type is _DynamicStructBuilder:
-        temp = _extract_dynamic_struct_builder(value)
-    elif value_type is _DynamicStructReader:
-        temp = _extract_dynamic_struct_reader(value)
-    elif value_type is _DynamicEnum:
-        temp = _extract_dynamic_enum(value)
-    else:
-        raise ValueError("Tried to convert value of: '{}' which is an unsupported type: '{}'".format(str(value), str(type(value))))
-
-    return temp
-
 
 cdef extern from "{{file.filename}}.h":
     {%- for node in code.nodes %}

docs/conf.py

@@ -1,3 +1,6 @@
+'''
+Docs configuration
+'''
 # -*- coding: utf-8 -*-
 #
 # capnp documentation build configuration file, created by
@@ -11,7 +14,9 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os, string
+import string
+# import sys, os
+import capnp
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -48,11 +53,10 @@ copyright = u'2013, Author'
 # built documents.
 #
 # The short X.Y version.
-import capnp
 
 vs = capnp.__version__
 # The short X.Y version.
-version = vs.rstrip(string.letters)
+version = vs.rstrip(string.ascii_letters)
 # The full version, including alpha/beta/rc tags.
 release = vs
 
@@ -173,7 +177,6 @@ htmlhelp_basename = 'capnpdoc'
 
 # -- Options for LaTeX output --------------------------------------------------
 
-latex_elements = {
 # The paper size ('letterpaper' or 'a4paper').
 # 'papersize': 'letterpaper',
 
@@ -182,6 +185,7 @@ latex_elements = {
 
 # Additional stuff for the LaTeX preamble.
 # 'preamble': '',
+latex_elements = {
 }
 
 # Grouping the document tree into LaTeX files. List of tuples

docs/quickstart.rst

@@ -304,6 +304,29 @@ There are also packed versions::
 
     alice2 = addressbook_capnp.Person.from_bytes_packed(alice.to_bytes_packed())
 
+
+Byte Segments
+~~~~~~~~~~~~~
+
+.. note:: This feature is not supported in PyPy at the moment, pending investigation.
+
+Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way).
+
+For compatibility on the Python side, use the ``to_segments()`` and ``from_segments()`` functions::
+
+    segments = alice.to_segments()
+
+This returns a list of segments, each a byte buffer. Each segment can be, e.g., turned into a ZeroMQ message frame. The list of segments can also be turned back into an object::
+
+    alice = addressbook_capnp.Person.from_segments(segments)
+
+For more information, please refer to the following links:
+
+- `Advice on minimizing copies from Cap'n Proto <https://stackoverflow.com/questions/28149139/serializing-mutable-state-and-sending-it-asynchronously-over-the-network-with-ne/28156323#28156323>`_ (from the author of Cap'n Proto)
+- `Advice on using Cap'n Proto over ZeroMQ <https://stackoverflow.com/questions/32041315/how-to-send-capn-proto-message-over-zmq/32042234#32042234>`_ (from the author of Cap'n Proto)
+- `Discussion about sending and reassembling Cap'n Proto message segments in C++ <https://groups.google.com/forum/#!topic/capnproto/ClDjGbO7egA>`_ (from the Cap'n Proto mailing list; includes sample code)
+
+
 RPC
 ----------
 

examples/addressbook.py

@@ -1,6 +1,7 @@
+#!/usr/bin/env python3
+
 from __future__ import print_function
-import os
+import capnp # noqa: F401
-import capnp
 
 import addressbook_capnp
 

examples/async_calculator_client.py

@@ -0,0 +1,341 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+import argparse
+import asyncio
+import socket
+import capnp
+
+import calculator_capnp
+
+
+class PowerFunction(calculator_capnp.Calculator.Function.Server):
+
+    '''An implementation of the Function interface wrapping pow().  Note that
+    we're implementing this on the client side and will pass a reference to
+    the server.  The server will then be able to make calls back to the client.'''
+
+    def call(self, params, **kwargs):
+        '''Note the **kwargs. This is very necessary to include, since
+        protocols can add parameters over time. Also, by default, a _context
+        variable is passed to all server methods, but you can also return
+        results directly as python objects, and they'll be added to the
+        results struct in the correct order'''
+
+        return pow(params[0], params[1])
+
+
+async def myreader(client, reader):
+    while True:
+        data = await reader.read(4096)
+        client.write(data)
+
+
+async def mywriter(client, writer):
+    while True:
+        data = await client.read(4096)
+        writer.write(data.tobytes())
+        await writer.drain()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='Connects to the Calculator server \
+at the given address and does some RPCs')
+    parser.add_argument("host", help="HOST:PORT")
+
+    return parser.parse_args()
+
+
+async def main(host):
+    host = host.split(':')
+    addr = host[0]
+    port = host[1]
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+        )
+    except Exception:
+        print("Try IPv6")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+            family=socket.AF_INET6
+        )
+
+    # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
+    client = capnp.TwoPartyClient()
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(client, reader), mywriter(client, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    # Pass "calculator" to ez_restore (there's also a `restore` function that
+    # takes a struct or AnyPointer as an argument), and then cast the returned
+    # capability to it's proper type. This casting is due to capabilities not
+    # having a reference to their schema
+    calculator = client.bootstrap().cast_as(calculator_capnp.Calculator)
+
+    '''Make a request that just evaluates the literal value 123.
+
+    What's interesting here is that evaluate() returns a "Value", which is
+    another interface and therefore points back to an object living on the
+    server.  We then have to call read() on that object to read it.
+    However, even though we are making two RPC's, this block executes in
+    *one* network round trip because of promise pipelining:  we do not wait
+    for the first call to complete before we send the second call to the
+    server.'''
+
+    print('Evaluating a literal... ', end="")
+
+    # Make the request. Note we are using the shorter function form (instead
+    # of evaluate_request), and we are passing a dictionary that represents a
+    # struct and its member to evaluate
+    eval_promise = calculator.evaluate({"literal": 123})
+
+    # This is equivalent to:
+    '''
+    request = calculator.evaluate_request()
+    request.expression.literal = 123
+
+    # Send it, which returns a promise for the result (without blocking).
+    eval_promise = request.send()
+    '''
+
+    # Using the promise, create a pipelined request to call read() on the
+    # returned object. Note that here we are using the shortened method call
+    # syntax read(), which is mostly just sugar for read_request().send()
+    read_promise = eval_promise.value.read()
+
+    # Now that we've sent all the requests, wait for the response.  Until this
+    # point, we haven't waited at all!
+    response = await read_promise.a_wait()
+    assert response.value == 123
+
+    print("PASS")
+
+    '''Make a request to evaluate 123 + 45 - 67.
+
+    The Calculator interface requires that we first call getOperator() to
+    get the addition and subtraction functions, then call evaluate() to use
+    them.  But, once again, we can get both functions, call evaluate(), and
+    then read() the result -- four RPCs -- in the time of *one* network
+    round trip, because of promise pipelining.'''
+
+    print("Using add and subtract... ", end='')
+
+    # Get the "add" function from the server.
+    add = calculator.getOperator(op='add').func
+    # Get the "subtract" function from the server.
+    subtract = calculator.getOperator(op='subtract').func
+
+    # Build the request to evaluate 123 + 45 - 67. Note the form is 'evaluate'
+    # + '_request', where 'evaluate' is the name of the method we want to call
+    request = calculator.evaluate_request()
+    subtract_call = request.expression.init('call')
+    subtract_call.function = subtract
+    subtract_params = subtract_call.init('params', 2)
+    subtract_params[1].literal = 67.0
+
+    add_call = subtract_params[0].init('call')
+    add_call.function = add
+    add_params = add_call.init('params', 2)
+    add_params[0].literal = 123
+    add_params[1].literal = 45
+
+    # Send the evaluate() request, read() the result, and wait for read() to finish.
+    eval_promise = request.send()
+    read_promise = eval_promise.value.read()
+
+    response = await read_promise.a_wait()
+    assert response.value == 101
+
+    print("PASS")
+
+    '''
+    Note: a one liner version of building the previous request (I highly
+    recommend not doing it this way for such a complicated structure, but I
+    just wanted to demonstrate it is possible to set all of the fields with a
+    dictionary):
+
+    eval_promise = calculator.evaluate(
+{'call': {'function': subtract,
+          'params': [{'call': {'function': add,
+                               'params': [{'literal': 123},
+                                          {'literal': 45}]}},
+                     {'literal': 67.0}]}})
+    '''
+
+    '''Make a request to evaluate 4 * 6, then use the result in two more
+    requests that add 3 and 5.
+
+    Since evaluate() returns its result wrapped in a `Value`, we can pass
+    that `Value` back to the server in subsequent requests before the first
+    `evaluate()` has actually returned.  Thus, this example again does only
+    one network round trip.'''
+
+    print("Pipelining eval() calls... ", end="")
+
+    # Get the "add" function from the server.
+    add = calculator.getOperator(op='add').func
+    # Get the "multiply" function from the server.
+    multiply = calculator.getOperator(op='multiply').func
+
+    # Build the request to evaluate 4 * 6
+    request = calculator.evaluate_request()
+
+    multiply_call = request.expression.init("call")
+    multiply_call.function = multiply
+    multiply_params = multiply_call.init("params", 2)
+    multiply_params[0].literal = 4
+    multiply_params[1].literal = 6
+
+    multiply_result = request.send().value
+
+    # Use the result in two calls that add 3 and add 5.
+
+    add_3_request = calculator.evaluate_request()
+    add_3_call = add_3_request.expression.init("call")
+    add_3_call.function = add
+    add_3_params = add_3_call.init("params", 2)
+    add_3_params[0].previousResult = multiply_result
+    add_3_params[1].literal = 3
+    add_3_promise = add_3_request.send().value.read()
+
+    add_5_request = calculator.evaluate_request()
+    add_5_call = add_5_request.expression.init("call")
+    add_5_call.function = add
+    add_5_params = add_5_call.init("params", 2)
+    add_5_params[0].previousResult = multiply_result
+    add_5_params[1].literal = 5
+    add_5_promise = add_5_request.send().value.read()
+
+    # Now wait for the results.
+    assert (await add_3_promise.a_wait()).value == 27
+    assert (await add_5_promise.a_wait()).value == 29
+
+    print("PASS")
+
+    '''Our calculator interface supports defining functions.  Here we use it
+    to define two functions and then make calls to them as follows:
+
+      f(x, y) = x * 100 + y
+      g(x) = f(x, x + 1) * 2;
+      f(12, 34)
+      g(21)
+
+    Once again, the whole thing takes only one network round trip.'''
+
+    print("Defining functions... ", end="")
+
+    # Get the "add" function from the server.
+    add = calculator.getOperator(op='add').func
+    # Get the "multiply" function from the server.
+    multiply = calculator.getOperator(op='multiply').func
+
+    # Define f.
+    request = calculator.defFunction_request()
+    request.paramCount = 2
+
+    # Build the function body.
+    add_call = request.body.init("call")
+    add_call.function = add
+    add_params = add_call.init("params", 2)
+    add_params[1].parameter = 1  # y
+
+    multiply_call = add_params[0].init("call")
+    multiply_call.function = multiply
+    multiply_params = multiply_call.init("params", 2)
+    multiply_params[0].parameter = 0  # x
+    multiply_params[1].literal = 100
+
+    f = request.send().func
+
+    # Define g.
+    request = calculator.defFunction_request()
+    request.paramCount = 1
+
+    # Build the function body.
+    multiply_call = request.body.init("call")
+    multiply_call.function = multiply
+    multiply_params = multiply_call.init("params", 2)
+    multiply_params[1].literal = 2
+
+    f_call = multiply_params[0].init("call")
+    f_call.function = f
+    f_params = f_call.init("params", 2)
+    f_params[0].parameter = 0
+
+    add_call = f_params[1].init("call")
+    add_call.function = add
+    add_params = add_call.init("params", 2)
+    add_params[0].parameter = 0
+    add_params[1].literal = 1
+
+    g = request.send().func
+
+    # OK, we've defined all our functions.  Now create our eval requests.
+
+    # f(12, 34)
+    f_eval_request = calculator.evaluate_request()
+    f_call = f_eval_request.expression.init("call")
+    f_call.function = f
+    f_params = f_call.init("params", 2)
+    f_params[0].literal = 12
+    f_params[1].literal = 34
+    f_eval_promise = f_eval_request.send().value.read()
+
+    # g(21)
+    g_eval_request = calculator.evaluate_request()
+    g_call = g_eval_request.expression.init("call")
+    g_call.function = g
+    g_call.init('params', 1)[0].literal = 21
+    g_eval_promise = g_eval_request.send().value.read()
+
+    # Wait for the results.
+    assert (await f_eval_promise.a_wait()).value == 1234
+    assert (await g_eval_promise.a_wait()).value == 4244
+
+    print("PASS")
+
+    '''Make a request that will call back to a function defined locally.
+
+    Specifically, we will compute 2^(4 + 5).  However, exponent is not
+    defined by the Calculator server.  So, we'll implement the Function
+    interface locally and pass it to the server for it to use when
+    evaluating the expression.
+
+    This example requires two network round trips to complete, because the
+    server calls back to the client once before finishing.  In this
+    particular case, this could potentially be optimized by using a tail
+    call on the server side -- see CallContext::tailCall().  However, to
+    keep the example simpler, we haven't implemented this optimization in
+    the sample server.'''
+
+    print("Using a callback... ", end="")
+
+    # Get the "add" function from the server.
+    add = calculator.getOperator(op='add').func
+
+    # Build the eval request for 2^(4+5).
+    request = calculator.evaluate_request()
+
+    pow_call = request.expression.init("call")
+    pow_call.function = PowerFunction()
+    pow_params = pow_call.init("params", 2)
+    pow_params[0].literal = 2
+
+    add_call = pow_params[1].init("call")
+    add_call.function = add
+    add_params = add_call.init("params", 2)
+    add_params[0].literal = 4
+    add_params[1].literal = 5
+
+    # Send the request and wait.
+    response = await request.send().value.read().a_wait()
+    assert response.value == 512
+
+    print("PASS")
+
+if __name__ == '__main__':
+    asyncio.run(main(parse_args().host))

examples/async_calculator_server.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+import argparse
+import asyncio
+import socket
+import capnp
+
+import calculator_capnp
+
+
+async def myreader(client, reader):
+    while True:
+        data = await reader.read(4096)
+        await client.write(data)
+
+
+async def mywriter(client, writer):
+    while True:
+        data = await client.read(4096)
+        writer.write(data.tobytes())
+        await writer.drain()
+
+
+def read_value(value):
+    '''Helper function to asynchronously call read() on a Calculator::Value and
+    return a promise for the result.  (In the future, the generated code might
+    include something like this automatically.)'''
+
+    return value.read().then(lambda result: result.value)
+
+
+def evaluate_impl(expression, params=None):
+    '''Implementation of CalculatorImpl::evaluate(), also shared by
+    FunctionImpl::call().  In the latter case, `params` are the parameter
+    values passed to the function; in the former case, `params` is just an
+    empty list.'''
+
+    which = expression.which()
+
+    if which == 'literal':
+        return capnp.Promise(expression.literal)
+    elif which == 'previousResult':
+        return read_value(expression.previousResult)
+    elif which == 'parameter':
+        assert expression.parameter < len(params)
+        return capnp.Promise(params[expression.parameter])
+    elif which == 'call':
+        call = expression.call
+        func = call.function
+
+        # Evaluate each parameter.
+        paramPromises = [evaluate_impl(param, params) for param in call.params]
+
+        joinedParams = capnp.join_promises(paramPromises)
+        # When the parameters are complete, call the function.
+        ret = (joinedParams
+               .then(lambda vals: func.call(vals))
+               .then(lambda result: result.value))
+
+        return ret
+    else:
+        raise ValueError("Unknown expression type: " + which)
+
+
+class ValueImpl(calculator_capnp.Calculator.Value.Server):
+
+    "Simple implementation of the Calculator.Value Cap'n Proto interface."
+
+    def __init__(self, value):
+        self.value = value
+
+    def read(self, **kwargs):
+        return self.value
+
+
+class FunctionImpl(calculator_capnp.Calculator.Function.Server):
+
+    '''Implementation of the Calculator.Function Cap'n Proto interface, where the
+    function is defined by a Calculator.Expression.'''
+
+    def __init__(self, paramCount, body):
+        self.paramCount = paramCount
+        self.body = body.as_builder()
+
+    def call(self, params, _context, **kwargs):
+        '''Note that we're returning a Promise object here, and bypassing the
+        helper functionality that normally sets the results struct from the
+        returned object. Instead, we set _context.results directly inside of
+        another promise'''
+
+        assert len(params) == self.paramCount
+        # using setattr because '=' is not allowed inside of lambdas
+        return evaluate_impl(self.body, params).then(lambda value: setattr(_context.results, 'value', value))
+
+
+class OperatorImpl(calculator_capnp.Calculator.Function.Server):
+
+    '''Implementation of the Calculator.Function Cap'n Proto interface, wrapping
+    basic binary arithmetic operators.'''
+
+    def __init__(self, op):
+        self.op = op
+
+    def call(self, params, **kwargs):
+        assert len(params) == 2
+
+        op = self.op
+
+        if op == 'add':
+            return params[0] + params[1]
+        elif op == 'subtract':
+            return params[0] - params[1]
+        elif op == 'multiply':
+            return params[0] * params[1]
+        elif op == 'divide':
+            return params[0] / params[1]
+        else:
+            raise ValueError('Unknown operator')
+
+
+class CalculatorImpl(calculator_capnp.Calculator.Server):
+
+    "Implementation of the Calculator Cap'n Proto interface."
+
+    def evaluate(self, expression, _context, **kwargs):
+        return evaluate_impl(expression).then(lambda value: setattr(_context.results, 'value', ValueImpl(value)))
+
+    def defFunction(self, paramCount, body, _context, **kwargs):
+        return FunctionImpl(paramCount, body)
+
+    def getOperator(self, op, **kwargs):
+        return OperatorImpl(op)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='''Runs the server bound to the\
+given address/port ADDRESS. ''')
+
+    parser.add_argument("address", help="ADDRESS:PORT")
+
+    return parser.parse_args()
+
+
+async def myserver(reader, writer):
+    # Start TwoPartyServer using TwoWayPipe (only requires bootstrap)
+    server = capnp.TwoPartyServer(bootstrap=CalculatorImpl())
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(server, reader), mywriter(server, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    await server.poll_forever()
+
+
+async def main():
+    address = parse_args().address
+    host = address.split(':')
+    addr = host[0]
+    port = host[1]
+
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+        )
+    except Exception:
+        print("Try IPv6")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+            family=socket.AF_INET6
+        )
+
+    async with server:
+        await server.serve_forever()
+
+if __name__ == '__main__':
+    asyncio.run(main())

examples/async_client.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import asyncio
+import argparse
+import time
+import capnp
+import socket
+
+import thread_capnp
+
+capnp.remove_event_loop()
+capnp.create_event_loop(threaded=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='Connects to the Example thread server \
+at the given address and does some RPCs')
+    parser.add_argument("host", help="HOST:PORT")
+
+    return parser.parse_args()
+
+
+class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server):
+    '''An implementation of the StatusSubscriber interface'''
+
+    def status(self, value, **kwargs):
+        print('status: {}'.format(time.time()))
+
+
+async def myreader(client, reader):
+    while True:
+        data = await reader.read(4096)
+        client.write(data)
+
+
+async def mywriter(client, writer):
+    while True:
+        data = await client.read(4096)
+        writer.write(data.tobytes())
+
+
+async def background(cap):
+    subscriber = StatusSubscriber()
+    promise = cap.subscribeStatus(subscriber)
+    await promise.a_wait()
+
+
+async def main(host):
+    host = host.split(':')
+    addr = host[0]
+    port = host[1]
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+        )
+    except Exception:
+        print("Try IPv6")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+            family=socket.AF_INET6
+        )
+
+    # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
+    client = capnp.TwoPartyClient()
+    cap = client.bootstrap().cast_as(thread_capnp.Example)
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(client, reader), mywriter(client, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    # Start background task for subscriber
+    tasks = [background(cap)]
+    asyncio.gather(*tasks, return_exceptions=True)
+
+    # Run blocking tasks
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+
+if __name__ == '__main__':
+    asyncio.run(main(parse_args().host))

examples/async_reconnecting_ssl_client.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import asyncio
+import argparse
+import os
+import time
+import socket
+import ssl
+
+import capnp
+
+import thread_capnp
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+capnp.remove_event_loop()
+capnp.create_event_loop(threaded=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='Connects to the Example thread server \
+at the given address and does some RPCs')
+    parser.add_argument("host", help="HOST:PORT")
+
+    return parser.parse_args()
+
+
+class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server):
+    '''An implementation of the StatusSubscriber interface'''
+
+    def status(self, value, **kwargs):
+        print('status: {}'.format(time.time()))
+
+
+async def myreader(client, reader):
+    while True:
+        try:
+            # Must be a wait_for in order to give watch_connection a slot
+            # to try again
+            data = await asyncio.wait_for(reader.read(4096), timeout=1.0)
+        except asyncio.TimeoutError:
+            continue
+        client.write(data)
+
+
+async def mywriter(client, writer):
+    while True:
+        try:
+            # Must be a wait_for in order to give watch_connection a slot
+            # to try again
+            data = await asyncio.wait_for(client.read(4096), timeout=1.0)
+            writer.write(data.tobytes())
+        except asyncio.TimeoutError:
+            continue
+
+
+async def watch_connection(cap):
+    while True:
+        try:
+            await asyncio.wait_for(cap.alive().a_wait(), timeout=5)
+            await asyncio.sleep(1)
+        except asyncio.TimeoutError:
+            print("Watch timeout!")
+            asyncio.get_running_loop().stop()
+            return False
+
+
+async def background(cap):
+    subscriber = StatusSubscriber()
+    promise = cap.subscribeStatus(subscriber)
+    await promise.a_wait()
+
+
+async def main(host):
+    host = host.split(':')
+    addr = host[0]
+    port = host[1]
+
+    # Setup SSL context
+    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert'))
+
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+            ssl=ctx,
+        )
+    except OSError:
+        print("Try IPv6")
+        try:
+            reader, writer = await asyncio.open_connection(
+                addr, port,
+                ssl=ctx,
+                family=socket.AF_INET6
+            )
+        except OSError:
+            return False
+
+    # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
+    client = capnp.TwoPartyClient()
+    cap = client.bootstrap().cast_as(thread_capnp.Example)
+
+    # Start watcher to restart socket connection if it is lost
+    overalltasks = []
+    watcher = [watch_connection(cap)]
+    overalltasks.append(asyncio.gather(*watcher, return_exceptions=True))
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(client, reader), mywriter(client, writer)]
+    overalltasks.append(asyncio.gather(*coroutines, return_exceptions=True))
+
+    # Start background task for subscriber
+    tasks = [background(cap)]
+    overalltasks.append(asyncio.gather(*tasks, return_exceptions=True))
+
+    # Run blocking tasks
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+
+    for task in overalltasks:
+        task.cancel()
+
+    return True
+
+if __name__ == '__main__':
+    # Using asyncio.run hits an asyncio ssl bug
+    # https://bugs.python.org/issue36709
+    # asyncio.run(main(parse_args().host), loop=loop, debug=True)
+    retry = True
+    while retry:
+        loop = asyncio.new_event_loop()
+        try:
+            retry = not loop.run_until_complete(main(parse_args().host))
+        except RuntimeError:
+            # If an IO is hung, the event loop will be stopped
+            # and will throw RuntimeError exception
+            continue
+        if retry:
+            time.sleep(1)
+            print("Retrying...")
+
+# How this works
+# - There are two retry mechanisms
+#   1. Connection retry
+#   2. alive RPC verification
+# - The connection retry just loops the connection (IPv4+IPv6 until there is a connection or Ctrl+C)
+# - The alive RPC verification attempts a very basic rpc call with a timeout
+#   * If there is a timeout, stop the current event loop
+#   * Use the RuntimeError exception to force a reconnect
+#   * myreader and mywriter must also be wrapped in wait_for in order for the events to get triggered correctly

examples/async_server.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import argparse
+import capnp
+
+import thread_capnp
+import asyncio
+import socket
+
+
+class ExampleImpl(thread_capnp.Example.Server):
+
+    "Implementation of the Example threading Cap'n Proto interface."
+
+    def subscribeStatus(self, subscriber, **kwargs):
+        return capnp.getTimer().after_delay(10**9) \
+            .then(lambda: subscriber.status(True)) \
+            .then(lambda _: self.subscribeStatus(subscriber))
+
+    def longRunning(self, **kwargs):
+        return capnp.getTimer().after_delay(1 * 10**9)
+
+
+async def myreader(server, reader):
+    while True:
+        data = await reader.read(4096)
+        # Close connection if 0 bytes read
+        if len(data) == 0:
+            server.close()
+        await server.write(data)
+
+
+async def mywriter(server, writer):
+    while True:
+        data = await server.read(4096)
+        writer.write(data.tobytes())
+
+
+async def myserver(reader, writer):
+    # Start TwoPartyServer using TwoWayPipe (only requires bootstrap)
+    server = capnp.TwoPartyServer(bootstrap=ExampleImpl())
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(server, reader), mywriter(server, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    await server.poll_forever()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='''Runs the server bound to the\
+given address/port ADDRESS. ''')
+
+    parser.add_argument("address", help="ADDRESS:PORT")
+
+    return parser.parse_args()
+
+
+async def main():
+    address = parse_args().address
+    host = address.split(':')
+    addr = host[0]
+    port = host[1]
+
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+        )
+    except Exception:
+        print("Try IPv6")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+            family=socket.AF_INET6
+        )
+
+    async with server:
+        await server.serve_forever()
+
+
+if __name__ == '__main__':
+    asyncio.run(main())

examples/async_ssl_client.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import asyncio
+import argparse
+import os
+import time
+import capnp
+import socket
+import ssl
+
+import thread_capnp
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+capnp.remove_event_loop()
+capnp.create_event_loop(threaded=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='Connects to the Example thread server \
+at the given address and does some RPCs')
+    parser.add_argument("host", help="HOST:PORT")
+
+    return parser.parse_args()
+
+
+class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server):
+    '''An implementation of the StatusSubscriber interface'''
+
+    def status(self, value, **kwargs):
+        print('status: {}'.format(time.time()))
+
+
+async def myreader(client, reader):
+    while True:
+        data = await reader.read(4096)
+        client.write(data)
+
+
+async def mywriter(client, writer):
+    while True:
+        data = await client.read(4096)
+        writer.write(data.tobytes())
+        await writer.drain()
+
+
+async def background(cap):
+    subscriber = StatusSubscriber()
+    promise = cap.subscribeStatus(subscriber)
+    await promise.a_wait()
+
+
+async def main(host):
+    host = host.split(':')
+    addr = host[0]
+    port = host[1]
+
+    # Setup SSL context
+    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, 'selfsigned.cert'))
+
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+            ssl=ctx,
+        )
+    except Exception:
+        print("Try IPv6")
+        reader, writer = await asyncio.open_connection(
+            addr, port,
+            ssl=ctx,
+            family=socket.AF_INET6
+        )
+
+    # Start TwoPartyClient using TwoWayPipe (takes no arguments in this mode)
+    client = capnp.TwoPartyClient()
+    cap = client.bootstrap().cast_as(thread_capnp.Example)
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(client, reader), mywriter(client, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    # Start background task for subscriber
+    tasks = [background(cap)]
+    asyncio.gather(*tasks, return_exceptions=True)
+
+    # Run blocking tasks
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+    await cap.longRunning().a_wait()
+    print('main: {}'.format(time.time()))
+
+if __name__ == '__main__':
+    # Using asyncio.run hits an asyncio ssl bug
+    # https://bugs.python.org/issue36709
+    # asyncio.run(main(parse_args().host), loop=loop, debug=True)
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main(parse_args().host))

examples/async_ssl_server.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+from __future__ import print_function
+
+import argparse
+import os
+import capnp
+
+import thread_capnp
+import asyncio
+import socket
+import ssl
+
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+
+
+class ExampleImpl(thread_capnp.Example.Server):
+
+    "Implementation of the Example threading Cap'n Proto interface."
+
+    def subscribeStatus(self, subscriber, **kwargs):
+        return capnp.getTimer().after_delay(10**9) \
+            .then(lambda: subscriber.status(True)) \
+            .then(lambda _: self.subscribeStatus(subscriber))
+
+    def longRunning(self, **kwargs):
+        return capnp.getTimer().after_delay(1 * 10**9)
+
+    def alive(self, **kwargs):
+        return True
+
+
+async def myreader(server, reader):
+    while True:
+        data = await reader.read(4096)
+        # Close connection if 0 bytes read
+        if len(data) == 0:
+            server.close()
+        await server.write(data)
+
+
+async def mywriter(server, writer):
+    while True:
+        data = await server.read(4096)
+        writer.write(data.tobytes())
+        await writer.drain()
+
+
+async def myserver(reader, writer):
+    # Start TwoPartyServer using TwoWayPipe (only requires bootstrap)
+    server = capnp.TwoPartyServer(bootstrap=ExampleImpl())
+
+    # Assemble reader and writer tasks, run in the background
+    coroutines = [myreader(server, reader), mywriter(server, writer)]
+    asyncio.gather(*coroutines, return_exceptions=True)
+
+    await server.poll_forever()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(usage='''Runs the server bound to the\
+given address/port ADDRESS. ''')
+
+    parser.add_argument("address", help="ADDRESS:PORT")
+
+    return parser.parse_args()
+
+
+async def main():
+    address = parse_args().address
+    host = address.split(':')
+    addr = host[0]
+    port = host[1]
+
+    # Setup SSL context
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
+    ctx.load_cert_chain(os.path.join(this_dir, 'selfsigned.cert'), os.path.join(this_dir, 'selfsigned.key'))
+
+    # Handle both IPv4 and IPv6 cases
+    try:
+        print("Try IPv4")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+            ssl=ctx,
+        )
+    except Exception:
+        print("Try IPv6")
+        server = await asyncio.start_server(
+            myserver,
+            addr, port,
+            ssl=ctx,
+            family=socket.AF_INET6,
+        )
+
+    async with server:
+        await server.serve_forever()
+
+if __name__ == '__main__':
+    asyncio.run(main())

examples/calculator_client.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 import argparse
-import socket
 import capnp
 
 import calculator_capnp
@@ -302,5 +301,6 @@ def main(host):
 
     print("PASS")
 
+
 if __name__ == '__main__':
     main(parse_args().host)

examples/calculator_server.py

@@ -1,9 +1,7 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 import argparse
-import socket
-import random
 import capnp
 
 import calculator_capnp
@@ -136,5 +134,6 @@ def main():
     server = capnp.TwoPartyServer(address, bootstrap=CalculatorImpl())
     server.run_forever()
 
+
 if __name__ == '__main__':
     main()

examples/selfsigned.cert

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICpDCCAYwCCQDR+CRWUUUdKDANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMTkwOTE3MDczODI2WhcNNDcwMjAyMDczODI2WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCg
++X+utjujNcq/zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXs
+czvUTWDoPhvvbxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9
+UNKzf4VQVoa0n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapL
+GO2kiPzp36388n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjS
+Fcml6J06CITpYeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEON
+vWZiM7Ul+9mOPejUMPwlAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAICkFOw0Dd1U
+r60rgVpUiHMoFNuBP3ikZfBQ+KXOtfTIYbxbi+iKvvPlB1NFA7Qy3FqqN4sUncvp
+QQLxdm+KClM+hvAogng/SyEJJW169vuQbqn5s/1iKCOtGFkI18thCr3rwsI6vTaR
+0TmTPtjSQKl5PqcS8kQJTED+CnQhqOAv7C68Bpg+x2dSD9VCq81cPeDbfnK6gico
+29qJYUm4RCXMicrzvEwNObx06TQKJb/pWjpl1NAmpFvcz+2MYPL/QTfH/cS5lhgx
+KCe4/kDO0HCueOi2MqBFaO2B0kZxanMoZ2KZe2b/Bp1CTJzXCXNYWQj6QbLzZxaD
+fOp0J8wAo1U=
+-----END CERTIFICATE-----

examples/selfsigned.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCg+X+utjujNcq/
+zLgcuj1o0BRfu8cF1ZNS/lLhSi+B064fs7905Ii8XS7rP7LBZhXsczvUTWDoPhvv
+bxkHzblPGqytAYuWWTE7YdXQNTIKm4TPZlK4vbEGMSJ1OGQxXbc9UNKzf4VQVoa0
+n0bEnnqXO4kqcNANM4U9+6jN8IFZ4B82eCJmdw5Hd3HHhrPbyapLGO2kiPzp3638
+8n6CwFngOCv4NvHt9G5fDP9Tp+fhdHGSA9ViuDRoM39C8yHtQTjSFcml6J06CITp
+YeMd9/Of43Y9TpCVfViVlbTDE/8B9uwLzEgXJs5UBCmUjGnWtEONvWZiM7Ul+9mO
+PejUMPwlAgMBAAECggEAD6Vwlai8zzZRSKc7Vf98LI3dDRkRVS3XLf/uSluNlo7e
+o9Iyz8fOypA8GT2NwGKNyvfAXvhObQRsbq9bvXhvhJLRKde2m5x7vovZ3mztOj63
+f/kwHSjC5hksgjxC8NFtGBadBDlm2dIvMasxk7bbr4tn36orbr0NPGMTm0C/Md8B
+bSUzuc/mT+6KWfW9g4svqebSbKvC7tGuAu3/RfL1cmbuuvtJJA+EPfRjgCtOiFSk
+8NLE0KLUYySf6M3MAHMeSwQhVr1xyYUkiOqQoxMC7CpaplNqaB2rrOe2nEjLpXgx
+80WLFbB22HNEkBSgX5zz4FmrLchwaI79f3PiGQ2DWQKBgQDSIsl1t13eR8ubsGU5
+Z097U8/eylyXJC+ZU1/TbWgaPNHlMf26EncSJwlmq7pNJ0Rk7p/edPMWKu9tOgQE
+iG0QeKxvxcbc1LzVmfLKXY8hW4DOhiBTiRNNb+YWuYmKYnHXM8hrKthNXBFfRrwb
+Pb+mid9FcK4GcIDkWbgXEahmwwKBgQDEG9sB7Ee4fDiQi1oVsPkMwxnxsiNpSRRG
+9CmG/xIvL81vSaONPmg1f5q5/Wqkd6s3QmMs4rE5U8+kgRvuw/MnN/UxQBH5/LAn
+T3hbo7qlIONOOpgQswg7wIQnKP31spNU2FthA1ACazRBsIyH+hPmgeqvtvFhL4Qy
+6q5tfhFy9wKBgGYZtu82aCqPkdOU0qogk1Ll9zNV+cUKNQJ3qzDMkO9mq8mED7cw
+L6CnTP8Q45WHRckQ1Ka/Bjm4JNtafAdDzlJZf9dTLnuv9gyHH5vJ97iKgDxYmS5d
+hP50J0TVY4nUqWGZ7IB9sdlsqZg0g0NtLkiZ5t0TkcrZMRdCrJqw3rUHAoGBAJNk
+wEmEti8Rpk31fsK43aba6KABHJ5gX84oayHcimVOz1/qf/ODyT0UaE2MC2AL1XLW
+AcZVp5AHzxO8OitNuW5rn2zh0+EJK7iQAU0XFQxRWKaOYYaDmReXzXvFUoMdMaDe
+cGfM3pDC1Gbe8/CrY9OnJ6XjoS5DUWAXhPwkeabnAoGBAMmI9xcyfcPjO6XKKymZ
+z+6bwvFaey9Acy5vkxLrEcRqH8pRR09CltWzR4vhpJwHABWPHVGqdtvlLw1/lUrj
+xr4RXvYXK28cK9Tdak0c3+HPSsozxrsX6AQzcG27ymo5s0KYaVuxWh98iZU7Eb52
+YfCx3eOSIHPH0ay7KBVAQocG
+-----END PRIVATE KEY-----

examples/thread.capnp

@@ -8,4 +8,5 @@ interface Example {
 
   longRunning @0 () -> (value: Bool);
   subscribeStatus @1 (subscriber: StatusSubscriber);
+  alive @2 () -> (value: Bool);
 }

examples/thread_client.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 
@@ -31,7 +31,7 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server):
 
 def start_status_thread(host):
     client = capnp.TwoPartyClient(host)
-  cap = client.ez_restore('example').cast_as(thread_capnp.Example)
+    cap = client.bootstrap().cast_as(thread_capnp.Example)
 
     subscriber = StatusSubscriber()
     promise = cap.subscribeStatus(subscriber)
@@ -40,7 +40,7 @@ def start_status_thread(host):
 
 def main(host):
     client = capnp.TwoPartyClient(host)
-  cap = client.ez_restore('example').cast_as(thread_capnp.Example)
+    cap = client.bootstrap().cast_as(thread_capnp.Example)
 
     status_thread = threading.Thread(target=start_status_thread, args=(host,))
     status_thread.daemon = True
@@ -54,5 +54,6 @@ def main(host):
     cap.longRunning().wait()
     print('main: {}'.format(time.time()))
 
+
 if __name__ == '__main__':
     main(parse_args().host)

examples/thread_server.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from __future__ import print_function
 
@@ -18,7 +18,7 @@ class ExampleImpl(thread_capnp.Example.Server):
             .then(lambda _: self.subscribeStatus(subscriber))
 
     def longRunning(self, **kwargs):
-        return capnp.getTimer().after_delay(3 * 10**9)
+        return capnp.getTimer().after_delay(1 * 10**9)
 
 
 def parse_args():
@@ -31,19 +31,12 @@ given address/port ADDRESS may be '*' to bind to all local addresses.\
     return parser.parse_args()
 
 
-impl = ExampleImpl()
-
-
-def restore(ref):
-    assert ref.as_text() == 'example'
-    return impl
-
-
 def main():
     address = parse_args().address
 
-    server = capnp.TwoPartyServer(address, restore)
+    server = capnp.TwoPartyServer(address, bootstrap=ExampleImpl())
     server.run_forever()
 
+
 if __name__ == '__main__':
     main()

requirements.txt

@@ -1,5 +1,5 @@
-jinja2 >= 2.7.3
+jinja2
-cython == 0.21.2
+cython
-setuptools >= 0.8
+setuptools
 pytest
 tox

scripts/capnp-json.py

@@ -41,4 +41,5 @@ def main():
 
     globals()[command](**kwargs) # hacky way to get defined functions, and call function with name=command
 
+
 main()

scripts/capnp_test_pycapnp.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 from __future__ import print_function
-import capnp
 import os
+import sys
+
+import capnp
 capnp.add_import_hook([os.getcwd(), "/usr/local/include/"]) # change this to be auto-detected?
 
-import test_capnp
+import test_capnp # noqa: E402
 
-import sys
 
 def decode(name):
     class_name = name[0].upper() + name[1:]
@@ -18,6 +19,7 @@ def encode(name):
     message = getattr(test_capnp, class_name).from_dict(val.to_dict())
     print(message.to_bytes())
 
+
 if sys.argv[1] == 'decode':
     decode(sys.argv[2])
 else:

setup.py

@@ -1,29 +1,41 @@
 #!/usr/bin/env python
+'''
+pycapnp-async distutils setup.py
+'''
+
 from __future__ import print_function
 
-use_cython = False
-
-from distutils.core import setup
 import os
+import struct
 import sys
-from buildutils import test_build, fetch_libcapnp, build_libcapnp, info
+
+from distutils.command.clean import clean as _clean
 from distutils.errors import CompileError
 from distutils.extension import Extension
+from distutils.spawn import find_executable
+
+from setuptools import setup, find_packages, Extension
+
+from buildutils import test_build, fetch_libcapnp, build_libcapnp, info
 
 _this_dir = os.path.dirname(__file__)
 
 MAJOR = 0
-MINOR = 5
+MINOR = 7
-MICRO = 6
+MICRO = 0
 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
 
 
 # Write version info
 def write_version_py(filename=None):
+    '''
+    Generate pycapnp-async version
+    '''
     cnt = """\
 version = '%s'
 short_version = '%s'
 
+# flake8: noqa E402 F401
 from .lib.capnp import _CAPNP_VERSION_MAJOR as LIBCAPNP_VERSION_MAJOR
 from .lib.capnp import _CAPNP_VERSION_MINOR as LIBCAPNP_VERSION_MINOR
 from .lib.capnp import _CAPNP_VERSION_MICRO as LIBCAPNP_VERSION_MICRO
@@ -39,21 +51,25 @@ from .lib.capnp import _CAPNP_VERSION as LIBCAPNP_VERSION
     finally:
         a.close()
 
+
 write_version_py()
 
 # Try to convert README using pandoc
 try:
     import pypandoc
-    long_description = pypandoc.convert('README.md', 'rst')
+    long_description = pypandoc.convert_file('README.md', 'rst')
-    changelog = pypandoc.convert('CHANGELOG.md', 'rst')
+    changelog = pypandoc.convert_file('CHANGELOG.md', 'rst')
     changelog = '\nChangelog\n=============\n' + changelog
     long_description += changelog
 except (IOError, ImportError):
+    if sys.argv and sys.argv[-1] == 'sdist':
+        raise
     long_description = ''
 
-# Clean command, invoked with `python setup.py clean`
-from distutils.command.clean import clean as _clean
 class clean(_clean):
+    '''
+    Clean command, invoked with `python setup.py clean`
+    '''
     def run(self):
         _clean.run(self)
         for x in [ 'capnp/lib/capnp.cpp', 'capnp/lib/capnp.h', 'capnp/version.py' ]:
@@ -63,10 +79,6 @@ class clean(_clean):
             except OSError:
                 pass
 
-# set use_cython if lib/capnp.cpp is not detected
-capnp_compiled_file = os.path.join(os.path.dirname(__file__), 'capnp', 'lib', 'capnp.cpp')
-if not os.path.isfile(capnp_compiled_file):
-    use_cython = True
 
 # hack to parse commandline arguments
 force_bundled_libcapnp = "--force-bundled-libcapnp" in sys.argv
@@ -78,61 +90,102 @@ if force_system_libcapnp:
 force_cython = "--force-cython" in sys.argv
 if force_cython:
     sys.argv.remove("--force-cython")
-    use_cython = True
+    # Always use cython, ignoring option
+libcapnp_url = None
+try:
+    libcapnp_url_index = sys.argv.index("--libcapnp-url")
+    libcapnp_url = sys.argv[libcapnp_url_index + 1]
+    sys.argv.remove("--libcapnp-url")
+    sys.argv.remove(libcapnp_url)
+except Exception:
+    pass
 
-if use_cython:
 from Cython.Distutils import build_ext as build_ext_c
-else:
-    from distutils.command.build_ext import build_ext as build_ext_c
 
 class build_libcapnp_ext(build_ext_c):
+    '''
+    Build capnproto library
+    '''
     def build_extension(self, ext):
         build_ext_c.build_extension(self, ext)
 
     def run(self):
-        build_failed = False
+        if force_bundled_libcapnp:
-        try:
+            need_build = True
-            test_build()
+        elif force_system_libcapnp:
-        except CompileError:
+            need_build = False
-            build_failed = True
+        else:
+            # Try to use capnp executable to find include and lib path
+            capnp_executable = find_executable("capnp")
+            if capnp_executable:
+                self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')]
+                self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')]
 
-        if build_failed and force_system_libcapnp:
+            # Try to autodetect presence of library. Requires compile/run
-            raise RuntimeError("libcapnp C++ library not detected and --force-system-libcapnp was used")
+            # step so only works for host (non-cross) compliation
-        if build_failed or force_bundled_libcapnp:
+            try:
-            if build_failed:
+                test_build(include_dirs=self.include_dirs, library_dirs=self.library_dirs)
-                info("*WARNING* no libcapnp detected. Will download and build it from source now. If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. Downloading and building libcapnp may take a while.")
+                need_build = False
+            except CompileError:
+                need_build = True
+
+        if need_build:
+            info(
+                "*WARNING* no libcapnp detected or rebuild forced. "
+                "Will download and build it from source now. "
+                "If you have C++ Cap'n Proto installed, it may be out of date or is not being detected. "
+                "Downloading and building libcapnp may take a while."
+            )
             bundle_dir = os.path.join(_this_dir, "bundled")
             if not os.path.exists(bundle_dir):
                 os.mkdir(bundle_dir)
-            build_dir = os.path.join(_this_dir, "build")
+            build_dir = os.path.join(_this_dir, "build{}".format(8 * struct.calcsize("P")))
             if not os.path.exists(build_dir):
                 os.mkdir(build_dir)
-            fetch_libcapnp(bundle_dir)
 
+            # Check if we've already built capnproto
+            capnp_bin = os.path.join(build_dir, 'bin', 'capnp')
+            if os.name == 'nt':
+                capnp_bin = os.path.join(build_dir, 'bin', 'capnp.exe')
+
+            if not os.path.exists(capnp_bin):
+                # Not built, fetch and build
+                fetch_libcapnp(bundle_dir, libcapnp_url)
                 build_libcapnp(bundle_dir, build_dir)
+            else:
+                info("capnproto already built at {}".format(build_dir))
 
             self.include_dirs += [os.path.join(build_dir, 'include')]
             self.library_dirs += [os.path.join(build_dir, 'lib')]
 
         return build_ext_c.run(self)
 
-if use_cython:
+extra_compile_args = ['--std=c++14']
-    from Cython.Build import cythonize
+extra_link_args = []
-    import Cython
+if os.name == 'nt':
-    extensions = cythonize('capnp/lib/*.pyx')
+    extra_compile_args = ['/std:c++14', '/MD']
-else:
+    extra_link_args = ['/MANIFEST']
-    extensions = [Extension("capnp.lib.capnp", ["capnp/lib/capnp.cpp"],
+
-                            include_dirs=["."],
+import Cython.Build
+import Cython # noqa: F401
+extensions = [Extension(
+    '*', ['capnp/lib/*.pyx'],
+    extra_compile_args=extra_compile_args,
+    extra_link_args=extra_link_args,
     language='c++',
-                            extra_compile_args=['--std=c++11'],
+)]
-                            libraries=['capnpc', 'capnp-rpc', 'capnp', 'kj-async', 'kj'])]
 
 setup(
-    name="pycapnp",
+    name="pycapnp-async",
     packages=["capnp"],
     version=VERSION,
-    package_data={'capnp': ['*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h', 'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*']},
+    package_data={
-    ext_modules=extensions,
+        'capnp': [
+            '*.pxd', '*.h', '*.capnp', 'helpers/*.pxd', 'helpers/*.h',
+            'includes/*.pxd', 'lib/*.pxd', 'lib/*.py', 'lib/*.pyx', 'templates/*'
+        ]
+    },
+    ext_modules=Cython.Build.cythonize(extensions),
     cmdclass={
         'clean': clean,
         'build_ext': build_libcapnp_ext
@@ -145,10 +198,10 @@ setup(
     description="A cython wrapping of the C++ Cap'n Proto library",
     long_description=long_description,
     license='BSD',
-    author="Jason Paryani",
+    author="Jacob Alexander",
-    author_email="pypi-contact@jparyani.com",
+    author_email="haata@kiibohd.com",
-    url = 'https://github.com/jparyani/pycapnp',
+    url='https://github.com/haata/pycapnp-async',
-    download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION,
+    download_url='https://github.com/haata/pycapnp-async/archive/v%s.zip' % VERSION,
     keywords=['capnp', 'capnproto', "Cap'n Proto"],
     classifiers=[
         'Development Status :: 4 - Beta',
@@ -158,12 +211,7 @@ setup(
         'Operating System :: POSIX',
         'Programming Language :: C++',
         'Programming Language :: Cython',
-        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 2.6',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.2',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Communications'],
 )

test/addressbook.capnp

@@ -37,3 +37,6 @@ struct AddressBook {
   people @0 :List(Person);
 }
 
+struct NestedList {
+  list @0 :List(List(Int32));
+}

test/all-types.txt

@@ -25,7 +25,7 @@
     uInt64Field = 345678901234567890,
     float32Field = -1.25e-10,
     float64Field = 345,
-    textField = "baz",
+    textField = "☃",
     dataField = "qux",
     structField = (
       voidField = void,

test/test_capability.capnp

@@ -77,3 +77,19 @@ interface TestTailCallee {
 interface TestTailCaller {
   foo @0 (i :Int32, callee :TestTailCallee) -> TestTailCallee.TailResult;
 }
+
+interface TestPassedCap {
+  foo @0 (cap :TestInterface) -> (x: Text);
+}
+
+interface TestStructArg {
+  bar @0 BarParams -> (c: Text);
+}
+struct BarParams {
+  a @0 :Text;
+  b @1 :Int32;
+}
+
+interface TestGeneric(MyObject)  {
+  foo @0 (a :MyObject) -> (b: Text);
+}

test/test_capability.py

@@ -1,8 +1,7 @@
 import pytest
-import capnp
-import os
 import time
 
+import capnp
 import test_capability_capnp as capability
 
 class Server(capability.TestInterface.Server):
@@ -160,6 +159,7 @@ class BadPipelineServer(capability.TestPipeline.Server):
             _results = _context.results
             _results.s = response.x + '_foo'
             _results.outBox.cap = Server(100)
+
         def _error(error):
             raise Exception('test was a success')
 
@@ -186,7 +186,7 @@ def test_pipeline_exception():
     pipelinePromise = outCap.foo(i=10)
 
     with pytest.raises(Exception):
-        loop.wait(pipelinePromise)
+        pipelinePromise.wait()
 
     with pytest.raises(Exception):
         remote.wait()
@@ -194,7 +194,7 @@ def test_pipeline_exception():
 def test_casting():
     client = capability.TestExtends._new_client(Server())
     client2 = client.upcast(capability.TestInterface)
-    client3 = client2.cast_as(capability.TestInterface)
+    _ = client2.cast_as(capability.TestInterface)
 
     with pytest.raises(Exception):
         client.upcast(capability.TestPipeline)
@@ -343,3 +343,43 @@ def test_inheritance():
     response = remote.wait()
 
     assert response.x == '26'
+
+
+class TestPassedCap(capability.TestPassedCap.Server):
+    def foo(self, cap, _context, **kwargs):
+        def set_result(res):
+            _context.results.x = res.x
+        return cap.foo(5).then(set_result)
+
+
+def test_null_cap():
+    client = capability.TestPassedCap._new_client(TestPassedCap())
+    assert client.foo(Server()).wait().x == '26'
+
+    with pytest.raises(capnp.KjException):
+        client.foo().wait()
+
+
+class TestStructArg(capability.TestStructArg.Server):
+    def bar(self, a, b, **kwargs):
+        return a + str(b)
+
+
+def test_struct_args():
+    client = capability.TestStructArg._new_client(TestStructArg())
+    assert client.bar(a='test', b=1).wait().c == 'test1'
+    with pytest.raises(capnp.KjException):
+        assert client.bar('test', 1).wait().c == 'test1'
+
+
+class TestGeneric(capability.TestGeneric.Server):
+    def foo(self, a, **kwargs):
+        return a.as_text() + 'test'
+
+
+def test_generic():
+    client = capability.TestGeneric._new_client(TestGeneric())
+
+    obj = capnp._MallocMessageBuilder().get_root_as_any()
+    obj.set_as_text("anypointer_")
+    assert client.foo(obj).wait().b == 'anypointer_test'

test/test_capability_context.py

@@ -1,11 +1,15 @@
-import pytest
-import capnp
 import os
+import pytest
+
+import capnp
 
 this_dir = os.path.dirname(__file__)
 
+# flake8: noqa: E501
+
 @pytest.fixture
 def capability():
+    capnp.cleanup_global_schema_parser()
     return capnp.load(os.path.join(this_dir, 'test_capability.capnp'))
 
 class Server:
@@ -115,7 +119,15 @@ def test_simple_client_context(capability):
     with pytest.raises(Exception):
         remote = client.foo(baz=5)
 
+@pytest.mark.xfail
 def test_pipeline_context(capability):
+    '''
+    E   capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: <class 'Failed'>:Fixture "capability" called directly. Fixtures are not meant to be called directly,
+    E   but are created automatically when test functions request them as parameters.
+    E   See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and
+    E   https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code.
+    E   stack: 7f87c1ac6e40 7f87c17c3250 7f87c17be260 7f87c17c49f0 7f87c17c0f50 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409
+    '''
     client = capability.TestPipeline._new_client(PipelineServer())
     foo_client = capability.TestInterface._new_client(Server())
 
@@ -150,6 +162,7 @@ class BadPipelineServer:
         def _then(response):
             context.results.s = response.x + '_foo'
             context.results.outBox.cap = capability().TestInterface._new_server(Server(100))
+
         def _error(error):
             raise Exception('test was a success')
 
@@ -176,7 +189,7 @@ def test_pipeline_exception_context(capability):
     pipelinePromise = outCap.foo(i=10)
 
     with pytest.raises(Exception):
-        loop.wait(pipelinePromise)
+        pipelinePromise.wait()
 
     with pytest.raises(Exception):
         remote.wait()
@@ -184,7 +197,7 @@ def test_pipeline_exception_context(capability):
 def test_casting_context(capability):
     client = capability.TestExtends._new_client(Server())
     client2 = client.upcast(capability.TestInterface)
-    client3 = client2.cast_as(capability.TestInterface)
+    _ = client2.cast_as(capability.TestInterface)
 
     with pytest.raises(Exception):
         client.upcast(capability.TestPipeline)
@@ -219,7 +232,15 @@ class TailCallee:
         results.t = context.params.t
         results.c = capability().TestCallOrder._new_server(TailCallOrder())
 
+@pytest.mark.xfail
 def test_tail_call(capability):
+    '''
+    E   capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:75: failed: <class 'Failed'>:Fixture "capability" called directly. Fixtures are not meant to be called directly,
+    E   but are created automatically when test functions request them as parameters.
+    E   See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and
+    E   https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code.
+    E   stack: 7f87c17c5540 7f87c17c51b0 7f87c17c5540 7f87c17d7bf0 7f87c1acb768 7f87c1aaf185 7f87c1aaf2dc 7f87c1a6da1d 7f87c3895459 7f87c3895713 7f87c38c72eb 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b6e7e 7f87c38fe48d 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c38fdb77 7f87c38b5767 7f87c38b67d2 7f87c38c71cf 7f87c3901409 7f87c38b6632 7f87c38c71cf 7f87c3901409 7f87c38b5767 7f87c38b6e7e 7f87c388ace7
+    '''
     callee_server = TailCallee()
     caller_server = TailCaller()
 

test/test_capability_old.py

@@ -1,9 +1,12 @@
-import pytest
-import capnp
 import os
+import pytest
+
+import capnp
 
 this_dir = os.path.dirname(__file__)
 
+# flake8: noqa: E501
+
 @pytest.fixture
 def capability():
     return capnp.load(os.path.join(this_dir, 'test_capability.capnp'))
@@ -116,7 +119,15 @@ def test_simple_client(capability):
     with pytest.raises(Exception):
         remote = client.foo(baz=5)
 
+@pytest.mark.xfail
 def test_pipeline(capability):
+    '''
+    E   capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:61: failed: <class 'Failed'>:Fixture "capability" called directly. Fixtures are not meant to be called directly,
+    E   but are created automatically when test functions request them as parameters.
+    E   See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and
+    E   https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code.
+    E   stack: 7f680f7fce40 7f680f4f9250 7f680f4f4260 7f680f4fa9f0 7f680f4f6f50 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409
+    '''
     client = capability.TestPipeline._new_client(PipelineServer())
     foo_client = capability.TestInterface._new_client(Server())
 
@@ -154,6 +165,7 @@ class BadPipelineServer:
             _results = _context.results
             _results.s = response.x + '_foo'
             _results.outBox.cap = capability().TestInterface._new_server(Server(100))
+
         def _error(error):
             raise Exception('test was a success')
 
@@ -180,7 +192,7 @@ def test_pipeline_exception(capability):
     pipelinePromise = outCap.foo(i=10)
 
     with pytest.raises(Exception):
-        loop.wait(pipelinePromise)
+        pipelinePromise.wait()
 
     with pytest.raises(Exception):
         remote.wait()
@@ -188,7 +200,7 @@ def test_pipeline_exception(capability):
 def test_casting(capability):
     client = capability.TestExtends._new_client(Server())
     client2 = client.upcast(capability.TestInterface)
-    client3 = client2.cast_as(capability.TestInterface)
+    _ = client2.cast_as(capability.TestInterface)
 
     with pytest.raises(Exception):
         client.upcast(capability.TestPipeline)
@@ -223,7 +235,15 @@ class TailCallee:
         results.t = t
         results.c = capability().TestCallOrder._new_server(TailCallOrder())
 
+@pytest.mark.xfail
 def test_tail_call(capability):
+    '''
+    E   capnp.lib.capnp.KjException: capnp/lib/capnp.pyx:104: failed: <class 'Failed'>:Fixture "capability" called directly. Fixtures are not meant to be called directly,
+    E   but are created automatically when test functions request them as parameters.
+    E   See https://docs.pytest.org/en/latest/fixture.html for more information about fixtures, and
+    E   https://docs.pytest.org/en/latest/deprecations.html#calling-fixtures-directly about how to update your code.
+    E   stack: 7f680f4fb540 7f680f4fb1b0 7f680f4fb540 7f680f50dbf0 7f680f801768 7f680f7e5185 7f680f7e52dc 7f680f7a3a1d 7f68115cb459 7f68115cb713 7f68115fd2eb 7f6811637409 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ece7e 7f681163448d 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811633b77 7f68115eb767 7f68115ec7d2 7f68115fd1cf 7f6811637409 7f68115ec632 7f68115fd1cf 7f6811637409 7f68115eb767 7f68115ece7e 7f68115c0ce7
+    '''
     callee_server = TailCallee()
     caller_server = TailCaller()
 

test/test_examples.py

@@ -0,0 +1,75 @@
+import os
+import socket
+import subprocess
+import sys
+import time
+
+examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples')
+
+
+def run_subprocesses(address, server, client):
+    cmd = [sys.executable, os.path.join(examples_dir, server), address]
+    server = subprocess.Popen(cmd)
+    retries = 30
+    addr, port = address.split(':')
+    while True:
+        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        result = sock.connect_ex((addr, int(port)))
+        if result == 0:
+            break
+        sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        result = sock.connect_ex((addr, int(port)))
+        if result == 0:
+            break
+        # Give the server some small amount of time to start listening
+        time.sleep(0.1)
+        retries -= 1
+        if retries == 0:
+            assert False, "Timed out waiting for server to start"
+    cmd = [sys.executable, os.path.join(examples_dir, client), address]
+    client = subprocess.Popen(cmd)
+
+    ret = client.wait(timeout=30)
+    server.kill()
+    assert ret == 0
+
+
+def test_async_calculator_example():
+    address = 'localhost:36432'
+    server = 'async_calculator_server.py'
+    client = 'async_calculator_client.py'
+    run_subprocesses(address, server, client)
+
+
+def test_thread_example():
+    address = 'localhost:36433'
+    server = 'thread_server.py'
+    client = 'thread_client.py'
+    run_subprocesses(address, server, client)
+
+
+def test_addressbook_example():
+    proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')])
+    ret = proc.wait()
+    assert ret == 0
+
+
+def test_async_example():
+    address = 'localhost:36434'
+    server = 'async_server.py'
+    client = 'async_client.py'
+    run_subprocesses(address, server, client)
+
+
+def test_ssl_async_example():
+    address = 'localhost:36435'
+    server = 'async_ssl_server.py'
+    client = 'async_ssl_client.py'
+    run_subprocesses(address, server, client)
+
+
+def test_ssl_reconnecting_async_example():
+    address = 'localhost:36436'
+    server = 'async_ssl_server.py'
+    client = 'async_reconnecting_ssl_client.py'
+    run_subprocesses(address, server, client)

test/test_large_read.capnp

@@ -0,0 +1,13 @@
+@0x86dbb3b256f5d2af;
+
+struct Row {
+  values @0 :List(Int32);
+}
+
+struct MultiArray {
+  rows @0 :List(Row);
+}
+
+struct Msg {
+  data @0 :List(UInt8);
+}

test/test_large_read.py

@@ -0,0 +1,84 @@
+import os
+import platform
+import tempfile
+
+import pytest
+
+import capnp
+
+this_dir = os.path.dirname(__file__)
+
+
+@pytest.fixture
+def test_capnp():
+    return capnp.load(os.path.join(this_dir, 'test_large_read.capnp'))
+
+
+def test_large_read(test_capnp):
+    f = tempfile.TemporaryFile()
+
+    array = test_capnp.MultiArray.new_message()
+
+    row = array.init('rows', 1)[0]
+    values = row.init('values', 10000)
+    for i in range(len(values)):
+        values[i] = i
+
+    array.write_packed(f)
+    f.seek(0)
+
+    array = test_capnp.MultiArray.read_packed(f)
+    del f
+    assert array.rows[0].values[9000] == 9000
+
+def test_large_read_multiple(test_capnp):
+    f = tempfile.TemporaryFile()
+    msg1 = test_capnp.Msg.new_message()
+    msg1.data = [0x41] * 8192
+    msg1.write(f)
+    msg2 = test_capnp.Msg.new_message()
+    msg2.write(f)
+    f.seek(0)
+
+    for m in test_capnp.Msg.read_multiple(f):
+        pass
+
+def get_two_adjacent_messages(test_capnp):
+    msg1 = test_capnp.Msg.new_message()
+    msg1.data = [0x41] * 8192
+    m1 = msg1.to_bytes()
+    msg2 = test_capnp.Msg.new_message()
+    m2 = msg2.to_bytes()
+
+    return m1 + m2
+
+def test_large_read_multiple_bytes(test_capnp):
+    data = get_two_adjacent_messages(test_capnp)
+    for m in test_capnp.Msg.read_multiple_bytes(data):
+        pass
+
+    with pytest.raises(capnp.KjException):
+        data = get_two_adjacent_messages(test_capnp)[:-1]
+        for m in test_capnp.Msg.read_multiple_bytes(data):
+            pass
+
+    with pytest.raises(capnp.KjException):
+        data = get_two_adjacent_messages(test_capnp) + b' '
+        for m in test_capnp.Msg.read_multiple_bytes(data):
+            pass
+
+@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="PyPy memoryview support is limited")
+def test_large_read_mutltiple_bytes_memoryview(test_capnp):
+    data = get_two_adjacent_messages(test_capnp)
+    for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)):
+        pass
+
+    with pytest.raises(capnp.KjException):
+        data = get_two_adjacent_messages(test_capnp)[:-1]
+        for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)):
+            pass
+
+    with pytest.raises(capnp.KjException):
+        data = get_two_adjacent_messages(test_capnp) + b' '
+        for m in test_capnp.Msg.read_multiple_bytes(memoryview(data)):
+            pass

test/test_load.py

@@ -56,17 +56,23 @@ def test_failed_import():
         bar.foo = foo
 
 def test_defualt_import_hook():
-    import addressbook_capnp
+    # Make sure any previous imports of addressbook_capnp are gone
+    capnp.cleanup_global_schema_parser()
+
+    import addressbook_capnp # noqa: F401
 
 def test_dash_import():
-    import addressbook_with_dashes_capnp
+    import addressbook_with_dashes_capnp # noqa: F401
 
 def test_spaces_import():
-    import addressbook_with_spaces_capnp
+    import addressbook_with_spaces_capnp # noqa: F401
 
 def test_add_import_hook():
     capnp.add_import_hook([this_dir])
 
+    # Make sure any previous imports of addressbook_capnp are gone
+    capnp.cleanup_global_schema_parser()
+
     import addressbook_capnp
     addressbook_capnp.AddressBook.new_message()
 
@@ -75,6 +81,9 @@ def test_multiple_add_import_hook():
     capnp.add_import_hook()
     capnp.add_import_hook([this_dir])
 
+    # Make sure any previous imports of addressbook_capnp are gone
+    capnp.cleanup_global_schema_parser()
+
     import addressbook_capnp
     addressbook_capnp.AddressBook.new_message()
 
@@ -86,4 +95,4 @@ def test_remove_import_hook():
         del sys.modules['addressbook_capnp'] # hack to deal with it being imported already
 
     with pytest.raises(ImportError):
-        import addressbook_capnp
+        import addressbook_capnp # noqa: F401

test/test_regression.py

@@ -1,10 +1,19 @@
+# -*- coding: utf-8 -*-
+
 import pytest
 import capnp
 import os
 import math
+import sys
 
 this_dir = os.path.dirname(__file__)
 
+if sys.version_info[0] < 3:
+    EXPECT_BYTES = True
+else:
+    EXPECT_BYTES = False
+
+
 @pytest.fixture
 def addressbook():
     return capnp.load(os.path.join(this_dir, 'addressbook.capnp'))
@@ -39,7 +48,7 @@ def test_addressbook_message_classes(addressbook):
 
 
     def printAddressBook(fd):
-        message = capnp._PackedFdMessageReader(f.fileno())
+        message = capnp._PackedFdMessageReader(f)
         addressBook = message.get_root(addressbook.AddressBook)
 
         people = addressBook.people
@@ -62,7 +71,7 @@ def test_addressbook_message_classes(addressbook):
         assert bobPhones[0].type == 'home'
         assert bobPhones[1].number == "555-7654"
         assert bobPhones[1].type == 'work'
-        assert bob.employment.unemployed == None
+        assert bob.employment.unemployed is None
 
     f = open('example', 'w')
     writeAddressBook(f.fileno())
@@ -121,7 +130,7 @@ def test_addressbook(addressbook):
         assert bobPhones[0].type == 'home'
         assert bobPhones[1].number == "555-7654"
         assert bobPhones[1].type == 'work'
-        assert bob.employment.unemployed == None
+        assert bob.employment.unemployed is None
 
 
     f = open('example', 'w')
@@ -183,7 +192,7 @@ def test_addressbook_resizable(addressbook):
         assert bobPhones[0].type == 'home'
         assert bobPhones[1].number == "555-7654"
         assert bobPhones[1].type == 'work'
-        assert bob.employment.unemployed == None
+        assert bob.employment.unemployed is None
 
 
     f = open('example', 'w')
@@ -253,7 +262,7 @@ def test_addressbook_explicit_fields(addressbook):
         assert bobPhones[1]._get_by_field(phone_fields['number']) == "555-7654"
         assert bobPhones[1]._get_by_field(phone_fields['type']) == 'work'
         employment = bob._get_by_field(person_fields['employment'])
-        employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) == None
+        employment._get_by_field(addressbook.Person.Employment.schema.fields['unemployed']) is None
 
 
     f = open('example', 'w')
@@ -300,7 +309,7 @@ def init_all_types(builder):
     subBuilder.uInt64Field = 345678901234567890
     subBuilder.float32Field = -1.25e-10
     subBuilder.float64Field = 345
-    subBuilder.textField = "baz"
+    subBuilder.textField = "☃"
     subBuilder.dataField = b"qux"
     subSubBuilder = subBuilder.structField
     subSubBuilder.textField = "nested"
@@ -362,8 +371,8 @@ def check_list(reader, expected):
             assert reader[i] == v
 
 def check_all_types(reader):
-    assert reader.voidField == None
+    assert reader.voidField is None
-    assert reader.boolField == True
+    assert reader.boolField
     assert reader.int8Field == -123
     assert reader.int16Field == -12345
     assert reader.int32Field == -12345678
@@ -378,8 +387,8 @@ def check_all_types(reader):
     assert reader.dataField == b"bar"
 
     subReader = reader.structField
-    assert subReader.voidField == None
+    assert subReader.voidField is None
-    assert subReader.boolField == True
+    assert subReader.boolField
     assert subReader.int8Field == -12
     assert subReader.int16Field == 3456
     assert subReader.int32Field == -78901234
@@ -390,7 +399,15 @@ def check_all_types(reader):
     assert subReader.uInt64Field == 345678901234567890
     assert_almost(subReader.float32Field, -1.25e-10)
     assert subReader.float64Field == 345
-    assert subReader.textField == "baz"
+
+    assert subReader.textField == "☃"
+    # This assertion highlights the encoding we expect to see here, since
+    # otherwise this appears a bit magical...
+    if EXPECT_BYTES:
+        assert len(subReader.textField) == 3
+    else:
+        assert len(subReader.textField) == 1
+
     assert subReader.dataField == b"qux"
 
     subSubReader = subReader.structField
@@ -398,6 +415,12 @@ def check_all_types(reader):
     assert subSubReader.structField.textField == "really nested"
 
     assert subReader.enumField == "baz"
+    # Check that enums are hashable and can be used as keys in dicts
+    # interchangably with their string version.
+    assert hash(subReader.enumField) == hash('baz')
+    assert {subReader.enumField: 17}.get(subReader.enumField) == 17
+    assert {subReader.enumField: 17}.get('baz') == 17
+    assert {'baz': 17}.get(subReader.enumField) == 17
 
     check_list(subReader.voidList, [None, None, None])
     check_list(subReader.boolList, [False, True, False, True, True])
@@ -463,26 +486,26 @@ def check_all_types(reader):
 def test_build(all_types):
     root = all_types.TestAllTypes.new_message()
     init_all_types(root)
-    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read()
     assert str(root) + '\n' == expectedText
 
 def test_build_first_segment_size(all_types):
     root = all_types.TestAllTypes.new_message(1)
     init_all_types(root)
-    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read()
     assert str(root) + '\n' == expectedText
 
     root = all_types.TestAllTypes.new_message(1024 * 1024)
     init_all_types(root)
-    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read()
     assert str(root) + '\n' == expectedText
 
 def test_binary_read(all_types):
-    f = open(os.path.join(this_dir, 'all-types.binary'), 'r')
+    f = open(os.path.join(this_dir, 'all-types.binary'), 'r', encoding='utf8')
     root = all_types.TestAllTypes.read(f)
     check_all_types(root)
 
-    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read()
     assert str(root) + '\n' == expectedText
 
     # Test set_root().
@@ -495,11 +518,11 @@ def test_binary_read(all_types):
     check_all_types(builder2.get_root(all_types.TestAllTypes))
 
 def test_packed_read(all_types):
-    f = open(os.path.join(this_dir, 'all-types.packed'), 'r')
+    f = open(os.path.join(this_dir, 'all-types.packed'), 'r', encoding='utf8')
     root = all_types.TestAllTypes.read_packed(f)
     check_all_types(root)
 
-    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r').read()
+    expectedText = open(os.path.join(this_dir, 'all-types.txt'), 'r', encoding='utf8').read()
     assert str(root) + '\n' == expectedText
 
 def test_binary_write(all_types):

test/test_response.capnp

@@ -0,0 +1,13 @@
+@0x84249be5c3bff005;
+
+interface Foo {
+  foo @0 () -> (val :UInt32);
+}
+
+struct Bar {
+  foo @0 :Foo;
+}
+
+interface Baz {
+  grault @0 () -> (bar: Bar);
+}

test/test_response.py

@@ -0,0 +1,35 @@
+import test_response_capnp
+
+class FooServer(test_response_capnp.Foo.Server):
+    def __init__(self, val=1):
+        self.val = val
+
+    def foo(self, **kwargs):
+        return 1
+
+class BazServer(test_response_capnp.Baz.Server):
+    def __init__(self, val=1):
+        self.val = val
+
+    def grault(self, **kwargs):
+        return {"foo": FooServer()}
+
+def test_response_reference():
+    baz = test_response_capnp.Baz._new_client(BazServer())
+
+    bar = baz.grault().wait().bar
+
+    foo = bar.foo
+    # This used to cause an exception about invalid pointers because the response got garbage collected
+    assert foo.foo().wait().val == 1
+
+def test_response_reference2():
+    baz = test_response_capnp.Baz._new_client(BazServer())
+
+    bar = baz.grault().wait().bar
+
+    # This always worked since it saved the intermediate response object
+    response = baz.grault().wait()
+    bar = response.bar
+    foo = bar.foo
+    assert foo.foo().wait().val == 1

test/test_rpc.py

@@ -1,6 +1,9 @@
+'''
+rpc test
+'''
+
 import pytest
 import capnp
-import os
 import socket
 
 import test_capability_capnp
@@ -8,89 +11,32 @@ import test_capability_capnp
 
 class Server(test_capability_capnp.TestInterface.Server):
 
-    def __init__(self, val=1):
+    def __init__(self, val=100):
         self.val = val
 
     def foo(self, i, j, **kwargs):
         return str(i * 5 + self.val)
 
 
-def restore_func(ref_id):
+def test_simple_rpc_with_options():
-    return Server(100)
+    read, write = socket.socketpair()
 
-
+    _ = capnp.TwoPartyServer(write, bootstrap=Server())
-class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer):
+    # This traversal limit is too low to receive the response in, so we expect
-
+    # an exception during the call.
-    def restore(self, ref_id):
+    client = capnp.TwoPartyClient(read, traversal_limit_in_words=1)
-        assert ref_id.tag == 'testInterface'
-        return Server(100)
-
-
-def test_simple_rpc():
-    read, write = socket.socketpair(socket.AF_UNIX)
-
-    restorer = SimpleRestorer()
-    server = capnp.TwoPartyServer(write, restorer)
-    client = capnp.TwoPartyClient(read)
-
-    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
-    cap = client.restore(ref)
-    cap = cap.cast_as(test_capability_capnp.TestInterface)
-
-    remote = cap.foo(i=5)
-    response = remote.wait()
-
-    assert response.x == '125'
-
-
-def test_simple_rpc_restore_func():
-    read, write = socket.socketpair(socket.AF_UNIX)
-
-    server = capnp.TwoPartyServer(write, restore_func)
-    client = capnp.TwoPartyClient(read)
-
-    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
-    cap = client.restore(ref)
-    cap = cap.cast_as(test_capability_capnp.TestInterface)
-
-    remote = cap.foo(i=5)
-    response = remote.wait()
-
-    assert response.x == '125'
-
-
-def text_restore_func(objectId):
-    text = objectId.as_text()
-    assert text == 'testInterface'
-    return Server(100)
-
-
-def test_ez_rpc():
-    read, write = socket.socketpair(socket.AF_UNIX)
-
-    server = capnp.TwoPartyServer(write, text_restore_func)
-    client = capnp.TwoPartyClient(read)
-
-    cap = client.ez_restore('testInterface')
-    cap = cap.cast_as(test_capability_capnp.TestInterface)
-
-    remote = cap.foo(i=5)
-    response = remote.wait()
-
-    assert response.x == '125'
-
-    cap = client.restore(test_capability_capnp.TestSturdyRefObjectId.new_message())
-    cap = cap.cast_as(test_capability_capnp.TestInterface)
-
-    remote = cap.foo(i=5)
 
     with pytest.raises(capnp.KjException):
-        response = remote.wait()
+        cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface)
+
+        remote = cap.foo(i=5)
+        _ = remote.wait()
+
 
 def test_simple_rpc_bootstrap():
-    read, write = socket.socketpair(socket.AF_UNIX)
+    read, write = socket.socketpair()
 
-    server = capnp.TwoPartyServer(write, bootstrap=Server(100))
+    _ = capnp.TwoPartyServer(write, bootstrap=Server(100))
     client = capnp.TwoPartyClient(read)
 
     cap = client.bootstrap()

test/test_rpc_calculator.py

@@ -1,21 +1,83 @@
-import capnp
-import os
-import socket
 import gc
-
+import os
+import pytest
+import socket
+import subprocess
 import sys  # add examples dir to sys.path
-sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'examples'))
+import time
-import calculator_client
+
-import calculator_server
+import capnp
+
+examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples')
+sys.path.append(examples_dir)
+
+import calculator_client # noqa: E402
+import calculator_server # noqa: E402
 
 
 def test_calculator():
-    read, write = socket.socketpair(socket.AF_UNIX)
+    read, write = socket.socketpair()
 
-    server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl())
+    _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl())
     calculator_client.main(read)
 
 
+def run_subprocesses(address):
+    cmd = [sys.executable, os.path.join(examples_dir, 'calculator_server.py'), address]
+    server = subprocess.Popen(cmd)
+    retries = 30
+    if 'unix' in address:
+        addr = address.split(':')[1]
+        while True:
+            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            result = sock.connect_ex(addr)
+            if result == 0:
+                break
+            # Give the server some small amount of time to start listening
+            time.sleep(0.1)
+            retries -= 1
+            if retries == 0:
+                assert False, "Timed out waiting for server to start"
+    else:
+        addr, port = address.split(':')
+        while True:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            result = sock.connect_ex((addr, int(port)))
+            if result == 0:
+                break
+            sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+            result = sock.connect_ex((addr, int(port)))
+            if result == 0:
+                break
+            # Give the server some small amount of time to start listening
+            time.sleep(0.1)
+            retries -= 1
+            if retries == 0:
+                assert False, "Timed out waiting for server to start"
+    cmd = [sys.executable, os.path.join(examples_dir, 'calculator_client.py'), address]
+    client = subprocess.Popen(cmd)
+
+    ret = client.wait()
+    server.kill()
+    assert ret == 0
+
+
+def test_calculator_tcp():
+    address = 'localhost:36431'
+    run_subprocesses(address)
+
+
+@pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows")
+def test_calculator_unix():
+    path = '/tmp/pycapnp-test'
+    try:
+        os.unlink(path)
+    except OSError:
+        pass
+
+    address = 'unix:' + path
+    run_subprocesses(address)
+
 def test_calculator_gc():
     def new_evaluate_impl(old_evaluate_impl):
         def call(*args, **kwargs):
@@ -23,13 +85,13 @@ def test_calculator_gc():
             return old_evaluate_impl(*args, **kwargs)
         return call
 
-    read, write = socket.socketpair(socket.AF_UNIX)
+    read, write = socket.socketpair()
 
     # inject a gc.collect to the beginning of every evaluate_impl call
     evaluate_impl_orig = calculator_server.evaluate_impl
     calculator_server.evaluate_impl = new_evaluate_impl(evaluate_impl_orig)
 
-    server = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl())
+    _ = capnp.TwoPartyServer(write, bootstrap=calculator_server.CalculatorImpl())
     calculator_client.main(read)
 
     calculator_server.evaluate_impl = evaluate_impl_orig

test/test_serialization.py

@@ -5,6 +5,8 @@ import platform
 import test_regression
 import tempfile
 import pickle
+import mmap
+import sys
 
 this_dir = os.path.dirname(__file__)
 
@@ -40,6 +42,50 @@ def test_roundtrip_bytes(all_types):
     msg = all_types.TestAllTypes.from_bytes(message_bytes)
     test_regression.check_all_types(msg)
 
+@pytest.mark.skipif(
+    platform.python_implementation() == 'PyPy',
+    reason="TODO: Investigate why this works on CPython but fails on PyPy."
+)
+def test_roundtrip_segments(all_types):
+    msg = all_types.TestAllTypes.new_message()
+    test_regression.init_all_types(msg)
+    segments = msg.to_segments()
+    msg = all_types.TestAllTypes.from_segments(segments)
+    test_regression.check_all_types(msg)
+
+@pytest.mark.skipif(sys.version_info[0] < 3, reason="mmap doesn't implement the buffer interface under python 2.")
+def test_roundtrip_bytes_mmap(all_types):
+    msg = all_types.TestAllTypes.new_message()
+    test_regression.init_all_types(msg)
+
+    with tempfile.TemporaryFile() as f:
+        msg.write(f)
+        length = f.tell()
+
+        f.seek(0)
+        memory = mmap.mmap(f.fileno(), length)
+
+        msg = all_types.TestAllTypes.from_bytes(memory)
+        test_regression.check_all_types(msg)
+
+@pytest.mark.skipif(sys.version_info[0] < 3, reason="memoryview is a builtin on Python 3")
+def test_roundtrip_bytes_buffer(all_types):
+    msg = all_types.TestAllTypes.new_message()
+    test_regression.init_all_types(msg)
+
+    b = msg.to_bytes()
+    v = memoryview(b)
+    msg = all_types.TestAllTypes.from_bytes(v)
+    test_regression.check_all_types(msg)
+
+def test_roundtrip_bytes_fail(all_types):
+    with pytest.raises(TypeError):
+        all_types.TestAllTypes.from_bytes(42)
+
+@pytest.mark.skipif(
+    platform.python_implementation() == 'PyPy',
+    reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test."
+)
 def test_roundtrip_bytes_packed(all_types):
     msg = all_types.TestAllTypes.new_message()
     test_regression.init_all_types(msg)
@@ -106,7 +152,10 @@ def test_roundtrip_bytes_multiple_packed(all_types):
         i += 1
     assert i == 3
 
-@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now.")
+@pytest.mark.skipif(
+    platform.python_implementation() == 'PyPy',
+    reason="This works on my local PyPy v2.5.0, but is for some reason broken on TravisCI. Skip for now."
+)
 def test_roundtrip_dict(all_types):
     msg = all_types.TestAllTypes.new_message()
     test_regression.init_all_types(msg)
@@ -142,3 +191,36 @@ def test_pickle(all_types):
     msg2 = pickle.loads(data)
 
     test_regression.check_all_types(msg2)
+
+def test_from_bytes_traversal_limit(all_types):
+    size = 1024
+    bld = all_types.TestAllTypes.new_message()
+    bld.init("structList", size)
+    data = bld.to_bytes()
+
+    msg = all_types.TestAllTypes.from_bytes(data)
+    with pytest.raises(capnp.KjException):
+        for i in range(0, size):
+            msg.structList[i].uInt8Field == 0
+
+    msg = all_types.TestAllTypes.from_bytes(data,
+        traversal_limit_in_words=2**62)
+    for i in range(0, size):
+        assert msg.structList[i].uInt8Field == 0
+
+
+def test_from_bytes_packed_traversal_limit(all_types):
+    size = 1024
+    bld = all_types.TestAllTypes.new_message()
+    bld.init("structList", size)
+    data = bld.to_bytes_packed()
+
+    msg = all_types.TestAllTypes.from_bytes_packed(data)
+    with pytest.raises(capnp.KjException):
+        for i in range(0, size):
+            msg.structList[i].uInt8Field == 0
+
+    msg = all_types.TestAllTypes.from_bytes_packed(data,
+        traversal_limit_in_words=2**62)
+    for i in range(0, size):
+        assert msg.structList[i].uInt8Field == 0

test/test_struct.py

@@ -77,7 +77,10 @@ def test_which_reader(addressbook):
         addresses.which
 
 
-@pytest.mark.skipif(capnp.version.LIBCAPNP_VERSION < 5000, reason="Using ints as enums requires v0.5.0+ of the C++ capnp library")
+@pytest.mark.skipif(
+    capnp.version.LIBCAPNP_VERSION < 5000,
+    reason="Using ints as enums requires v0.5.0+ of the C++ capnp library"
+)
 def test_enum(addressbook):
     addresses = addressbook.AddressBook.new_message()
     people = addresses.init('people', 2)
@@ -107,6 +110,22 @@ def test_builder_set(addressbook):
         person.foo = 'test'
 
 
+def test_builder_set_from_list(all_types):
+    msg = all_types.TestAllTypes.new_message()
+
+    msg.int32List = [0, 1, 2]
+
+    assert list(msg.int32List) == [0, 1, 2]
+
+
+def test_builder_set_from_tuple(all_types):
+    msg = all_types.TestAllTypes.new_message()
+
+    msg.int32List = (0, 1, 2)
+
+    assert list(msg.int32List) == [0, 1, 2]
+
+
 def test_null_str(all_types):
     msg = all_types.TestAllTypes.new_message()
 
@@ -172,11 +191,6 @@ def test_set_dict_union(addressbook):
 
     assert person.employment.employer.name == 'foo'
 
-try:
-    basestring  # attempt to evaluate basestring
-    def isstr(s):
-        return isinstance(s, basestring)
-except NameError:
 def isstr(s):
     return isinstance(s, str)
 
@@ -211,10 +225,28 @@ def test_to_dict_verbose(addressbook):
 
 
 def test_to_dict_ordered(addressbook):
-    person = addressbook.Person.new_message(**{'name': 'Alice', 'phones': [{'type': 'mobile', 'number': '555-1212'}], 'id': 123, 'employment': {'school': 'MIT'}, 'email': 'alice@example.com'})
+    person = addressbook.Person.new_message(**{
+        'name': 'Alice',
+        'phones': [{'type': 'mobile', 'number': '555-1212'}],
+        'id': 123,
+        'employment': {'school': 'MIT'}, 'email': 'alice@example.com'
+    })
 
     if sys.version_info >= (2, 7):
         assert list(person.to_dict(ordered=True).keys()) == ['id', 'name', 'email', 'phones', 'employment']
     else:
         with pytest.raises(Exception):
             person.to_dict(ordered=True)
+
+def test_nested_list(addressbook):
+    struct = addressbook.NestedList.new_message()
+    struct.init('list', 2)
+
+    struct.list.init(0, 1)
+    struct.list.init(1, 2)
+
+    struct.list[0][0] = 1
+    struct.list[1][0] = 2
+    struct.list[1][1] = 3
+
+    assert struct.to_dict()["list"] == [[1], [2, 3]]

test/test_threads.py

@@ -1,20 +1,38 @@
-import capnp
+'''
-import pytest
+thread test
-import test_capability_capnp
+'''
+
+import platform
 import socket
 import threading
-import platform
 
-@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy")
+import pytest
+
+import capnp
+import test_capability_capnp
+
+@pytest.mark.skipif(
+    platform.python_implementation() == 'PyPy',
+    reason="pycapnp's GIL handling isn't working properly at the moment for PyPy"
+)
 def test_making_event_loop():
+    '''
+    Event loop test
+    '''
     capnp.remove_event_loop(True)
     capnp.create_event_loop()
 
     capnp.remove_event_loop()
     capnp.create_event_loop()
 
-@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy")
+@pytest.mark.skipif(
+    platform.python_implementation() == 'PyPy',
+    reason="pycapnp's GIL handling isn't working properly at the moment for PyPy"
+)
 def test_making_threaded_event_loop():
+    '''
+    Threaded event loop test
+    '''
     capnp.remove_event_loop(True)
     capnp.create_event_loop(True)
 
@@ -23,31 +41,34 @@ def test_making_threaded_event_loop():
 
 
 class Server(test_capability_capnp.TestInterface.Server):
-
+    '''
-    def __init__(self, val=1):
+    Server
+    '''
+    def __init__(self, val=100):
         self.val = val
 
     def foo(self, i, j, **kwargs):
+        '''
+        foo
+        '''
         return str(i * 5 + self.val)
 
 
-class SimpleRestorer(test_capability_capnp.TestSturdyRefObjectId.Restorer):
+@pytest.mark.skipif(
-
+    platform.python_implementation() == 'PyPy',
-    def restore(self, ref_id):
+    reason="pycapnp's GIL handling isn't working properly at the moment for PyPy"
-        assert ref_id.tag == 'testInterface'
+)
-        return Server(100)
-
-
-@pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="pycapnp's GIL handling isn't working properly at the moment for PyPy")
 def test_using_threads():
+    '''
+    Thread test
+    '''
     capnp.remove_event_loop(True)
     capnp.create_event_loop(True)
 
-    read, write = socket.socketpair(socket.AF_UNIX)
+    read, write = socket.socketpair()
 
     def run_server():
-        restorer = SimpleRestorer()
+        _ = capnp.TwoPartyServer(write, bootstrap=Server())
-        server = capnp.TwoPartyServer(write, restorer)
         capnp.wait_forever()
 
     server_thread = threading.Thread(target=run_server)
@@ -55,10 +76,7 @@ def test_using_threads():
     server_thread.start()
 
     client = capnp.TwoPartyClient(read)
-
+    cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface)
-    ref = test_capability_capnp.TestSturdyRefObjectId.new_message(tag='testInterface')
-    cap = client.restore(ref)
-    cap = cap.cast_as(test_capability_capnp.TestInterface)
 
     remote = cap.foo(i=5)
     response = remote.wait()

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27,py32,py33
+envlist = py27,py34,py35,py36
 
 [testenv]
 deps=