IQ.Lvbs/pycapnp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updating to capnproto v0.8.0

Browse Source 
- Includes some test stabilization
- Fixes manylinux2010 build issues (linker flag order due to old gcc)
- More rigorous python setup.py clean
- Requires capnproto v0.8.0 or greater
- Including system libcapnp include path for import (e.g. import
  stream_capnp)
- Bundle libcapnp .capnp files when not using system libcapnp
- Removing more distutils usage. Now using pkg-config to determine the
  system version of libcapnp (mainly for Linux, but should work on macOS
  with brew)
- Removed dead code

Resolves issues #215 #216 #217
Lots of fixes for Issue #218 (all sorts of retry methods needed for
GitHub Actions)
This commit is contained in:
Jacob Alexander
2020-06-03 22:49:52 -07:00
parent 7bfd5b962b
commit 255119b838
28 changed files with 206 additions and 834 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/manylinux2010.yml vendored
View File

@@ -24,6 +24,8 @@ jobs:
/opt/python/${{ matrix.python-version }}/bin/python -m pip install -r requirements.txt /opt/python/${{ matrix.python-version }}/bin/python -m pip install -r requirements.txt
/opt/python/${{ matrix.python-version }}/bin/python -m pip install auditwheel /opt/python/${{ matrix.python-version }}/bin/python -m pip install auditwheel
- name: Build pycapnp and install - name: Build pycapnp and install
env:
LDFLAGS: -Wl,--no-as-needed -lrt
run: | run: |
/opt/python/${{ matrix.python-version }}/bin/python setup.py build /opt/python/${{ matrix.python-version }}/bin/python setup.py build
- name: Packaging - name: Packaging

5
.gitignore vendored
View File

@@ -8,6 +8,8 @@
*.egg-info *.egg-info
dist dist
build build
build32
build64
eggs eggs
parts parts
bin bin
@@ -47,3 +49,6 @@ capnp/lib/capnp.h
bundled/ bundled/
example example
*.iml *.iml
# capnp files
*.capnp

14
README.md
View File

@@ -20,7 +20,7 @@
- ninja (macOS + Linux) - ninja (macOS + Linux)
- Visual Studio 2017+ - Visual Studio 2017+
* capnproto-0.7.0 * capnproto-0.8.0
- Not necessary if using bundled capnproto - Not necessary if using bundled capnproto
32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake. 32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake.
@@ -55,6 +55,18 @@ To force bundled python:
pip install --install-option "--force-bundled-libcapnp" . pip install --install-option "--force-bundled-libcapnp" .
``` ```
Slightly more prompt error messages using distutils rather than pip.
```bash
python setup.py install --force-bundled-libcapnp
```
The bundling system isn't that smart so it might be necessary to clean up the bundled build when changing versions:
```bash
python setup.py clean
```
## Python Versions ## Python Versions

4
buildutils/__init__.py
View File

@@ -5,9 +5,5 @@ Largely adapted from h5py
# flake8: noqa F401 F403 # flake8: noqa F401 F403
from .msg import * from .msg import *
from .config import *
from .detect import *
from .bundle import * from .bundle import *
from .misc import *
from .patch import *
from .build import * from .build import *

6
buildutils/build.py
View File

@@ -22,7 +22,9 @@ def build_libcapnp(bundle_dir, build_dir):
os.mkdir(tmp_dir) os.mkdir(tmp_dir)
cxxflags = os.environ.get('CXXFLAGS', None) cxxflags = os.environ.get('CXXFLAGS', None)
ldflags = os.environ.get('LDFLAGS', None)
os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG' os.environ['CXXFLAGS'] = (cxxflags or '') + ' -O2 -DNDEBUG'
os.environ['LDFLAGS'] = (ldflags or '')
# Enable ninja for compilation if available # Enable ninja for compilation if available
build_type = [] build_type = []
@@ -77,5 +79,9 @@ def build_libcapnp(bundle_dir, build_dir):
del os.environ['CXXFLAGS'] del os.environ['CXXFLAGS']
else: else:
os.environ['CXXFLAGS'] = cxxflags os.environ['CXXFLAGS'] = cxxflags
if ldflags is None:
del os.environ['LDFLAGS']
else:
os.environ['LDFLAGS'] = ldflags
if returncode != 0: if returncode != 0:
raise RuntimeError('capnproto compilation failed: {}'.format(returncode)) raise RuntimeError('capnproto compilation failed: {}'.format(returncode))

13
buildutils/bundle.py
View File

@@ -27,7 +27,7 @@ pjoin = os.path.join
# #
bundled_version = (0, 7, 0) bundled_version = (0, 8, 0)
libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_name = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version)
libcapnp_url = "https://capnproto.org/" + libcapnp_name libcapnp_url = "https://capnproto.org/" + libcapnp_name
@@ -92,14 +92,3 @@ def fetch_libcapnp(savedir, url=None):
else: else:
cpp_dir = os.path.join(with_version, 'c++') cpp_dir = os.path.join(with_version, 'c++')
shutil.move(cpp_dir, dest) shutil.move(cpp_dir, dest)
# XXX (HaaTa): There's a bug in capnproto-0.7.0 on CentOS 6 (or any glibc older than 2.16)
# This patches the issue
with fileinput.FileInput(os.path.join(dest, 'src', 'kj', 'table.c++'), inplace=True, backup='.bak') as f:
for line in f:
print(
line.replace(
'__APPLE__ || __BIONIC__', '__APPLE__ || __BIONIC__ || (defined(__GLIBC__) && (__GLIBC__ == 2) && (__GLIBC_MINOR__ < 16))'
),
end=''
)

21
buildutils/config.py
View File

@@ -1,21 +0,0 @@
"""Config functions"""
#
# Copyright (C) PyZMQ Developers
#
# This file is part of pyzmq, copied and adapted from h5py.
# h5py source used under the New BSD license
#
# h5py: <http://code.google.com/p/h5py/>
#
# Distributed under the terms of the New BSD License. The full license is in
# the file COPYING.BSD, distributed as part of this software.
#
#
# Utility functions (adapted from h5py: http://h5py.googlecode.com)
#
def v_str(v_tuple):
"""turn (2,0,1) into '2.0.1'."""
return ".".join(str(x) for x in v_tuple)

154
buildutils/detect.py
View File

@@ -1,154 +0,0 @@
"""Detect zmq version"""
#
# Copyright (C) PyZMQ Developers
#
# This file is part of pyzmq, copied and adapted from h5py.
# h5py source used under the New BSD license
#
# h5py: <http://code.google.com/p/h5py/>
#
# Distributed under the terms of the New BSD License. The full license is in
# the file COPYING.BSD, distributed as part of this software.
#
#
# Adapted for use in pycapnp from pyzmq. See https://github.com/zeromq/pyzmq
# for original project.
import shutil
import sys
import os
import logging
import platform
from distutils import ccompiler
from distutils.ccompiler import get_default_compiler
import tempfile
from .misc import get_compiler, get_output_error
from .patch import patch_lib_paths
pjoin = os.path.join
#
# Utility functions (adapted from h5py: http://h5py.googlecode.com)
#
def test_compilation(cfile, compiler=None, **compiler_attrs):
"""Test simple compilation with given settings"""
cc = get_compiler(compiler, **compiler_attrs)
efile, _ = os.path.splitext(cfile)
cpreargs = lpreargs = []
if sys.platform == 'darwin':
# use appropriate arch for compiler
if platform.architecture()[0] == '32bit':
if platform.processor() == 'powerpc':
cpu = 'ppc'
else:
cpu = 'i386'
cpreargs = ['-arch', cpu]
lpreargs = ['-arch', cpu, '-undefined', 'dynamic_lookup']
else:
# allow for missing UB arch, since it will still work:
lpreargs = ['-undefined', 'dynamic_lookup']
if sys.platform == 'sunos5':
if platform.architecture()[0] == '32bit':
lpreargs = ['-m32']
else:
lpreargs = ['-m64']
extra_compile_args = compiler_attrs.get('extra_compile_args', [])
if os.name != 'nt':
extra_compile_args += ['--std=c++14']
extra_link_args = compiler_attrs.get('extra_link_args', [])
if cc.compiler_type == 'msvc':
extra_link_args += ['/MANIFEST']
objs = cc.compile([cfile], extra_preargs=cpreargs, extra_postargs=extra_compile_args)
cc.link_executable(objs, efile, extra_preargs=lpreargs, extra_postargs=extra_link_args)
return efile
def detect_version(basedir, compiler=None, **compiler_attrs):
"""Compile, link & execute a test program, in empty directory `basedir`.
The C compiler will be updated with any keywords given via setattr.
Parameters
----------
basedir : path
The location where the test program will be compiled and run
compiler : str
The distutils compiler key (e.g. 'unix', 'msvc', or 'mingw32')
**compiler_attrs : dict
Any extra compiler attributes, which will be set via ``setattr(cc)``.
Returns
-------
A dict of properties for zmq compilation, with the following two keys:
vers : tuple
The ZMQ version as a tuple of ints, e.g. (2,2,0)
settings : dict
The compiler options used to compile the test function, e.g. `include_dirs`,
`library_dirs`, `libs`, etc.
"""
if compiler is None:
compiler = get_default_compiler()
cfile = pjoin(basedir, 'vers.cpp')
shutil.copy(pjoin(os.path.dirname(__file__), 'vers.cpp'), cfile)
# check if we need to link against Realtime Extensions library
if sys.platform.startswith('linux'):
cc = ccompiler.new_compiler(compiler=compiler)
cc.output_dir = basedir
if not cc.has_function('timer_create'):
if 'libraries' not in compiler_attrs:
compiler_attrs['libraries'] = []
compiler_attrs['libraries'].append('rt')
cc = get_compiler(compiler=compiler, **compiler_attrs)
efile = test_compilation(cfile, compiler=cc)
patch_lib_paths(efile, cc.library_dirs)
rc, so, se = get_output_error([efile])
if rc:
msg = "Error running version detection script:\n%s\n%s" % (so, se)
logging.error(msg)
raise IOError(msg)
handlers = {'vers': lambda val: tuple(int(v) for v in val.split('.'))}
props = {}
for line in (x for x in so.split('\n') if x):
key, val = line.split(':')
props[key] = handlers[key](val)
return props
def test_build(**compiler_attrs):
"""do a test build of libcapnp"""
tmp_dir = tempfile.mkdtemp()
# line()
# info("Configure: Autodetecting Cap'n Proto settings...")
# info(" Custom Cap'n Proto dir: %s" % prefix)
try:
detected = detect_version(tmp_dir, None, **compiler_attrs)
finally:
erase_dir(tmp_dir)
# info(" Cap'n Proto version detected: %s" % v_str(detected['vers']))
return detected
def erase_dir(path):
"""Erase directory"""
try:
shutil.rmtree(path)
except Exception:
pass

64
buildutils/misc.py
View File

@@ -1,64 +0,0 @@
"""misc build utility functions"""
# Copyright (c) PyZMQ Developers
# Distributed under the terms of the Modified BSD License.
import os
import logging
from distutils import ccompiler
from distutils.sysconfig import customize_compiler
from pipes import quote
from subprocess import Popen, PIPE
pjoin = os.path.join
def customize_mingw(cc):
"""customize mingw"""
# strip -mno-cygwin from mingw32 (Python Issue #12641)
for cmd in [cc.compiler, cc.compiler_cxx, cc.compiler_so, cc.linker_exe, cc.linker_so]:
if '-mno-cygwin' in cmd:
cmd.remove('-mno-cygwin')
# remove problematic msvcr90
if 'msvcr90' in cc.dll_libraries:
cc.dll_libraries.remove('msvcr90')
def customize_msvc(cc):
pass
def get_compiler(compiler, **compiler_attrs):
"""get and customize a compiler"""
if compiler is None or isinstance(compiler, str):
cc = ccompiler.new_compiler(compiler=compiler)
customize_compiler(cc)
if cc.compiler_type == 'mingw32':
customize_mingw(cc)
elif cc.compiler_type == 'msvc':
customize_msvc(cc)
else:
cc = compiler
for name, val in compiler_attrs.items():
setattr(cc, name, val)
return cc
def get_output_error(cmd):
"""Return the exit status, stdout, stderr of a command"""
if not isinstance(cmd, list):
cmd = [cmd]
logging.debug("Running: %s", ' '.join(map(quote, cmd)))
try:
result = Popen(cmd, stdout=PIPE, stderr=PIPE)
except IOError as e:
return -1, '', 'Failed to run %r: %r' % (cmd, e)
so, se = result.communicate()
# unicode:
so = so.decode('utf8', 'replace')
se = se.decode('utf8', 'replace')
return result.returncode, so, se

67
buildutils/patch.py
View File

@@ -1,67 +0,0 @@
"""utils for patching libraries"""
# Copyright (c) PyZMQ Developers.
# Distributed under the terms of the Modified BSD License.
import re
import sys
import os
import logging
from .misc import get_output_error
pjoin = os.path.join
# LIB_PAT from delocate
LIB_PAT = re.compile(
r"\s*(.*) \(compatibility version (\d+\.\d+\.\d+), current version (\d+\.\d+\.\d+)\)"
)
def _get_libs(fname):
rc, so, se = get_output_error(['otool', '-L', fname])
if rc:
logging.error("otool -L %s failed: %r", fname, se)
return
for line in so.splitlines()[1:]:
m = LIB_PAT.match(line)
if m:
yield m.group(1)
def _find_library(lib, path):
"""Find a library"""
for d in path[::-1]:
real_lib = os.path.join(d, lib)
if os.path.exists(real_lib):
return real_lib
return None
def _install_name_change(fname, lib, real_lib):
rc, so, se = get_output_error(['install_name_tool', '-change', lib, real_lib, fname])
if rc:
logging.error("Couldn't update load path: %s", se)
def patch_lib_paths(fname, library_dirs):
"""Load any weakly-defined libraries from their real location
(only on OS X)
- Find libraries with `otool -L`
- Update with `install_name_tool -change`
"""
if sys.platform != 'darwin':
return
libs = _get_libs(fname)
for lib in libs:
if not lib.startswith(('@', '/')):
real_lib = _find_library(lib, library_dirs)
if real_lib:
_install_name_change(fname, lib, real_lib)
__all__ = ['patch_lib_paths']

9
buildutils/vers.cpp
View File

@@ -1,9 +0,0 @@
// check libcapnp version
#include <stdio.h>
#include "capnp/common.h"
int main(int argc, char **argv){
fprintf(stdout, "vers: %d.%d.%d\n", CAPNP_VERSION_MAJOR, CAPNP_VERSION_MINOR, CAPNP_VERSION_MICRO);
return 0;
}

26
capnp/c++.capnp
View File

@@ -1,26 +0,0 @@
# Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
# Licensed under the MIT License:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
@0xbdf87d7bb8304e81;
$namespace("capnp::annotations");
annotation namespace(file): Text;
annotation name(field, enumerant, struct, enum, interface, method, param, group, union): Text;

2
capnp/helpers/checkCompiler.h
View File

@@ -5,4 +5,4 @@
#include "capnp/dynamic.h" #include "capnp/dynamic.h"
static_assert(CAPNP_VERSION >= 7000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.7 and then re-install this python library"); static_assert(CAPNP_VERSION >= 8000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.8 and then re-install this python library");

11
capnp/lib/capnp.pyx
View File

@@ -4070,6 +4070,17 @@ def add_import_hook(additional_paths=[]):
if _importer is not None: if _importer is not None:
remove_import_hook() remove_import_hook()
# Automatically include the system and built-in capnp paths
# Highest priority at position 0
extra_paths = [
_os.path.join(_os.path.dirname(__file__), '..'), # Built-in (only used if bundled)
'/usr/local/include/capnp', # Common macOS brew location
'/usr/include/capnp', # Common posix location
]
for path in extra_paths:
if _os.path.isdir(path):
additional_paths.append(path)
_importer = _Importer(additional_paths) _importer = _Importer(additional_paths)
_sys.meta_path.append(_importer) _sys.meta_path.append(_importer)

383
capnp/schema.capnp
View File

@@ -1,383 +0,0 @@
# Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
using Cxx = import "c++.capnp";
@0xa93fc509624c72d9;
$Cxx.namespace("capnp::schema");
using Id = UInt64;
# The globally-unique ID of a file, type, or annotation.
struct Node {
id @0 :Id;
displayName @1 :Text;
# Name to present to humans to identify this Node. You should not attempt to parse this. Its
# format could change. It is not guaranteed to be unique.
#
# (On Zooko's triangle, this is the node's nickname.)
displayNamePrefixLength @2 :UInt32;
# If you want a shorter version of `displayName` (just naming this node, without its surrounding
# scope), chop off this many characters from the beginning of `displayName`.
scopeId @3 :Id;
# ID of the lexical parent node. Typically, the scope node will have a NestedNode pointing back
# at this node, but robust code should avoid relying on this (and, in fact, group nodes are not
# listed in the outer struct's nestedNodes, since they are listed in the fields). `scopeId` is
# zero if the node has no parent, which is normally only the case with files, but should be
# allowed for any kind of node (in order to make runtime type generation easier).
nestedNodes @4 :List(NestedNode);
# List of nodes nested within this node, along with the names under which they were declared.
struct NestedNode {
name @0 :Text;
# Unqualified symbol name. Unlike Node.name, this *can* be used programmatically.
#
# (On Zooko's triangle, this is the node's petname according to its parent scope.)
id @1 :Id;
# ID of the nested node. Typically, the target node's scopeId points back to this node, but
# robust code should avoid relying on this.
}
annotations @5 :List(Annotation);
# Annotations applied to this node.
union {
# Info specific to each kind of node.
file @6 :Void;
struct :group {
dataWordCount @7 :UInt16;
# Size of the data section, in words.
pointerCount @8 :UInt16;
# Size of the pointer section, in pointers (which are one word each).
preferredListEncoding @9 :ElementSize;
# The preferred element size to use when encoding a list of this struct. If this is anything
# other than `inlineComposite` then the struct is one word or less in size and is a candidate
# for list packing optimization.
isGroup @10 :Bool;
# If true, then this "struct" node is actually not an independent node, but merely represents
# some named union or group within a particular parent struct. This node's scopeId refers
# to the parent struct, which may itself be a union/group in yet another struct.
#
# All group nodes share the same dataWordCount and pointerCount as the top-level
# struct, and their fields live in the same ordinal and offset spaces as all other fields in
# the struct.
#
# Note that a named union is considered a special kind of group -- in fact, a named union
# is exactly equivalent to a group that contains nothing but an unnamed union.
discriminantCount @11 :UInt16;
# Number of fields in this struct which are members of an anonymous union, and thus may
# overlap. If this is non-zero, then a 16-bit discriminant is present indicating which
# of the overlapping fields is active. This can never be 1 -- if it is non-zero, it must be
# two or more.
#
# Note that the fields of an unnamed union are considered fields of the scope containing the
# union -- an unnamed union is not its own group. So, a top-level struct may contain a
# non-zero discriminant count. Named unions, on the other hand, are equivalent to groups
# containing unnamed unions. So, a named union has its own independent schema node, with
# `isGroup` = true.
discriminantOffset @12 :UInt32;
# If `discriminantCount` is non-zero, this is the offset of the union discriminant, in
# multiples of 16 bits.
fields @13 :List(Field);
# Fields defined within this scope (either the struct's top-level fields, or the fields of
# a particular group; see `isGroup`).
#
# The fields are sorted by ordinal number, but note that because groups share the same
# ordinal space, the field's index in this list is not necessarily exactly its ordinal.
# On the other hand, the field's position in this list does remain the same even as the
# protocol evolves, since it is not possible to insert or remove an earlier ordinal.
# Therefore, for most use cases, if you want to identify a field by number, it may make the
# most sense to use the field's index in this list rather than its ordinal.
}
enum :group {
enumerants@14 :List(Enumerant);
# Enumerants ordered by numeric value (ordinal).
}
interface :group {
methods @15 :List(Method);
# Methods ordered by ordinal.
extends @31 :List(Id);
# Superclasses of this interface.
}
const :group {
type @16 :Type;
value @17 :Value;
}
annotation :group {
type @18 :Type;
targetsFile @19 :Bool;
targetsConst @20 :Bool;
targetsEnum @21 :Bool;
targetsEnumerant @22 :Bool;
targetsStruct @23 :Bool;
targetsField @24 :Bool;
targetsUnion @25 :Bool;
targetsGroup @26 :Bool;
targetsInterface @27 :Bool;
targetsMethod @28 :Bool;
targetsParam @29 :Bool;
targetsAnnotation @30 :Bool;
}
}
}
struct Field {
# Schema for a field of a struct.
name @0 :Text;
codeOrder @1 :UInt16;
# Indicates where this member appeared in the code, relative to other members.
# Code ordering may have semantic relevance -- programmers tend to place related fields
# together. So, using code ordering makes sense in human-readable formats where ordering is
# otherwise irrelevant, like JSON. The values of codeOrder are tightly-packed, so the maximum
# value is count(members) - 1. Fields that are members of a union are only ordered relative to
# the other members of that union, so the maximum value there is count(union.members).
annotations @2 :List(Annotation);
const noDiscriminant :UInt16 = 0xffff;
discriminantValue @3 :UInt16 = Field.noDiscriminant;
# If the field is in a union, this is the value which the union's discriminant should take when
# the field is active. If the field is not in a union, this is 0xffff.
union {
slot :group {
# A regular, non-group, non-fixed-list field.
offset @4 :UInt32;
# Offset, in units of the field's size, from the beginning of the section in which the field
# resides. E.g. for a UInt32 field, multiply this by 4 to get the byte offset from the
# beginning of the data section.
type @5 :Type;
defaultValue @6 :Value;
hadExplicitDefault @10 :Bool;
# Whether the default value was specified explicitly. Non-explicit default values are always
# zero or empty values. Usually, whether the default value was explicit shouldn't matter.
# The main use case for this flag is for structs representing method parameters:
# explicitly-defaulted parameters may be allowed to be omitted when calling the method.
}
group :group {
# A group.
typeId @7 :Id;
# The ID of the group's node.
}
}
ordinal :union {
implicit @8 :Void;
explicit @9 :UInt16;
# The original ordinal number given to the field. You probably should NOT use this; if you need
# a numeric identifier for a field, use its position within the field array for its scope.
# The ordinal is given here mainly just so that the original schema text can be reproduced given
# the compiled version -- i.e. so that `capnp compile -ocapnp` can do its job.
}
}
struct Enumerant {
# Schema for member of an enum.
name @0 :Text;
codeOrder @1 :UInt16;
# Specifies order in which the enumerants were declared in the code.
# Like Struct.Field.codeOrder.
annotations @2 :List(Annotation);
}
struct Method {
# Schema for method of an interface.
name @0 :Text;
codeOrder @1 :UInt16;
# Specifies order in which the methods were declared in the code.
# Like Struct.Field.codeOrder.
paramStructType @2 :Id;
# ID of the parameter struct type. If a named parameter list was specified in the method
# declaration (rather than a single struct parameter type) then a corresponding struct type is
# auto-generated. Such an auto-generated type will not be listed in the interface's
# `nestedNodes` and its `scopeId` will be zero -- it is completely detached from the namespace.
resultStructType @3 :Id;
# ID of the return struct type; similar to `paramStructType`.
annotations @4 :List(Annotation);
}
struct Type {
# Represents a type expression.
union {
# The ordinals intentionally match those of Value.
void @0 :Void;
bool @1 :Void;
int8 @2 :Void;
int16 @3 :Void;
int32 @4 :Void;
int64 @5 :Void;
uint8 @6 :Void;
uint16 @7 :Void;
uint32 @8 :Void;
uint64 @9 :Void;
float32 @10 :Void;
float64 @11 :Void;
text @12 :Void;
data @13 :Void;
list :group {
elementType @14 :Type;
}
enum :group {
typeId @15 :Id;
}
struct :group {
typeId @16 :Id;
}
interface :group {
typeId @17 :Id;
}
anyPointer @18 :Void;
}
}
struct Value {
# Represents a value, e.g. a field default value, constant value, or annotation value.
union {
# The ordinals intentionally match those of Type.
void @0 :Void;
bool @1 :Bool;
int8 @2 :Int8;
int16 @3 :Int16;
int32 @4 :Int32;
int64 @5 :Int64;
uint8 @6 :UInt8;
uint16 @7 :UInt16;
uint32 @8 :UInt32;
uint64 @9 :UInt64;
float32 @10 :Float32;
float64 @11 :Float64;
text @12 :Text;
data @13 :Data;
list @14 :AnyPointer;
enum @15 :UInt16;
struct @16 :AnyPointer;
interface @17 :Void;
# The only interface value that can be represented statically is "null", whose methods always
# throw exceptions.
anyPointer @18 :AnyPointer;
}
}
struct Annotation {
# Describes an annotation applied to a declaration. Note AnnotationNode describes the
# annotation's declaration, while this describes a use of the annotation.
id @0 :Id;
# ID of the annotation node.
value @1 :Value;
}
enum ElementSize {
# Possible element sizes for encoded lists. These correspond exactly to the possible values of
# the 3-bit element size component of a list pointer.
empty @0; # aka "void", but that's a keyword.
bit @1;
byte @2;
twoBytes @3;
fourBytes @4;
eightBytes @5;
pointer @6;
inlineComposite @7;
}
struct CodeGeneratorRequest {
nodes @0 :List(Node);
# All nodes parsed by the compiler, including for the files on the command line and their
# imports.
requestedFiles @1 :List(RequestedFile);
# Files which were listed on the command line.
struct RequestedFile {
id @0 :Id;
# ID of the file.
filename @1 :Text;
# Name of the file as it appeared on the command-line (minus the src-prefix). You may use
# this to decide where to write the output.
imports @2 :List(Import);
# List of all imported paths seen in this file.
struct Import {
id @0 :Id;
# ID of the imported file.
name @1 :Text;
# Name which *this* file used to refer to the foreign file. This may be a relative name.
# This information is provided because it might be useful for code generation, e.g. to
# generate #include directives in C++. We don't put this in Node.file because this
# information is only meaningful at compile time anyway.
#
# (On Zooko's triangle, this is the import's petname according to the importing file.)
}
}
}

1
examples/async_calculator_client.py
View File

@@ -55,6 +55,7 @@ async def main(host):
print("Try IPv4") print("Try IPv4")
reader, writer = await asyncio.open_connection( reader, writer = await asyncio.open_connection(
addr, port, addr, port,
family=socket.AF_INET
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

1
examples/async_calculator_server.py
View File

@@ -214,6 +214,7 @@ async def main():
server = await asyncio.start_server( server = await asyncio.start_server(
new_connection, new_connection,
addr, port, addr, port,
family=socket.AF_INET
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

1
examples/async_client.py
View File

@@ -56,6 +56,7 @@ async def main(host):
print("Try IPv4") print("Try IPv4")
reader, writer = await asyncio.open_connection( reader, writer = await asyncio.open_connection(
addr, port, addr, port,
family=socket.AF_INET
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

1
examples/async_reconnecting_ssl_client.py
View File

@@ -86,6 +86,7 @@ async def main(host):
reader, writer = await asyncio.open_connection( reader, writer = await asyncio.open_connection(
addr, port, addr, port,
ssl=ctx, ssl=ctx,
family=socket.AF_INET
) )
except OSError: except OSError:
print("Try IPv6") print("Try IPv6")

1
examples/async_server.py
View File

@@ -116,6 +116,7 @@ async def main():
server = await asyncio.start_server( server = await asyncio.start_server(
new_connection, new_connection,
addr, port, addr, port,
family=socket.AF_INET
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

1
examples/async_ssl_client.py
View File

@@ -65,6 +65,7 @@ async def main(host):
reader, writer = await asyncio.open_connection( reader, writer = await asyncio.open_connection(
addr, port, addr, port,
ssl=ctx, ssl=ctx,
family=socket.AF_INET
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

1
examples/async_ssl_server.py
View File

@@ -128,6 +128,7 @@ async def main():
new_connection, new_connection,
addr, port, addr, port,
ssl=ctx, ssl=ctx,
family=socket.AF_INET,
) )
except Exception: except Exception:
print("Try IPv6") print("Try IPv6")

4
examples/thread_client.py
View File

@@ -30,7 +30,7 @@ class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server):
def start_status_thread(host): def start_status_thread(host):
client = capnp.TwoPartyClient(host) client = capnp.TwoPartyClient(host, nesting_limit=64)
cap = client.bootstrap().cast_as(thread_capnp.Example) cap = client.bootstrap().cast_as(thread_capnp.Example)
subscriber = StatusSubscriber() subscriber = StatusSubscriber()
@@ -39,7 +39,7 @@ def start_status_thread(host):
def main(host): def main(host):
client = capnp.TwoPartyClient(host) client = capnp.TwoPartyClient(host, nesting_limit=64)
cap = client.bootstrap().cast_as(thread_capnp.Example) cap = client.bootstrap().cast_as(thread_capnp.Example)
status_thread = threading.Thread(target=start_status_thread, args=(host,)) status_thread = threading.Thread(target=start_status_thread, args=(host,))

1
requirements.txt
View File

@@ -1,6 +1,7 @@
jinja2 jinja2
cython cython
setuptools setuptools
pkgconfig
pytest pytest
tox tox
wheel wheel

42
setup.py
View File

@@ -5,18 +5,19 @@ pycapnp distutils setup.py
from __future__ import print_function from __future__ import print_function
import glob
import os import os
import shutil
import struct import struct
import sys import sys
import pkgconfig
from distutils.command.clean import clean as _clean from distutils.command.clean import clean as _clean
from distutils.errors import CompileError
from distutils.extension import Extension
from distutils.spawn import find_executable
from setuptools import setup, find_packages, Extension from setuptools import setup, find_packages, Extension
from buildutils import test_build, fetch_libcapnp, build_libcapnp, info from buildutils import fetch_libcapnp, build_libcapnp, info
_this_dir = os.path.dirname(__file__) _this_dir = os.path.dirname(__file__)
@@ -69,12 +70,20 @@ class clean(_clean):
''' '''
def run(self): def run(self):
_clean.run(self) _clean.run(self)
for x in [ 'capnp/lib/capnp.cpp', 'capnp/lib/capnp.h', 'capnp/version.py' ]: for x in [
os.path.join('capnp', 'lib', 'capnp.cpp'),
os.path.join('capnp', 'lib', 'capnp.h'),
os.path.join('capnp', 'version.py'),
'build',
'build32',
'build64',
'bundled'
] + glob.glob(os.path.join('capnp', '*.capnp')):
print('removing %s' % x) print('removing %s' % x)
try: try:
os.remove(x) os.remove(x)
except OSError: except OSError:
pass shutil.rmtree(x, ignore_errors=True)
# hack to parse commandline arguments # hack to parse commandline arguments
@@ -113,18 +122,20 @@ class build_libcapnp_ext(build_ext_c):
need_build = False need_build = False
else: else:
# Try to use capnp executable to find include and lib path # Try to use capnp executable to find include and lib path
capnp_executable = find_executable("capnp") capnp_executable = shutil.which("capnp")
if capnp_executable: if capnp_executable:
self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')] self.include_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'include')]
self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib{}'.format(8 * struct.calcsize("P")))] self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib{}'.format(8 * struct.calcsize("P")))]
self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')] self.library_dirs += [os.path.join(os.path.dirname(capnp_executable), '..', 'lib')]
# Try to autodetect presence of library. Requires compile/run # Look for capnproto using pkg-config (and minimum version)
# step so only works for host (non-cross) compliation
try: try:
test_build(include_dirs=self.include_dirs, library_dirs=self.library_dirs) if pkgconfig.installed('capnp', '>= 0.8.0'):
need_build = False need_build = False
except CompileError: else:
need_build = True
except EnvironmentError:
# pkg-config not available in path
need_build = True need_build = True
if need_build: if need_build:
@@ -157,6 +168,13 @@ class build_libcapnp_ext(build_ext_c):
self.library_dirs += [os.path.join(build_dir, 'lib{}'.format(8 * struct.calcsize("P")))] self.library_dirs += [os.path.join(build_dir, 'lib{}'.format(8 * struct.calcsize("P")))]
self.library_dirs += [os.path.join(build_dir, 'lib')] self.library_dirs += [os.path.join(build_dir, 'lib')]
# Copy .capnp files from source
src_glob = glob.glob(os.path.join(build_dir, 'include', 'capnp', '*.capnp'))
dst_dir = os.path.join(self.build_lib, "capnp")
for file in src_glob:
info("copying {} -> {}".format(file, dst_dir))
shutil.copy(file, dst_dir)
return build_ext_c.run(self) return build_ext_c.run(self)
extra_compile_args = ['--std=c++14'] extra_compile_args = ['--std=c++14']

138
test/test_examples.py
View File

@@ -1,75 +1,147 @@
import os import os
import pytest
import socket import socket
import subprocess import subprocess
import sys import sys
import time import time
examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples') examples_dir = os.path.join(os.path.dirname(__file__), '..', 'examples')
hostname = 'localhost'
processes = []
@pytest.fixture
def cleanup():
yield
for p in processes:
p.kill()
def run_subprocesses(address, server, client): def run_subprocesses(address, server, client):
cmd = [sys.executable, os.path.join(examples_dir, server), address] server_attempt = 0
server = subprocess.Popen(cmd) server_attempts = 2
retries = 30 done = False
addr, port = address.split(':') addr, port = address.split(':')
while True: while not done:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) assert server_attempt < server_attempts, "Failed {} server attempts".format(server_attempts)
result = sock.connect_ex((addr, int(port))) server_attempt += 1
if result == 0:
break
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
result = sock.connect_ex((addr, int(port)))
if result == 0:
break
# Give the server some small amount of time to start listening
time.sleep(0.1)
retries -= 1
if retries == 0:
assert False, "Timed out waiting for server to start"
cmd = [sys.executable, os.path.join(examples_dir, client), address]
client = subprocess.Popen(cmd)
ret = client.wait(timeout=30) # Start server
server.kill() cmd = [sys.executable, os.path.join(examples_dir, server), address]
assert ret == 0 serverp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
print("Server started (Attempt #{})".format(server_attempt))
processes.append(serverp)
retries = 300
# Loop until we have a socket connection to the server (with timeout)
while True:
try:
if 'unix' in address:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
result = sock.connect_ex(port)
if result == 0:
break
else:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((addr, int(port)))
if result == 0:
break
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
result = sock.connect_ex((addr, int(port)))
if result == 0:
break
except socket.gaierror as err:
print("gaierror: {}".format(err))
# Give the server some small amount of time to start listening
time.sleep(0.1)
retries -= 1
if retries == 0:
serverp.kill()
print("Timed out waiting for server to start")
break
if serverp.poll() is not None:
print("Server exited prematurely: {}".format(serverp.returncode))
break
# 3 tries per server try
client_attempt = 0
client_attempts = 3
while not done:
if client_attempt >= client_attempts:
print("Failed {} client attempts".format(client_attempts))
break
client_attempt += 1
# Start client
cmd = [sys.executable, os.path.join(examples_dir, client), address]
clientp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
print("Client started (Attempt #{})".format(client_attempt))
processes.append(clientp)
retries = 30 * 10
# Loop until the client is finished (with timeout)
while True:
if clientp.poll() == 0:
done = True
break
if clientp.poll() is not None:
print("Client exited prematurely: {}".format(clientp.returncode))
break
time.sleep(0.1)
retries -= 1
if retries == 0:
print("Timed out waiting for client to finish")
clientp.kill()
break
# Retrying with different address (ipv4)
if 'unix' not in addr:
addr = socket.gethostbyname(addr)
address = '{}:{}'.format(addr, port)
print("Forcing ipv4 -> {}".format(address))
serverp.kill()
serverp.kill()
def test_async_calculator_example(): def test_async_calculator_example(cleanup):
address = 'localhost:36432' address = '{}:36432'.format(hostname)
server = 'async_calculator_server.py' server = 'async_calculator_server.py'
client = 'async_calculator_client.py' client = 'async_calculator_client.py'
run_subprocesses(address, server, client) run_subprocesses(address, server, client)
def test_thread_example(): def test_thread_example(cleanup):
address = 'localhost:36433' address = '{}:36433'.format(hostname)
server = 'thread_server.py' server = 'thread_server.py'
client = 'thread_client.py' client = 'thread_client.py'
run_subprocesses(address, server, client) run_subprocesses(address, server, client)
def test_addressbook_example(): def test_addressbook_example(cleanup):
proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')]) proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, 'addressbook.py')])
ret = proc.wait() ret = proc.wait()
assert ret == 0 assert ret == 0
def test_async_example(): def test_async_example(cleanup):
address = 'localhost:36434' address = '{}:36434'.format(hostname)
server = 'async_server.py' server = 'async_server.py'
client = 'async_client.py' client = 'async_client.py'
run_subprocesses(address, server, client) run_subprocesses(address, server, client)
def test_ssl_async_example(): def test_ssl_async_example(cleanup):
address = 'localhost:36435' address = '{}:36435'.format(hostname)
server = 'async_ssl_server.py' server = 'async_ssl_server.py'
client = 'async_ssl_client.py' client = 'async_ssl_client.py'
run_subprocesses(address, server, client) run_subprocesses(address, server, client)
def test_ssl_reconnecting_async_example(): def test_ssl_reconnecting_async_example(cleanup):
address = 'localhost:36436' address = '{}:36436'.format(hostname)
server = 'async_ssl_server.py' server = 'async_ssl_server.py'
client = 'async_reconnecting_ssl_client.py' client = 'async_reconnecting_ssl_client.py'
run_subprocesses(address, server, client) run_subprocesses(address, server, client)

6
test/test_load.py
View File

@@ -111,3 +111,9 @@ def test_remove_import_hook():
with pytest.raises(ImportError): with pytest.raises(ImportError):
import addressbook_capnp # noqa: F401 import addressbook_capnp # noqa: F401
def test_bundled_import_hook():
# stream.capnp should be bundled, or provided by the system capnproto
capnp.add_import_hook()
import stream_capnp

59
test/test_rpc_calculator.py
View File

@@ -14,6 +14,17 @@ sys.path.append(examples_dir)
import calculator_client # noqa: E402 import calculator_client # noqa: E402
import calculator_server # noqa: E402 import calculator_server # noqa: E402
# Uses run_subprocesses function
import test_examples
processes = []
@pytest.fixture
def cleanup():
yield
for p in processes:
p.kill()
def test_calculator(): def test_calculator():
read, write = socket.socketpair() read, write = socket.socketpair()
@@ -22,53 +33,13 @@ def test_calculator():
calculator_client.main(read) calculator_client.main(read)
def run_subprocesses(address): def test_calculator_tcp(cleanup):
cmd = [sys.executable, os.path.join(examples_dir, 'calculator_server.py'), address]
server = subprocess.Popen(cmd)
retries = 30
if 'unix' in address:
addr = address.split(':')[1]
while True:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
result = sock.connect_ex(addr)
if result == 0:
break
# Give the server some small amount of time to start listening
time.sleep(0.1)
retries -= 1
if retries == 0:
assert False, "Timed out waiting for server to start"
else:
addr, port = address.split(':')
while True:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((addr, int(port)))
if result == 0:
break
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
result = sock.connect_ex((addr, int(port)))
if result == 0:
break
# Give the server some small amount of time to start listening
time.sleep(0.1)
retries -= 1
if retries == 0:
assert False, "Timed out waiting for server to start"
cmd = [sys.executable, os.path.join(examples_dir, 'calculator_client.py'), address]
client = subprocess.Popen(cmd)
ret = client.wait()
server.kill()
assert ret == 0
def test_calculator_tcp():
address = 'localhost:36431' address = 'localhost:36431'
run_subprocesses(address) test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py')
@pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows") @pytest.mark.skipif(os.name == 'nt', reason="socket.AF_UNIX not supported on Windows")
def test_calculator_unix(): def test_calculator_unix(cleanup):
path = '/tmp/pycapnp-test' path = '/tmp/pycapnp-test'
try: try:
os.unlink(path) os.unlink(path)
@@ -76,7 +47,7 @@ def test_calculator_unix():
pass pass
address = 'unix:' + path address = 'unix:' + path
run_subprocesses(address) test_examples.run_subprocesses(address, 'calculator_server.py', 'calculator_client.py')
def test_calculator_gc(): def test_calculator_gc():