From 88db4abc48311aa77c9a7c74d10b9d81d442061e Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 28 Jan 2015 16:08:47 -0800 Subject: [PATCH 01/16] Add read_multiple_bytes/read_multiple_bytes_packed methods --- capnp/lib/capnp.pyx | 102 +++++++++++++++++++++++++++++++++++++ test/test_serialization.py | 22 ++++++++ 2 files changed, 124 insertions(+) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 2f76d3f..b3baf89 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2853,6 +2853,36 @@ class _StructModule(object): :rtype: Iterable with elements of :class:`_DynamicStructReader`""" reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) return reader + def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None): + """Returns an iterable, that when traversed will return Readers for messages. + + :type buf: buffer + :param buf: Any Python object that supports the buffer interface. + + :type traversal_limit_in_words: int + :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. + + :type nesting_limit: int + :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" + reader = _MultipleBytesMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) + return reader + def read_multiple_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): + """Returns an iterable, that when traversed will return Readers for messages. + + :type buf: buffer + :param buf: Any Python object that supports the buffer interface. + + :type traversal_limit_in_words: int + :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. + + :type nesting_limit: int + :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" + reader = _MultipleBytesPackedMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) + return reader def from_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None, builder=False): """Returns a Reader for the unpacked object in buf. @@ -3426,6 +3456,78 @@ cdef class _MultiplePackedMessageReader: def __iter__(self): return self +cdef class _MultipleBytesMessageReader: + cdef schema_cpp.ArrayInputStream * stream + cdef schema_cpp.BufferedInputStream * buffered_stream + + cdef public object traversal_limit_in_words, nesting_limit, schema, buf + + def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.schema = schema + self.traversal_limit_in_words = traversal_limit_in_words + self.nesting_limit = nesting_limit + + cdef const void *ptr + cdef Py_ssize_t sz + PyObject_AsReadBuffer(buf, &ptr, &sz) + + self.buf = buf + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) + + def __dealloc__(self): + del self.stream + del self.buffered_stream + + def __next__(self): + try: + reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + return reader.get_root(self.schema) + except KjException as e: + if 'EOF' in str(e): + raise StopIteration + else: + raise + + def __iter__(self): + return self + +cdef class _MultipleBytesPackedMessageReader: + cdef schema_cpp.ArrayInputStream * stream + cdef schema_cpp.BufferedInputStream * buffered_stream + + cdef public object traversal_limit_in_words, nesting_limit, schema, buf + + def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.schema = schema + self.traversal_limit_in_words = traversal_limit_in_words + self.nesting_limit = nesting_limit + + cdef const void *ptr + cdef Py_ssize_t sz + PyObject_AsReadBuffer(buf, &ptr, &sz) + + self.buf = buf + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) + + def __dealloc__(self): + del self.stream + del self.buffered_stream + + def __next__(self): + try: + reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + return reader.get_root(self.schema) + except KjException as e: + if 'EOF' in str(e): + raise StopIteration + else: + raise + + def __iter__(self): + return self + @cython.internal cdef class _AlignedBuffer: cdef char * buf diff --git a/test/test_serialization.py b/test/test_serialization.py index 3928c31..d1bfc66 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -60,6 +60,17 @@ def test_roundtrip_file_multiple(all_types): for msg in all_types.TestAllTypes.read_multiple(f): test_regression.check_all_types(msg) +def test_roundtrip_bytes_multiple(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + msgs = msg.to_bytes() + msgs += msg.to_bytes() + msgs += msg.to_bytes() + + for msg in all_types.TestAllTypes.read_multiple_bytes(msgs): + test_regression.check_all_types(msg) + def test_roundtrip_file_multiple_packed(all_types): f = tempfile.TemporaryFile() msg = all_types.TestAllTypes.new_message() @@ -72,6 +83,17 @@ def test_roundtrip_file_multiple_packed(all_types): for msg in all_types.TestAllTypes.read_multiple_packed(f): test_regression.check_all_types(msg) +def test_roundtrip_bytes_multiple_packed(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + msgs = msg.to_bytes_packed() + msgs += msg.to_bytes_packed() + msgs += msg.to_bytes_packed() + + for msg in all_types.TestAllTypes.read_multiple_bytes_packed(msgs): + test_regression.check_all_types(msg) + def test_roundtrip_dict(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) From c6754f84d98e35d8f2a451680cea56e638089752 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 28 Jan 2015 16:13:13 -0800 Subject: [PATCH 02/16] Make test_roundtrip_*multiple tests more robust --- test/test_serialization.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_serialization.py b/test/test_serialization.py index d1bfc66..d6a9bb3 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -57,8 +57,11 @@ def test_roundtrip_file_multiple(all_types): msg.write(f) f.seek(0) + i = 0 for msg in all_types.TestAllTypes.read_multiple(f): test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_bytes_multiple(all_types): msg = all_types.TestAllTypes.new_message() @@ -68,8 +71,11 @@ def test_roundtrip_bytes_multiple(all_types): msgs += msg.to_bytes() msgs += msg.to_bytes() + i = 0 for msg in all_types.TestAllTypes.read_multiple_bytes(msgs): test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_file_multiple_packed(all_types): f = tempfile.TemporaryFile() @@ -80,8 +86,11 @@ def test_roundtrip_file_multiple_packed(all_types): msg.write_packed(f) f.seek(0) + i = 0 for msg in all_types.TestAllTypes.read_multiple_packed(f): test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_bytes_multiple_packed(all_types): msg = all_types.TestAllTypes.new_message() @@ -91,8 +100,11 @@ def test_roundtrip_bytes_multiple_packed(all_types): msgs += msg.to_bytes_packed() msgs += msg.to_bytes_packed() + i = 0 for msg in all_types.TestAllTypes.read_multiple_bytes_packed(msgs): test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_dict(all_types): msg = all_types.TestAllTypes.new_message() From a62e392febcd5eee56238bb529c65c868b27d943 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 30 Jan 2015 13:39:58 -0800 Subject: [PATCH 03/16] Fix subtle dealloc ordering problem with MessageReader It appears that Cython has a subtle issue in deconstruction ordering that I forgot about. Subclasses will have their deconstructors before their parent's, and in this case we were dropping a reference to a buffer that the parent deconstructor expected to exist. --- capnp/lib/capnp.pyx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index b3baf89..33f191a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -3227,6 +3227,7 @@ cdef class _MessageReader: .. warning:: Don't ever instantiate this class. It is only used for inheritance. """ + cdef public object _parent cdef schema_cpp.MessageReader * thisptr def __dealloc__(self): del self.thisptr @@ -3301,7 +3302,6 @@ cdef class _PackedMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ - cdef public object _parent def __init__(self): pass @@ -3319,7 +3319,6 @@ cdef class _PackedMessageReader(_MessageReader): return self cdef class _PackedMessageReaderBytes(_MessageReader): - cdef public object _parent cdef schema_cpp.ArrayInputStream * stream def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): @@ -3355,7 +3354,6 @@ cdef class _InputMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ - cdef public object _parent def __init__(self): pass @@ -3476,8 +3474,8 @@ cdef class _MultipleBytesMessageReader: self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): - del self.stream del self.buffered_stream + del self.stream def __next__(self): try: @@ -3512,8 +3510,8 @@ cdef class _MultipleBytesPackedMessageReader: self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) def __dealloc__(self): - del self.stream del self.buffered_stream + del self.stream def __next__(self): try: From 78cc7092ac564f65979996e718a68f2ab70ceb21 Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Sun, 1 Feb 2015 12:26:17 +0200 Subject: [PATCH 04/16] Added Python 3.4 to the build matrix. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a02f3b8..ef0cf6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - 2.6 - 2.7 - 3.3 + - 3.4 - pypy env: From 7fd2a391a538adff0e437bd33c5a72e25e2f9adb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Feb 2015 07:18:29 +0200 Subject: [PATCH 05/16] Used travis_retry in order to avoid build failures due to network errors. --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef0cf6e..4e6eb67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,12 +15,12 @@ compiler: gcc before_install: - buildutils/setup_travis.sh - - pip install -U setuptools - - pip install cython - - pip install pytest + - travis_retry pip install -U setuptools + - travis_retry pip install cython + - travis_retry pip install pytest install: - - pip install --editable . + - travis_retry pip install --editable . # skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason script: if [[ $TRAVIS_PYTHON_VERSION != 'pypy' || $BUILD_CAPNP == 'true' ]]; then py.test test; fi From 26610adbecf5682647a4fc90c85e9a624dcc4fcb Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Feb 2015 07:21:15 +0200 Subject: [PATCH 06/16] Bumped capnproto to 0.5.1. --- buildutils/setup_travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh index 008f0c6..ad0f9ce 100755 --- a/buildutils/setup_travis.sh +++ b/buildutils/setup_travis.sh @@ -9,5 +9,5 @@ sudo update-alternatives --quiet --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 if ! [ -z "${BUILD_CAPNP}" ]; then - wget https://capnproto.org/capnproto-c++-0.5.0.tar.gz && tar xzvf capnproto-c++-0.5.0.tar.gz && cd capnproto-c++-0.5.0 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. + wget https://capnproto.org/capnproto-c++-0.5.1.tar.gz && tar xzvf capnproto-c++-0.5.1.tar.gz && cd capnproto-c++-0.5.1 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. fi From cfd0188449e16601ba0d1ae99db69013a695d1df Mon Sep 17 00:00:00 2001 From: Omer Katz Date: Mon, 2 Feb 2015 07:26:17 +0200 Subject: [PATCH 07/16] Provided a cleaner way to exclude build jobs from the matrix. --- .travis.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ef0cf6e..79e861a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,12 @@ env: - BUILD_CAPNP=true - BUILD_CAPNP= +# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason +matrix: + exclude: + - python: pypy + env: BUILD_CAPNP= + compiler: gcc before_install: @@ -22,5 +28,4 @@ before_install: install: - pip install --editable . -# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason -script: if [[ $TRAVIS_PYTHON_VERSION != 'pypy' || $BUILD_CAPNP == 'true' ]]; then py.test test; fi +script: py.test test; From 09875ac4a515acd03657c83b3eb9e60b36b19b9b Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Sun, 1 Feb 2015 22:17:32 -0800 Subject: [PATCH 08/16] Bump version for bundled C++ libcapnp to v0.5.1 --- buildutils/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildutils/bundle.py b/buildutils/bundle.py index b2d645b..e7ba836 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,7 +35,7 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,5,0) +bundled_version = (0,5,1) libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp From 441090300f3454461a56a2ef78e65522a4683232 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 3 Feb 2015 08:30:20 -0800 Subject: [PATCH 09/16] Ignoring IntelliJ folder. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 34db373..ad81cce 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ nosetests.xml .project .pydevproject +# IntelliJ +.idea/ + # Cpp files capnp/*.cpp From b3575cb1b47a205cc450ac5571b3be3c55291da5 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 3 Feb 2015 08:35:04 -0800 Subject: [PATCH 10/16] Added --force-cython & --disable-cython options. Fixes #50. --- setup.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup.py b/setup.py index ac46c13..84f2eb8 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,14 @@ if force_bundled_libcapnp: force_system_libcapnp = "--force-system-libcapnp" in sys.argv if force_system_libcapnp: sys.argv.remove("--force-system-libcapnp") +disable_cython = "--disable-cython" in sys.argv +if disable_cython: + sys.argv.remove("--disable-cython") + use_cython = False +force_cython = "--force-cython" in sys.argv +if force_cython: + sys.argv.remove("--force-cython") + use_cython = True class build_libcapnp_ext(build_ext_c): From dc88de9d9d5fde891a1d7c4f515ac38751e503c2 Mon Sep 17 00:00:00 2001 From: Matthew Taylor Date: Tue, 3 Feb 2015 08:37:54 -0800 Subject: [PATCH 11/16] Binary package creation instructions --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index e671520..125d7a0 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,18 @@ Note: for OSX, if using clang from Xcode 5, you will need to set `CFLAGS` like s CFLAGS='-stdlib=libc++' pip install pycapnp +### Binary Packages + +In order to build binary packages from this source code, you must specify the `--disable-cython` option: + +Building a dumb binary distribution: + + python setup.py bdist_dumb --disable-cython + +Building a Python wheel distributiion: + + python setup.py bdist_wheel --disable-cython + ## Python Versions Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. From 0aebde18d5068b91c2f485377e184263f77f4dcb Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 20 Feb 2015 17:20:04 -0800 Subject: [PATCH 12/16] Update pycapnp to beta status in trove classifier --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84f2eb8..abbfc38 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ setup( download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, keywords = ['capnp', 'capnproto', "Cap'n Proto"], classifiers = [ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', From 5010db4ef4a0d72fe5b563dcb679f572b94b1516 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 20 Feb 2015 17:20:35 -0800 Subject: [PATCH 13/16] Add note for binary build instructions --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 125d7a0..8cc915e 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ Building a Python wheel distributiion: python setup.py bdist_wheel --disable-cython +If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with: + + python setup.py build --force-cython + ## Python Versions Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. From fb363c47b56da19f93264af50c5529b6f29ecf27 Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 20 Feb 2015 17:21:00 -0800 Subject: [PATCH 14/16] Pin Cython to v0.21.2 until upstream bug is fixed --- .travis.yml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b6f68f..c847910 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ compiler: gcc before_install: - buildutils/setup_travis.sh - travis_retry pip install -U setuptools - - travis_retry pip install cython + - travis_retry pip install cython==0.21.2 - travis_retry pip install pytest install: diff --git a/requirements.txt b/requirements.txt index 617ceb5..8712fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 >= 2.7.3 -cython >= 0.21 +cython == 0.21.2 setuptools >= 0.8 pytest tox From fe768ba7c86f28095f3303c09d1e4d3b5cfca06a Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 20 Feb 2015 17:24:58 -0800 Subject: [PATCH 15/16] Bump version to v0.5.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index abbfc38..51a9fd3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 1 +MICRO = 2 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) From ff2823e1306674a48d08c034580d64e07103012a Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Fri, 20 Feb 2015 17:28:47 -0800 Subject: [PATCH 16/16] Update CHANGELOG for v0.5.2 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee45ec9..9464ead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.5.2 (2015-02-20) +- Add read\_multiple\_bytes/read\_multiple\_bytes\_packed methods +- Added Python 3.4 to the travis build matrix +- Bump version for bundled C++ libcapnp to v0.5.1 + + ## v0.5.1 (2014-12-27) - Remove installation dependency on cython. We now have no dependencies since libcapnp will automatically build as well.