Merge branch 'release/v0.5.2'

.gitignore

@@ -32,6 +32,9 @@ nosetests.xml
 .project
 .pydevproject
 
+# IntelliJ 
+.idea/
+
 # Cpp files
 capnp/*.cpp
 

.travis.yml

@@ -4,22 +4,28 @@ python:
   - 2.6
   - 2.7
   - 3.3
+  - 3.4
   - pypy
 
 env:
   - BUILD_CAPNP=true
   - BUILD_CAPNP=
 
+# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason
+matrix:
+  exclude:
+    - python: pypy
+      env: BUILD_CAPNP=
+
 compiler: gcc
 
 before_install:
   - buildutils/setup_travis.sh
-  - pip install -U setuptools
+  - travis_retry pip install -U setuptools
-  - pip install cython
+  - travis_retry pip install cython==0.21.2
-  - pip install pytest
+  - travis_retry pip install pytest
 
 install:
-  - pip install --editable .
+  - travis_retry pip install --editable .
 
-# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason
+script: py.test test;
-script: if [[ $TRAVIS_PYTHON_VERSION != 'pypy' || $BUILD_CAPNP == 'true' ]]; then py.test test; fi

CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.5.2 (2015-02-20)
+- Add read\_multiple\_bytes/read\_multiple\_bytes\_packed methods
+- Added Python 3.4 to the travis build matrix
+- Bump version for bundled C++ libcapnp to v0.5.1
+
+
 ## v0.5.1 (2014-12-27)
 - Remove installation dependency on cython. We now have no dependencies since libcapnp will automatically build as well.
 

README.md

@@ -35,6 +35,22 @@ Note: for OSX, if using clang from Xcode 5, you will need to set `CFLAGS` like s
 
     CFLAGS='-stdlib=libc++' pip install pycapnp
 
+### Binary Packages
+
+In order to build binary packages from this source code, you must specify the `--disable-cython` option:
+
+Building a dumb binary distribution:
+
+    python setup.py bdist_dumb --disable-cython
+
+Building a Python wheel distributiion:
+
+    python setup.py bdist_wheel --disable-cython
+
+If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with:
+
+    python setup.py build --force-cython
+
 ## Python Versions
 
 Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported.

buildutils/bundle.py

@@ -35,7 +35,7 @@ pjoin = os.path.join
 # Constants
 #-----------------------------------------------------------------------------
 
-bundled_version = (0,5,0)
+bundled_version = (0,5,1)
 libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version)
 libcapnp_url = "https://capnproto.org/" + libcapnp
 

buildutils/setup_travis.sh

@@ -9,5 +9,5 @@ sudo update-alternatives --quiet --install /usr/bin/gcc  gcc  /usr/bin/gcc-4.8
 sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8
 
 if ! [ -z "${BUILD_CAPNP}" ]; then
-  wget https://capnproto.org/capnproto-c++-0.5.0.tar.gz && tar xzvf capnproto-c++-0.5.0.tar.gz && cd capnproto-c++-0.5.0 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd ..
+  wget https://capnproto.org/capnproto-c++-0.5.1.tar.gz && tar xzvf capnproto-c++-0.5.1.tar.gz && cd capnproto-c++-0.5.1 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd ..
 fi

capnp/lib/capnp.pyx

@@ -2853,6 +2853,36 @@ class _StructModule(object):
         :rtype: Iterable with elements of :class:`_DynamicStructReader`"""
         reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit)
         return reader
+    def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns an iterable, that when traversed will return Readers for messages.
+
+        :type buf: buffer
+        :param buf: Any Python object that supports the buffer interface.
+
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: Iterable with elements of :class:`_DynamicStructReader`"""
+        reader = _MultipleBytesMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit)
+        return reader
+    def read_multiple_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None):
+        """Returns an iterable, that when traversed will return Readers for messages.
+
+        :type buf: buffer
+        :param buf: Any Python object that supports the buffer interface.
+
+        :type traversal_limit_in_words: int
+        :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024.
+
+        :type nesting_limit: int
+        :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64.
+
+        :rtype: Iterable with elements of :class:`_DynamicStructReader`"""
+        reader = _MultipleBytesPackedMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit)
+        return reader
     def from_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None, builder=False):
         """Returns a Reader for the unpacked object in buf.
 
@@ -3197,6 +3227,7 @@ cdef class _MessageReader:
 
     .. warning:: Don't ever instantiate this class. It is only used for inheritance.
     """
+    cdef public object _parent
     cdef schema_cpp.MessageReader * thisptr
     def __dealloc__(self):
         del self.thisptr
@@ -3271,7 +3302,6 @@ cdef class _PackedMessageReader(_MessageReader):
 
     :Parameters: - fd (`int`) - A file descriptor
     """
-    cdef public object _parent
     def __init__(self):
         pass
 
@@ -3289,7 +3319,6 @@ cdef class _PackedMessageReader(_MessageReader):
         return self
 
 cdef class _PackedMessageReaderBytes(_MessageReader):
-    cdef public object _parent
     cdef schema_cpp.ArrayInputStream * stream
 
     def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None):
@@ -3325,7 +3354,6 @@ cdef class _InputMessageReader(_MessageReader):
 
     :Parameters: - fd (`int`) - A file descriptor
     """
-    cdef public object _parent
     def __init__(self):
         pass
 
@@ -3426,6 +3454,78 @@ cdef class _MultiplePackedMessageReader:
     def __iter__(self):
         return self
 
+cdef class _MultipleBytesMessageReader:
+    cdef schema_cpp.ArrayInputStream * stream
+    cdef schema_cpp.BufferedInputStream * buffered_stream
+
+    cdef public object traversal_limit_in_words, nesting_limit, schema, buf
+
+    def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None):
+        self.schema = schema
+        self.traversal_limit_in_words = traversal_limit_in_words
+        self.nesting_limit = nesting_limit
+
+        cdef const void *ptr
+        cdef Py_ssize_t sz
+        PyObject_AsReadBuffer(buf, &ptr, &sz)
+
+        self.buf = buf
+        self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(<byte *>ptr, sz))
+        self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream))
+
+    def __dealloc__(self):
+        del self.buffered_stream
+        del self.stream
+
+    def __next__(self):
+        try:
+            reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self)
+            return reader.get_root(self.schema)
+        except KjException as e:
+            if 'EOF' in str(e):
+                raise StopIteration
+            else:
+                raise
+
+    def __iter__(self):
+        return self
+
+cdef class _MultipleBytesPackedMessageReader:
+    cdef schema_cpp.ArrayInputStream * stream
+    cdef schema_cpp.BufferedInputStream * buffered_stream
+
+    cdef public object traversal_limit_in_words, nesting_limit, schema, buf
+
+    def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None):
+        self.schema = schema
+        self.traversal_limit_in_words = traversal_limit_in_words
+        self.nesting_limit = nesting_limit
+
+        cdef const void *ptr
+        cdef Py_ssize_t sz
+        PyObject_AsReadBuffer(buf, &ptr, &sz)
+
+        self.buf = buf
+        self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(<byte *>ptr, sz))
+        self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream))
+
+    def __dealloc__(self):
+        del self.buffered_stream
+        del self.stream
+
+    def __next__(self):
+        try:
+            reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self)
+            return reader.get_root(self.schema)
+        except KjException as e:
+            if 'EOF' in str(e):
+                raise StopIteration
+            else:
+                raise
+
+    def __iter__(self):
+        return self
+
 @cython.internal
 cdef class _AlignedBuffer:
     cdef char * buf

requirements.txt

@@ -1,5 +1,5 @@
 jinja2 >= 2.7.3
-cython >= 0.21
+cython == 0.21.2
 setuptools >= 0.8
 pytest
 tox

setup.py

@@ -32,7 +32,7 @@ _this_dir = os.path.dirname(__file__)
 
 MAJOR = 0
 MINOR = 5
-MICRO = 1
+MICRO = 2
 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO)
 
 
@@ -85,6 +85,14 @@ if force_bundled_libcapnp:
 force_system_libcapnp = "--force-system-libcapnp" in sys.argv
 if force_system_libcapnp:
     sys.argv.remove("--force-system-libcapnp")
+disable_cython = "--disable-cython" in sys.argv
+if disable_cython:
+    sys.argv.remove("--disable-cython")
+    use_cython = False
+force_cython = "--force-cython" in sys.argv
+if force_cython:
+    sys.argv.remove("--force-cython")
+    use_cython = True
 
 
 class build_libcapnp_ext(build_ext_c):
@@ -151,7 +159,7 @@ setup(
     download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION,
     keywords = ['capnp', 'capnproto', "Cap'n Proto"],
     classifiers = [
-        'Development Status :: 3 - Alpha',
+        'Development Status :: 4 - Beta',
         'Intended Audience :: Developers',
         'License :: OSI Approved :: BSD License',
         'Operating System :: MacOS :: MacOS X',

test/test_serialization.py

@@ -57,8 +57,25 @@ def test_roundtrip_file_multiple(all_types):
     msg.write(f)
 
     f.seek(0)
+    i = 0
     for msg in all_types.TestAllTypes.read_multiple(f):
         test_regression.check_all_types(msg)
+        i += 1
+    assert i == 3
+
+def test_roundtrip_bytes_multiple(all_types):
+    msg = all_types.TestAllTypes.new_message()
+    test_regression.init_all_types(msg)
+
+    msgs = msg.to_bytes()
+    msgs += msg.to_bytes()
+    msgs += msg.to_bytes()
+
+    i = 0
+    for msg in all_types.TestAllTypes.read_multiple_bytes(msgs):
+        test_regression.check_all_types(msg)
+        i += 1
+    assert i == 3
 
 def test_roundtrip_file_multiple_packed(all_types):
     f = tempfile.TemporaryFile()
@@ -69,8 +86,25 @@ def test_roundtrip_file_multiple_packed(all_types):
     msg.write_packed(f)
 
     f.seek(0)
+    i = 0
     for msg in all_types.TestAllTypes.read_multiple_packed(f):
         test_regression.check_all_types(msg)
+        i += 1
+    assert i == 3
+
+def test_roundtrip_bytes_multiple_packed(all_types):
+    msg = all_types.TestAllTypes.new_message()
+    test_regression.init_all_types(msg)
+
+    msgs = msg.to_bytes_packed()
+    msgs += msg.to_bytes_packed()
+    msgs += msg.to_bytes_packed()
+
+    i = 0
+    for msg in all_types.TestAllTypes.read_multiple_bytes_packed(msgs):
+        test_regression.check_all_types(msg)
+        i += 1
+    assert i == 3
 
 def test_roundtrip_dict(all_types):
     msg = all_types.TestAllTypes.new_message()