diff --git a/.gitignore b/.gitignore index 34db373..ad81cce 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ nosetests.xml .project .pydevproject +# IntelliJ +.idea/ + # Cpp files capnp/*.cpp diff --git a/.travis.yml b/.travis.yml index a02f3b8..c847910 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,22 +4,28 @@ python: - 2.6 - 2.7 - 3.3 + - 3.4 - pypy env: - BUILD_CAPNP=true - BUILD_CAPNP= +# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason +matrix: + exclude: + - python: pypy + env: BUILD_CAPNP= + compiler: gcc before_install: - buildutils/setup_travis.sh - - pip install -U setuptools - - pip install cython - - pip install pytest + - travis_retry pip install -U setuptools + - travis_retry pip install cython==0.21.2 + - travis_retry pip install pytest install: - - pip install --editable . + - travis_retry pip install --editable . -# skip testing for pypy + BUILD_CAPNP=false since it's failing in travis for some reason -script: if [[ $TRAVIS_PYTHON_VERSION != 'pypy' || $BUILD_CAPNP == 'true' ]]; then py.test test; fi +script: py.test test; diff --git a/CHANGELOG.md b/CHANGELOG.md index ee45ec9..9464ead 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.5.2 (2015-02-20) +- Add read\_multiple\_bytes/read\_multiple\_bytes\_packed methods +- Added Python 3.4 to the travis build matrix +- Bump version for bundled C++ libcapnp to v0.5.1 + + ## v0.5.1 (2014-12-27) - Remove installation dependency on cython. We now have no dependencies since libcapnp will automatically build as well. diff --git a/README.md b/README.md index e671520..8cc915e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,22 @@ Note: for OSX, if using clang from Xcode 5, you will need to set `CFLAGS` like s CFLAGS='-stdlib=libc++' pip install pycapnp +### Binary Packages + +In order to build binary packages from this source code, you must specify the `--disable-cython` option: + +Building a dumb binary distribution: + + python setup.py bdist_dumb --disable-cython + +Building a Python wheel distributiion: + + python setup.py bdist_wheel --disable-cython + +If it fails with an error like `clang: error: no such file or directory: 'capnp/lib/capnp.cpp'`, then you need to cythonize fist. This can be done with: + + python setup.py build --force-cython + ## Python Versions Python 2.6/2.7 are supported as well as Python 3.2+. PyPy 2.1+ is also supported. diff --git a/buildutils/bundle.py b/buildutils/bundle.py index b2d645b..e7ba836 100644 --- a/buildutils/bundle.py +++ b/buildutils/bundle.py @@ -35,7 +35,7 @@ pjoin = os.path.join # Constants #----------------------------------------------------------------------------- -bundled_version = (0,5,0) +bundled_version = (0,5,1) libcapnp = "capnproto-c++-%i.%i.%i.tar.gz" % (bundled_version) libcapnp_url = "https://capnproto.org/" + libcapnp diff --git a/buildutils/setup_travis.sh b/buildutils/setup_travis.sh index 008f0c6..ad0f9ce 100755 --- a/buildutils/setup_travis.sh +++ b/buildutils/setup_travis.sh @@ -9,5 +9,5 @@ sudo update-alternatives --quiet --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 sudo update-alternatives --quiet --set gcc /usr/bin/gcc-4.8 if ! [ -z "${BUILD_CAPNP}" ]; then - wget https://capnproto.org/capnproto-c++-0.5.0.tar.gz && tar xzvf capnproto-c++-0.5.0.tar.gz && cd capnproto-c++-0.5.0 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. + wget https://capnproto.org/capnproto-c++-0.5.1.tar.gz && tar xzvf capnproto-c++-0.5.1.tar.gz && cd capnproto-c++-0.5.1 && ./configure && make -j6 check && sudo make install && sudo ldconfig && cd .. fi diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 2f76d3f..33f191a 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -2853,6 +2853,36 @@ class _StructModule(object): :rtype: Iterable with elements of :class:`_DynamicStructReader`""" reader = _MultiplePackedMessageReader(file.fileno(), self.schema, traversal_limit_in_words, nesting_limit) return reader + def read_multiple_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None): + """Returns an iterable, that when traversed will return Readers for messages. + + :type buf: buffer + :param buf: Any Python object that supports the buffer interface. + + :type traversal_limit_in_words: int + :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. + + :type nesting_limit: int + :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" + reader = _MultipleBytesMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) + return reader + def read_multiple_bytes_packed(self, buf, traversal_limit_in_words = None, nesting_limit = None): + """Returns an iterable, that when traversed will return Readers for messages. + + :type buf: buffer + :param buf: Any Python object that supports the buffer interface. + + :type traversal_limit_in_words: int + :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. + + :type nesting_limit: int + :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. + + :rtype: Iterable with elements of :class:`_DynamicStructReader`""" + reader = _MultipleBytesPackedMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) + return reader def from_bytes(self, buf, traversal_limit_in_words = None, nesting_limit = None, builder=False): """Returns a Reader for the unpacked object in buf. @@ -3197,6 +3227,7 @@ cdef class _MessageReader: .. warning:: Don't ever instantiate this class. It is only used for inheritance. """ + cdef public object _parent cdef schema_cpp.MessageReader * thisptr def __dealloc__(self): del self.thisptr @@ -3271,7 +3302,6 @@ cdef class _PackedMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ - cdef public object _parent def __init__(self): pass @@ -3289,7 +3319,6 @@ cdef class _PackedMessageReader(_MessageReader): return self cdef class _PackedMessageReaderBytes(_MessageReader): - cdef public object _parent cdef schema_cpp.ArrayInputStream * stream def __init__(self, buf, traversal_limit_in_words = None, nesting_limit = None): @@ -3325,7 +3354,6 @@ cdef class _InputMessageReader(_MessageReader): :Parameters: - fd (`int`) - A file descriptor """ - cdef public object _parent def __init__(self): pass @@ -3426,6 +3454,78 @@ cdef class _MultiplePackedMessageReader: def __iter__(self): return self +cdef class _MultipleBytesMessageReader: + cdef schema_cpp.ArrayInputStream * stream + cdef schema_cpp.BufferedInputStream * buffered_stream + + cdef public object traversal_limit_in_words, nesting_limit, schema, buf + + def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.schema = schema + self.traversal_limit_in_words = traversal_limit_in_words + self.nesting_limit = nesting_limit + + cdef const void *ptr + cdef Py_ssize_t sz + PyObject_AsReadBuffer(buf, &ptr, &sz) + + self.buf = buf + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) + + def __dealloc__(self): + del self.buffered_stream + del self.stream + + def __next__(self): + try: + reader = _InputMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + return reader.get_root(self.schema) + except KjException as e: + if 'EOF' in str(e): + raise StopIteration + else: + raise + + def __iter__(self): + return self + +cdef class _MultipleBytesPackedMessageReader: + cdef schema_cpp.ArrayInputStream * stream + cdef schema_cpp.BufferedInputStream * buffered_stream + + cdef public object traversal_limit_in_words, nesting_limit, schema, buf + + def __init__(self, buf, schema, traversal_limit_in_words = None, nesting_limit = None): + self.schema = schema + self.traversal_limit_in_words = traversal_limit_in_words + self.nesting_limit = nesting_limit + + cdef const void *ptr + cdef Py_ssize_t sz + PyObject_AsReadBuffer(buf, &ptr, &sz) + + self.buf = buf + self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(ptr, sz)) + self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) + + def __dealloc__(self): + del self.buffered_stream + del self.stream + + def __next__(self): + try: + reader = _PackedMessageReader()._init(deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) + return reader.get_root(self.schema) + except KjException as e: + if 'EOF' in str(e): + raise StopIteration + else: + raise + + def __iter__(self): + return self + @cython.internal cdef class _AlignedBuffer: cdef char * buf diff --git a/requirements.txt b/requirements.txt index 617ceb5..8712fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ jinja2 >= 2.7.3 -cython >= 0.21 +cython == 0.21.2 setuptools >= 0.8 pytest tox diff --git a/setup.py b/setup.py index ac46c13..51a9fd3 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ _this_dir = os.path.dirname(__file__) MAJOR = 0 MINOR = 5 -MICRO = 1 +MICRO = 2 VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) @@ -85,6 +85,14 @@ if force_bundled_libcapnp: force_system_libcapnp = "--force-system-libcapnp" in sys.argv if force_system_libcapnp: sys.argv.remove("--force-system-libcapnp") +disable_cython = "--disable-cython" in sys.argv +if disable_cython: + sys.argv.remove("--disable-cython") + use_cython = False +force_cython = "--force-cython" in sys.argv +if force_cython: + sys.argv.remove("--force-cython") + use_cython = True class build_libcapnp_ext(build_ext_c): @@ -151,7 +159,7 @@ setup( download_url = 'https://github.com/jparyani/pycapnp/archive/v%s.zip' % VERSION, keywords = ['capnp', 'capnproto', "Cap'n Proto"], classifiers = [ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', diff --git a/test/test_serialization.py b/test/test_serialization.py index 3928c31..d6a9bb3 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -57,8 +57,25 @@ def test_roundtrip_file_multiple(all_types): msg.write(f) f.seek(0) + i = 0 for msg in all_types.TestAllTypes.read_multiple(f): test_regression.check_all_types(msg) + i += 1 + assert i == 3 + +def test_roundtrip_bytes_multiple(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + msgs = msg.to_bytes() + msgs += msg.to_bytes() + msgs += msg.to_bytes() + + i = 0 + for msg in all_types.TestAllTypes.read_multiple_bytes(msgs): + test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_file_multiple_packed(all_types): f = tempfile.TemporaryFile() @@ -69,8 +86,25 @@ def test_roundtrip_file_multiple_packed(all_types): msg.write_packed(f) f.seek(0) + i = 0 for msg in all_types.TestAllTypes.read_multiple_packed(f): test_regression.check_all_types(msg) + i += 1 + assert i == 3 + +def test_roundtrip_bytes_multiple_packed(all_types): + msg = all_types.TestAllTypes.new_message() + test_regression.init_all_types(msg) + + msgs = msg.to_bytes_packed() + msgs += msg.to_bytes_packed() + msgs += msg.to_bytes_packed() + + i = 0 + for msg in all_types.TestAllTypes.read_multiple_bytes_packed(msgs): + test_regression.check_all_types(msg) + i += 1 + assert i == 3 def test_roundtrip_dict(all_types): msg = all_types.TestAllTypes.new_message()