Merge pull request #285 from madhavajay/madhava/python_310

Add Python 3.10 support

.github/workflows/manylinux2010.yml

@@ -9,7 +9,7 @@ jobs:
     container: ${{ matrix.container-image }}
     strategy:
       matrix:
-          python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39']
+          python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39', 'cp310-cp310']
           container-image: ['quay.io/pypa/manylinux2010_x86_64', 'quay.io/pypa/manylinux2010_i686']
 
     steps:
@@ -43,7 +43,7 @@ jobs:
       runs-on: ubuntu-latest
       strategy:
         matrix:
-          python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39']
+          python-version: ['cp37-cp37m', 'cp38-cp38', 'cp39-cp39', 'cp310-cp310']
         fail-fast: false
       env:
         py: /opt/python/${{ matrix.python-version }}/bin/python

.github/workflows/packagingtest.yml

@@ -12,7 +12,7 @@ jobs:
       matrix:
         # Some asyncio commands require 3.7+
         # It may be possible to use 3.6 and maybe 3.5; however, this will take some patching to get examples to work
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, "3.10"]
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:

benchmark/addressbook.capnp.orphan.py

@@ -35,13 +35,12 @@ def writeAddressBook():
 
 
 def printAddressBook(msg_bytes):
-    addressBook = addressbook.AddressBook.from_bytes(msg_bytes)
+    with addressbook.AddressBook.from_bytes(msg_bytes) as addressBook:
-
+        for person in addressBook.people:
-    for person in addressBook.people:
+            print(person.name, ":", person.email)
-        print(person.name, ":", person.email)
+            for phone in person.phones:
-        for phone in person.phones:
+                print(phone.type, ":", phone.number)
-            print(phone.type, ":", phone.number)
+            print()
-        print()
 
 
 if __name__ == "__main__":

benchmark/addressbook.capnp.py

@@ -40,12 +40,11 @@ def writeAddressBook():
 
 @profile
 def printAddressBook(msg_bytes):
-    addressBook = addressbook.AddressBook.from_bytes(msg_bytes)
+    with addressbook.AddressBook.from_bytes(msg_bytes) as addressBook:
-
+        for person in addressBook.people:
-    for person in addressBook.people:
+            person.name, person.email
-        person.name, person.email
+            for phone in person.phones:
-        for phone in person.phones:
+                phone.type, phone.number
-            phone.type, phone.number
 
 
 @profile

benchmark/eval_proto.py

@@ -4,8 +4,8 @@ from common import rand_int, rand_double, rand_bool, from_bytes_helper
 from random import choice
 import eval_pb2
 
-MAX_INT = 2 ** 31 - 1
+MAX_INT = 2**31 - 1
-MIN_INT = -(2 ** 31)
+MIN_INT = -(2**31)
 
 OPERATIONS = ["add", "subtract", "multiply", "divide", "modulus"]
 

benchmark/eval_pycapnp.py

@@ -5,8 +5,8 @@ import eval_capnp
 from common import rand_int, rand_double, rand_bool
 from random import choice
 
-MAX_INT = 2 ** 31 - 1
+MAX_INT = 2**31 - 1
-MIN_INT = -(2 ** 31)
+MIN_INT = -(2**31)
 
 OPERATIONS = ["add", "subtract", "multiply", "divide", "modulus"]
 

capnp/lib/capnp.pyx

@@ -22,6 +22,7 @@ from libc.string cimport memcpy
 import array
 import asyncio
 import collections as _collections
+import contextlib
 import enum as _enum
 import inspect as _inspect
 import os as _os
@@ -1098,7 +1099,8 @@ if getattr(_sys, 'subversion', [''])[0] == 'PyPy':
     from pickle_helper import _struct_reducer
 else:
     def _struct_reducer(schema_id, data):
-        return _global_schema_parser.modules_by_id[schema_id].from_bytes(data)
+        with _global_schema_parser.modules_by_id[schema_id].from_bytes(data) as msg:
+            return msg
 
 
 cdef class _DynamicStructReader:
@@ -3320,6 +3322,7 @@ class _StructModule(object):
         reader = _MultipleBytesPackedMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit)
         return reader
 
+    @contextlib.contextmanager
     def from_bytes(self, buf, traversal_limit_in_words=None, nesting_limit=None, builder=False):
         """Returns a Reader for the unpacked object in buf.
 
@@ -3340,13 +3343,18 @@ class _StructModule(object):
 
         :rtype: :class:`_DynamicStructReader` or :class:`_DynamicStructBuilder`
         """
-        if builder:
+        message = None
-            # message = _FlatMessageBuilder(buf)
+        try:
-            message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit)
+            if builder:
-            return message.get_root(self.schema).as_builder()
+                # message = _FlatMessageBuilder(buf)
-        else:
+                message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit)
-            message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit)
+                yield message.get_root(self.schema).as_builder()
-            return message.get_root(self.schema)
+            else:
+                message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit)
+                yield message.get_root(self.schema)
+        finally:
+            if message:
+                message.close()
 
     def from_segments(self, segments, traversal_limit_in_words=None, nesting_limit=None):
         """Returns a Reader for a list of segment bytes.
@@ -4088,15 +4096,22 @@ cdef class _AlignedBuffer:
 cdef class _BufferView:
     cdef Py_buffer view
     cdef char * buf
+    cdef int closed
 
     def __init__(self, other):
         cdef int ret = PyObject_GetBuffer(other, &self.view, PyBUF_SIMPLE)
         if ret < 0:
             raise ValueError("Invalid buffer passed to BufferView")
         self.buf = <char*>self.view.buf
+        self.closed = False
+
+    def close(self):
+        if not self.closed:
+            PyBuffer_Release(&self.view)
+            self.closed = True
 
     def __dealloc__(self):
-        PyBuffer_Release(&self.view)
+        self.close()
 
 
 @cython.internal
@@ -4133,6 +4148,7 @@ cdef class _FlatArrayMessageReaderAligned(_MessageReader):
 @cython.internal
 cdef class _FlatArrayMessageReader(_MessageReader):
     cdef object _object_to_pin
+    cdef _BufferView _buffer_view
 
     def __init__(self, buf, traversal_limit_in_words=None, nesting_limit=None):
         cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit)
@@ -4151,10 +4167,12 @@ cdef class _FlatArrayMessageReader(_MessageReader):
                 self._object_to_pin = aligned
             else:
                 self._object_to_pin = buf
+            self._buffer_view = None
         elif PyObject_CheckBuffer(buf):
             view = _BufferView(buf)
             ptr = view.buf
             self._object_to_pin = view
+            self._buffer_view = view
         else:
             raise TypeError('expected buffer-like object in FlatArrayMessageReader')
 
@@ -4162,7 +4180,12 @@ cdef class _FlatArrayMessageReader(_MessageReader):
             schema_cpp.WordArrayPtr(<schema_cpp.word*>ptr, sz//8),
             opts)
 
+    def close(self):
+        if self._buffer_view:
+            self._buffer_view.close()
+
     def __dealloc__(self):
+        self.close()
         del self.thisptr
 
 

capnp/lib/pickle_helper.py

@@ -3,4 +3,5 @@ import capnp
 
 def _struct_reducer(schema_id, data):
     'Hack to deal with pypy not allowing reduce functions to be "built-in" methods (ie. compiled from a .pyx)'
-    return capnp._global_schema_parser.modules_by_id[schema_id].from_bytes(data)
+    with capnp._global_schema_parser.modules_by_id[schema_id].from_bytes(data) as msg:
+        return msg

docs/conf.py

@@ -51,8 +51,8 @@ source_suffix = ".rst"
 master_doc = "index"
 
 # General information about the project.
-project = u"capnp"
+project = "capnp"
-copyright = u"2013-2019 (Jason Paryani), 2019-2020 (Jacob Alexander)"
+copyright = "2013-2019 (Jason Paryani), 2019-2020 (Jacob Alexander)"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -204,7 +204,7 @@ latex_elements = {}
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-    ("index", "capnp.tex", u"capnp Documentation", u"Author", "manual"),
+    ("index", "capnp.tex", "capnp Documentation", "Author", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -232,7 +232,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [("index", "capnp", u"capnp Documentation", [u"Author"], 1)]
+man_pages = [("index", "capnp", "capnp Documentation", ["Author"], 1)]
 
 # If true, show URL addresses after external links.
 # man_show_urls = False
@@ -247,8 +247,8 @@ texinfo_documents = [
     (
         "index",
         "capnp",
-        u"capnp Documentation",
+        "capnp Documentation",
-        u"Author",
+        "Author",
         "capnp",
         "One line description of project.",
         "Miscellaneous",
@@ -268,10 +268,10 @@ texinfo_documents = [
 # -- Options for Epub output ---------------------------------------------------
 
 # Bibliographic Dublin Core info.
-epub_title = u"capnp"
+epub_title = "capnp"
-epub_author = u"Author"
+epub_author = "Author"
-epub_publisher = u"Author"
+epub_publisher = "Author"
-epub_copyright = u"2013, Author"
+epub_copyright = "2013, Author"
 
 # The language of the text. It defaults to the language option
 # or en if the language is not set.

examples/async_server.py

@@ -20,13 +20,13 @@ class ExampleImpl(thread_capnp.Example.Server):
     def subscribeStatus(self, subscriber, **kwargs):
         return (
             capnp.getTimer()
-            .after_delay(10 ** 9)
+            .after_delay(10**9)
             .then(lambda: subscriber.status(True))
             .then(lambda _: self.subscribeStatus(subscriber))
         )
 
     def longRunning(self, **kwargs):
-        return capnp.getTimer().after_delay(1 * 10 ** 9)
+        return capnp.getTimer().after_delay(1 * 10**9)
 
 
 class Server:

examples/async_ssl_server.py

@@ -24,13 +24,13 @@ class ExampleImpl(thread_capnp.Example.Server):
     def subscribeStatus(self, subscriber, **kwargs):
         return (
             capnp.getTimer()
-            .after_delay(10 ** 9)
+            .after_delay(10**9)
             .then(lambda: subscriber.status(True))
             .then(lambda _: self.subscribeStatus(subscriber))
         )
 
     def longRunning(self, **kwargs):
-        return capnp.getTimer().after_delay(1 * 10 ** 9)
+        return capnp.getTimer().after_delay(1 * 10**9)
 
     def alive(self, **kwargs):
         return True

examples/thread_server.py

@@ -14,13 +14,13 @@ class ExampleImpl(thread_capnp.Example.Server):
     def subscribeStatus(self, subscriber, **kwargs):
         return (
             capnp.getTimer()
-            .after_delay(10 ** 9)
+            .after_delay(10**9)
             .then(lambda: subscriber.status(True))
             .then(lambda _: self.subscribeStatus(subscriber))
         )
 
     def longRunning(self, **kwargs):
-        return capnp.getTimer().after_delay(1 * 10 ** 9)
+        return capnp.getTimer().after_delay(1 * 10**9)
 
 
 def parse_args():

scripts/capnp_test_pycapnp.py

@@ -13,7 +13,8 @@ import test_capnp  # noqa: E402
 
 def decode(name):
     class_name = name[0].upper() + name[1:]
-    print(getattr(test_capnp, class_name).from_bytes(sys.stdin.read())._short_str())
+    with getattr(test_capnp, class_name).from_bytes(sys.stdin.read()) as msg:
+        print(msg._short_str())
 
 
 def encode(name):

test/test_serialization.py

@@ -46,8 +46,8 @@ def test_roundtrip_bytes(all_types):
     test_regression.init_all_types(msg)
     message_bytes = msg.to_bytes()
 
-    msg = all_types.TestAllTypes.from_bytes(message_bytes)
+    with all_types.TestAllTypes.from_bytes(message_bytes) as msg:
-    test_regression.check_all_types(msg)
+        test_regression.check_all_types(msg)
 
 
 @pytest.mark.skipif(
@@ -77,8 +77,8 @@ def test_roundtrip_bytes_mmap(all_types):
         f.seek(0)
         memory = mmap.mmap(f.fileno(), length)
 
-        msg = all_types.TestAllTypes.from_bytes(memory)
+        with all_types.TestAllTypes.from_bytes(memory) as msg:
-        test_regression.check_all_types(msg)
+            test_regression.check_all_types(msg)
 
 
 @pytest.mark.skipif(
@@ -91,15 +91,16 @@ def test_roundtrip_bytes_buffer(all_types):
     b = msg.to_bytes()
     v = memoryview(b)
     try:
-        msg = all_types.TestAllTypes.from_bytes(v)
+        with all_types.TestAllTypes.from_bytes(v) as msg:
-        test_regression.check_all_types(msg)
+            test_regression.check_all_types(msg)
     finally:
         v.release()
 
 
 def test_roundtrip_bytes_fail(all_types):
     with pytest.raises(TypeError):
-        all_types.TestAllTypes.from_bytes(42)
+        with all_types.TestAllTypes.from_bytes(42) as _:
+            pass
 
 
 @pytest.mark.skipif(
@@ -232,14 +233,16 @@ def test_from_bytes_traversal_limit(all_types):
     bld.init("structList", size)
     data = bld.to_bytes()
 
-    msg = all_types.TestAllTypes.from_bytes(data)
+    with all_types.TestAllTypes.from_bytes(data) as msg:
-    with pytest.raises(capnp.KjException):
+        with pytest.raises(capnp.KjException):
-        for i in range(0, size):
+            for i in range(0, size):
-            msg.structList[i].uInt8Field == 0
+                msg.structList[i].uInt8Field == 0
 
-    msg = all_types.TestAllTypes.from_bytes(data, traversal_limit_in_words=2 ** 62)
+    with all_types.TestAllTypes.from_bytes(
-    for i in range(0, size):
+        data, traversal_limit_in_words=2**62
-        assert msg.structList[i].uInt8Field == 0
+    ) as msg:
+        for i in range(0, size):
+            assert msg.structList[i].uInt8Field == 0
 
 
 def test_from_bytes_packed_traversal_limit(all_types):
@@ -254,7 +257,7 @@ def test_from_bytes_packed_traversal_limit(all_types):
             msg.structList[i].uInt8Field == 0
 
     msg = all_types.TestAllTypes.from_bytes_packed(
-        data, traversal_limit_in_words=2 ** 62
+        data, traversal_limit_in_words=2**62
     )
     for i in range(0, size):
         assert msg.structList[i].uInt8Field == 0

test/test_struct.py

@@ -150,9 +150,9 @@ def test_unicode_str(all_types):
     msg = all_types.TestAllTypes.new_message()
 
     if sys.version_info[0] == 2:
-        msg.textField = u"f\u00e6oo".encode("utf-8")
+        msg.textField = "f\u00e6oo".encode("utf-8")
 
-        assert msg.textField.decode("utf-8") == u"f\u00e6oo"
+        assert msg.textField.decode("utf-8") == "f\u00e6oo"
     else:
         msg.textField = "f\u00e6oo"