Optimize numeric array conversion and serialize directly into Python bytes

.github/workflows/wheels.yml

@@ -21,7 +21,7 @@ jobs:
         env:
           CIBW_BUILD: cp312-*
           CIBW_ARCHS: ${{ matrix.arch }}
-          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_REQUIRES: pytest numpy
           CIBW_TEST_COMMAND: python -m pytest {project}/test
           CMAKE_OSX_ARCHITECTURES: "${{ runner.os == 'macOS' && matrix.arch || '' }}"
       - uses: actions/upload-artifact@v6

MANIFEST.in

@@ -1,5 +1,5 @@
 include README.md LICENSE.md
 recursive-include vendor/capnproto *.h *.c++ *.txt *.md *.capnp
 recursive-include capnp *.py *.pyx *.pxd *.h *.cpp
-recursive-include test *.py *.capnp *.binary *.txt
+recursive-include test *.py *.capnp *.binary *.txt *.json
 exclude capnp/lib/capnp.cpp capnp/lib/capnp.h capnp/lib/capnp_api.h

README.md

@@ -1,4 +1,23 @@
-# pycapnp for openpilot
+# pycapnp for IQPilot
+
+IQ.Lvbs maintains this fork at https://git.konn3kt.com/IQ.Lvbs/pycapnp.
+It extends comma's minimal branch at `caee18aeef0a9a54ad8f98292a325a1675a43bec`.
+The upstream licenses and history are retained.
+
+IQPilot additions:
+
+- Native NumPy conversion for floating-point list readers and builders through
+  `__array__`, retaining inferred dtype, explicit casts and warnings, writable
+  independent output, and rejection of unsupported `copy=False` requests.
+- `to_float64_bytes()` for an owned native-endian numeric snapshot without
+  constructing intermediate Python scalar objects. Non-numeric values are rejected.
+- Direct serialization into the final Python bytes object, preserving segment
+  framing and byte identity while removing the intermediate flat-array copy.
+
+NumPy is optional at runtime and loaded lazily only when its array protocol is used.
+The validation suite additionally requires NumPy. Serialization reference hashes
+were generated by the unmodified comma commit above and cover one and multiple
+segments through megabyte payloads. The native helpers do not borrow message memory.
 
 A serialization-only fork of [pycapnp](https://github.com/capnproto/pycapnp).
 The `minimal` branch starts at upstream commit
@@ -39,7 +58,7 @@ CMake are required to build the vendored library.
 
 ```sh
 uv venv --python 3.12
-uv pip install cython setuptools wheel pytest build
+uv pip install cython setuptools wheel pytest numpy build
 .venv/bin/python setup.py build_ext --inplace
 .venv/bin/python -m pytest
 .venv/bin/python -m build

capnp/helpers/serialize.h

@@ -0,0 +1,34 @@
+// Copyright (c) 2026 IQ.Lvbs. All rights reserved.
+#pragma once
+
+#include <Python.h>
+#include <capnp/message.h>
+#include <capnp/serialize.h>
+#include <kj/debug.h>
+#include <cstring>
+
+inline PyObject* messageToPythonBytes(capnp::MessageBuilder& builder) {
+  auto segments = builder.getSegmentsForOutput();
+  auto words = capnp::computeSerializedSizeInWords(segments);
+  KJ_REQUIRE(words <= static_cast<size_t>(PY_SSIZE_T_MAX) / sizeof(capnp::word));
+  PyObject* result = PyBytes_FromStringAndSize(nullptr, words * sizeof(capnp::word));
+  if (result == nullptr) return nullptr;
+  char* data = PyBytes_AS_STRING(result);
+  auto write32 = [data](size_t index, uint32_t value) {
+    for (unsigned int byte = 0; byte < 4; ++byte) {
+      data[index * 4 + byte] = static_cast<char>(value >> (byte * 8));
+    }
+  };
+  write32(0, segments.size() - 1);
+  for (size_t index = 0; index < segments.size(); ++index) {
+    write32(index + 1, segments[index].size());
+  }
+  if (segments.size() % 2 == 0) write32(segments.size() + 1, 0);
+  char* output = data + (segments.size() / 2 + 1) * sizeof(capnp::word);
+  for (auto segment : segments) {
+    auto size = segment.size() * sizeof(capnp::word);
+    std::memcpy(output, segment.begin(), size);
+    output += size;
+  }
+  return result;
+}

capnp/lib/capnp.pyx

@@ -13,6 +13,7 @@ from capnp.helpers.helpers cimport init_capnp_api
 from builtins import memoryview as BuiltinsMemoryview
 from cpython cimport Py_buffer, PyObject_CheckBuffer
 from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_CONTIG_RO
+from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING
 from cpython.exc cimport PyErr_Clear
 from cython.operator cimport dereference as deref
 from libc.stdlib cimport malloc, free
@@ -28,6 +29,11 @@ import warnings as _warnings
 
 from types import ModuleType as _ModuleType
 
+cdef object _numpy = None
+
+cdef extern from "capnp/helpers/serialize.h":
+    object messageToPythonBytes(schema_cpp.MessageBuilder&) except +reraise_kj_exception
+
 _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR
 _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR
 _CAPNP_VERSION_MICRO = capnp.CAPNP_VERSION_MICRO
@@ -243,6 +249,49 @@ cdef class _DynamicListReader:
         ptr = self.thisptr[index]
         return to_python_reader(ptr, self._parent)
 
+    def to_float64_bytes(self):
+        cdef Py_ssize_t size = self.thisptr.size()
+        cdef bytes result = PyBytes_FromStringAndSize(NULL, size * sizeof(double))
+        cdef char* target = PyBytes_AS_STRING(result)
+        cdef Py_ssize_t index
+        cdef double value
+        cdef C_DynamicValue.Reader item
+        cdef int kind
+        if size:
+            item = self.thisptr[0]
+            kind = item.getType()
+            if kind != capnp.TYPE_FLOAT and kind != capnp.TYPE_INT and kind != capnp.TYPE_UINT:
+                raise TypeError("to_float64_bytes requires a numeric list")
+        for index in range(size):
+            item = self.thisptr[index]
+            value = item.asDouble()
+            memcpy(target + index * sizeof(double), &value, sizeof(double))
+        return result
+
+    def __array__(self, dtype=None, copy=None):
+        global _numpy
+        if _numpy is None:
+            _numpy = __import__("numpy")
+        np = _numpy
+        if copy is False:
+            raise ValueError("Cap'n Proto lists require a copy for NumPy conversion")
+        cdef Py_ssize_t size = self.thisptr.size()
+        cdef Py_ssize_t index
+        cdef C_DynamicValue.Reader item
+        cdef int kind
+        cdef double[::1] values64
+        if size:
+            item = self.thisptr[0]
+            kind = item.getType()
+            if kind != capnp.TYPE_FLOAT:
+                return np.array(list(self), dtype=dtype)
+        result = np.empty(size, dtype=np.float64)
+        values64 = result
+        for index in range(size):
+            item = self.thisptr[index]
+            values64[index] = item.asDouble()
+        return result if dtype is None else result.astype(dtype, copy=False)
+
     def __getitem__(self, int64_t index):
         cdef uint size = self.thisptr.size()
         if index >= size:
@@ -289,6 +338,9 @@ cdef class _DynamicListBuilder:
         ptr = self.thisptr[index]
         return to_python_builder(ptr, self._parent)
 
+    def __array__(self, dtype=None, copy=None):
+        return _DynamicListReader()._init(self.thisptr.asReader(), self._parent).__array__(dtype=dtype, copy=copy)
+
     def __getitem__(self, int64_t index):
         cdef uint size = self.thisptr.size()
         if index >= size:
@@ -888,17 +940,13 @@ cdef class _DynamicStructBuilder:
     cpdef to_bytes(_DynamicStructBuilder self):
         """Returns the struct's containing message as a Python bytes object in the unpacked binary format.
 
-        This is inefficient; it makes several copies.
-
         :rtype: bytes
 
         :Raises: :exc:`KjException` if this isn't the message's root struct.
         """
         self._check_write()
         cdef _MessageBuilder builder = self._parent
-        array = schema_cpp.messageToFlatArray(deref(builder.thisptr))
+        cdef bytes ret = messageToPythonBytes(deref(builder.thisptr))
-        cdef const char* ptr = <const char *>array.begin()
-        cdef bytes ret = ptr[:8*array.size()]
         self._is_written = True
         return ret
 

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = []
 
 [dependency-groups]
 dev = ["cython>=3.0", "setuptools", "wheel", "build", {include-group = "test"}, {include-group = "lint"}]
-test = ["pytest"]
+test = ["pytest", "numpy>=2"]
 lint = ["ruff==0.16.8"]
 
 [tool.pytest.ini_options]

setup.py

@@ -18,7 +18,7 @@ from pathlib import Path
 MAJOR = 2
 MINOR = 2
 MICRO = 4
-TAG = ""
+TAG = ".post1"
 VERSION = "%d.%d.%d%s" % (MAJOR, MINOR, MICRO, TAG)
 
 
@@ -155,8 +155,8 @@ setup(
     # (setup.py only supports 1 author...)
     author="Jacob Alexander",  # <- Current maintainer; Original author -> Jason Paryani
     author_email="haata@kiibohd.com",
-    url="https://github.com/commaai/pycapnp",
+    url="https://git.konn3kt.com/IQ.Lvbs/pycapnp",
-    download_url="https://github.com/commaai/pycapnp/archive/v%s.zip" % VERSION,
+    download_url="https://git.konn3kt.com/IQ.Lvbs/pycapnp/archive/v%s.zip" % VERSION,
     keywords=["capnp", "capnproto", "Cap'n Proto", "pycapnp"],
     classifiers=[
         "Development Status :: 5 - Production/Stable",

test/serialization-reference.json

@@ -0,0 +1,61 @@
+{
+  "source_commit": "caee18aeef0a9a54ad8f98292a325a1675a43bec",
+  "cases": [
+    {
+      "size": 0,
+      "text_size": 0,
+      "sha256": "5d1e945d489c17931e0bd82f6daba3f8657795d6abbe81bded5dd34e3c4bab6d",
+      "bytes": 256,
+      "segments": 1
+    },
+    {
+      "size": 1,
+      "text_size": 1,
+      "sha256": "c5f683aec8691d1049acf8913bfdea5d35a126ee4c2745814aebbeea070bde16",
+      "bytes": 264,
+      "segments": 1
+    },
+    {
+      "size": 7,
+      "text_size": 7,
+      "sha256": "18478024be5786d8d07e68355a5f81f66dbd1ee48d328e4e375d04ca2bda9682",
+      "bytes": 272,
+      "segments": 1
+    },
+    {
+      "size": 1024,
+      "text_size": 1024,
+      "sha256": "fc40c903202d5c6eee2ffa7bd3be3e651badbc8623dbebe3fbc7361010bed360",
+      "bytes": 3328,
+      "segments": 1
+    },
+    {
+      "size": 8192,
+      "text_size": 8192,
+      "sha256": "c4eefda6030b879873ff39dc6a0d17cdc8f059af8ba24869a8b8f3802928dd89",
+      "bytes": 24856,
+      "segments": 3
+    },
+    {
+      "size": 65536,
+      "text_size": 65536,
+      "sha256": "a846392964ce60df5e15ff8c50d400b9783ca2d8b07216f85e9dc24c0706f7fa",
+      "bytes": 196888,
+      "segments": 3
+    },
+    {
+      "size": 1048576,
+      "text_size": 1048576,
+      "sha256": "ef3d77a4e1f80f2cf839f73c04688410957d6948f284ae0fd46f29a679f973fe",
+      "bytes": 3146008,
+      "segments": 3
+    },
+    {
+      "size": 8192,
+      "text_size": 0,
+      "sha256": "bf2e44b6eaa518a054ab8a260fcb03e154b0e07866096fa7c222c018abb7baaf",
+      "bytes": 8464,
+      "segments": 2
+    }
+  ]
+}

test/test_bulk_numeric.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2026 IQ.Lvbs. All rights reserved.
+import gc
+
+import capnp
+import numpy as np
+import pytest
+
+
+@pytest.fixture(scope="module")
+def numeric_schema(tmp_path_factory):
+    path = tmp_path_factory.mktemp("bulk-schema") / "numeric.capnp"
+    path.write_text("""# Copyright (c) 2026 IQ.Lvbs. All rights reserved.
+@0xe7ab11f278d8b04a;
+struct Numbers {
+  f32 @0 :List(Float32);
+  f64 @1 :List(Float64);
+  i64 @2 :List(Int64);
+  u64 @3 :List(UInt64);
+  text @4 :List(Text);
+  nested @5 :List(List(Float32));
+}
+""")
+    return capnp.load(str(path)).Numbers
+
+
+@pytest.mark.parametrize(
+    ("field", "values"),
+    [
+        ("f32", []),
+        ("f32", [0.0, -0.0, 1.25, -2.5, float("inf"), -float("inf"), float("nan")]),
+        ("f64", [1e-300, 1e300, 1.23456789012345, -0.0]),
+        ("i64", [-(2**63), -1, 0, 2**63 - 1]),
+        ("u64", [0, 2**53 + 1, 2**64 - 1]),
+    ],
+)
+def test_bulk_values_and_lifetime(numeric_schema, field, values):
+    builder = numeric_schema.new_message(**{field: values})
+    encoded = builder.to_bytes()
+    with numeric_schema.from_bytes(encoded) as reader:
+        data = getattr(reader, field)
+        expected = np.array(data, dtype=np.float64)
+        buffer = data.to_float64_bytes()
+        actual = np.frombuffer(buffer, dtype=np.float64)
+        np.testing.assert_array_equal(actual, expected)
+        np.testing.assert_array_equal(np.signbit(actual), np.signbit(expected))
+        assert not actual.flags.writeable
+    del reader, data, encoded, builder, buffer
+    gc.collect()
+    np.testing.assert_array_equal(actual, expected)
+
+
+def test_bulk_rejects_text(numeric_schema):
+    builder = numeric_schema.new_message(text=["not numeric"])
+    with numeric_schema.from_bytes(builder.to_bytes()) as reader:
+        with pytest.raises(TypeError):
+            reader.text.to_float64_bytes()
+
+
+def test_bulk_is_an_owned_snapshot(numeric_schema):
+    builder = numeric_schema.new_message(f32=[1.0, 2.0])
+    reader = builder.as_reader()
+    array = np.frombuffer(reader.f32.to_float64_bytes(), dtype=np.float64)
+    builder.f32[0] = 9.0
+    np.testing.assert_array_equal(array, [1.0, 2.0])
+    np.testing.assert_array_equal(np.frombuffer(reader.f32.to_float64_bytes(), dtype=np.float64), [9.0, 2.0])
+
+
+@pytest.mark.parametrize(
+    ("field", "values"),
+    [
+        ("f32", []),
+        ("f32", [0.0, -0.0, 1.25, float("inf"), float("nan")]),
+        ("f64", [1e-300, 1e300, -0.0]),
+        ("i64", [-(2**63), 0, 2**63 - 1]),
+        ("u64", [0, 2**64 - 1]),
+        ("text", ["a", "bc"]),
+    ],
+)
+@pytest.mark.parametrize("dtype", [None, np.float64, np.complex128, "U32"])
+@pytest.mark.parametrize("reader", [False, True])
+def test_array_protocol(numeric_schema, field, values, dtype, reader):
+    builder = numeric_schema.new_message(**{field: values})
+    data = getattr(builder.as_reader() if reader else builder, field)
+    if field == "text" and dtype in (np.float64, np.complex128):
+        with pytest.raises(ValueError):
+            np.array(data, dtype=dtype)
+        return
+    expected = np.array(list(data), dtype=dtype)
+    actual = np.array(data, dtype=dtype)
+    np.testing.assert_array_equal(actual, expected)
+    assert actual.dtype == expected.dtype
+    assert actual.flags.writeable
+    assert actual.flags.owndata
+    with pytest.raises(ValueError):
+        np.asarray(data, copy=False)
+
+
+def test_float32_array_ownership(numeric_schema):
+    builder = numeric_schema.new_message(f32=[1.25, -0.0, float("nan")])
+    actual = np.asarray(builder.as_reader().f32, dtype=np.float32)
+    expected = np.asarray(list(builder.f32), dtype=np.float32)
+    np.testing.assert_array_equal(actual, expected)
+    np.testing.assert_array_equal(np.signbit(actual), np.signbit(expected))
+    builder.f32[0] = 99.0
+    del builder
+    gc.collect()
+    actual[2] = 7.0
+    assert actual[0] == 1.25
+    assert actual[2] == 7.0
+
+
+def test_array_cast_overflow(numeric_schema):
+    builder = numeric_schema.new_message(f64=[1e300])
+    with pytest.warns(RuntimeWarning, match="overflow"):
+        expected = np.array(list(builder.f64), dtype=np.float32)
+    with pytest.warns(RuntimeWarning, match="overflow"):
+        actual = np.array(builder.as_reader().f64, dtype=np.float32)
+    np.testing.assert_array_equal(actual, expected)
+
+
+def test_nested_arrays(numeric_schema):
+    builder = numeric_schema.new_message(nested=[[1.0, 2.0], [3.0, 4.0]])
+    for message in (builder, builder.as_reader()):
+        actual = np.array(message.nested)
+        expected = np.array([list(row) for row in message.nested])
+        np.testing.assert_array_equal(actual, expected)
+        assert actual.dtype == expected.dtype

test/test_serialization_identity.py

@@ -0,0 +1,26 @@
+# Copyright (c) 2026 IQ.Lvbs. All rights reserved.
+import hashlib
+import json
+import struct
+from pathlib import Path
+
+import capnp
+import pytest
+
+ROOT = Path(__file__).parent
+CASES = json.loads((ROOT / "serialization-reference.json").read_text())["cases"]
+
+
+@pytest.mark.parametrize("case", CASES, ids=lambda case: str(case["size"]))
+def test_serialization_identity(case):
+    schema = capnp.load(str(ROOT / "all_types.capnp")).TestAllTypes
+    size = case["size"]
+    text_size = case["text_size"]
+    message = schema.new_message(dataField=b"Z" * size, textField="iq" * text_size, float64List=[1.25, -0.0, 123.0])
+    data = message.to_bytes()
+    assert len(data) == case["bytes"]
+    assert struct.unpack_from("<I", data)[0] + 1 == case["segments"]
+    assert hashlib.sha256(data).hexdigest() == case["sha256"]
+    with schema.from_bytes(data) as reader:
+        assert reader.dataField == b"Z" * size
+        assert reader.textField == "iq" * text_size