From 05cc505ef48a271cad622238d08917906892c02b Mon Sep 17 00:00:00 2001 From: "teal@teallvbs.xyz" Date: Mon, 21 Sep 2026 22:48:21 -0500 Subject: [PATCH] Optimize numeric array conversion and serialize directly into Python bytes --- .github/workflows/wheels.yml | 2 +- MANIFEST.in | 2 +- README.md | 23 ++++- capnp/helpers/serialize.h | 34 ++++++++ capnp/lib/capnp.pyx | 58 +++++++++++-- pyproject.toml | 2 +- setup.py | 6 +- test/serialization-reference.json | 61 +++++++++++++ test/test_bulk_numeric.py | 127 ++++++++++++++++++++++++++++ test/test_serialization_identity.py | 26 ++++++ 10 files changed, 328 insertions(+), 13 deletions(-) create mode 100644 capnp/helpers/serialize.h create mode 100644 test/serialization-reference.json create mode 100644 test/test_bulk_numeric.py create mode 100644 test/test_serialization_identity.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index b8130cf..f1cfc6d 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -21,7 +21,7 @@ jobs: env: CIBW_BUILD: cp312-* CIBW_ARCHS: ${{ matrix.arch }} - CIBW_TEST_REQUIRES: pytest + CIBW_TEST_REQUIRES: pytest numpy CIBW_TEST_COMMAND: python -m pytest {project}/test CMAKE_OSX_ARCHITECTURES: "${{ runner.os == 'macOS' && matrix.arch || '' }}" - uses: actions/upload-artifact@v6 diff --git a/MANIFEST.in b/MANIFEST.in index ab27005..55d3d2e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.md LICENSE.md recursive-include vendor/capnproto *.h *.c++ *.txt *.md *.capnp recursive-include capnp *.py *.pyx *.pxd *.h *.cpp -recursive-include test *.py *.capnp *.binary *.txt +recursive-include test *.py *.capnp *.binary *.txt *.json exclude capnp/lib/capnp.cpp capnp/lib/capnp.h capnp/lib/capnp_api.h diff --git a/README.md b/README.md index e7108e5..59a0cdc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,23 @@ -# pycapnp for openpilot +# pycapnp for IQPilot + +IQ.Lvbs maintains this fork at https://git.konn3kt.com/IQ.Lvbs/pycapnp. +It extends comma's minimal branch at `caee18aeef0a9a54ad8f98292a325a1675a43bec`. +The upstream licenses and history are retained. + +IQPilot additions: + +- Native NumPy conversion for floating-point list readers and builders through + `__array__`, retaining inferred dtype, explicit casts and warnings, writable + independent output, and rejection of unsupported `copy=False` requests. +- `to_float64_bytes()` for an owned native-endian numeric snapshot without + constructing intermediate Python scalar objects. Non-numeric values are rejected. +- Direct serialization into the final Python bytes object, preserving segment + framing and byte identity while removing the intermediate flat-array copy. + +NumPy is optional at runtime and loaded lazily only when its array protocol is used. +The validation suite additionally requires NumPy. Serialization reference hashes +were generated by the unmodified comma commit above and cover one and multiple +segments through megabyte payloads. The native helpers do not borrow message memory. A serialization-only fork of [pycapnp](https://github.com/capnproto/pycapnp). The `minimal` branch starts at upstream commit @@ -39,7 +58,7 @@ CMake are required to build the vendored library. ```sh uv venv --python 3.12 -uv pip install cython setuptools wheel pytest build +uv pip install cython setuptools wheel pytest numpy build .venv/bin/python setup.py build_ext --inplace .venv/bin/python -m pytest .venv/bin/python -m build diff --git a/capnp/helpers/serialize.h b/capnp/helpers/serialize.h new file mode 100644 index 0000000..8c6a83c --- /dev/null +++ b/capnp/helpers/serialize.h @@ -0,0 +1,34 @@ +// Copyright (c) 2026 IQ.Lvbs. All rights reserved. +#pragma once + +#include +#include +#include +#include +#include + +inline PyObject* messageToPythonBytes(capnp::MessageBuilder& builder) { + auto segments = builder.getSegmentsForOutput(); + auto words = capnp::computeSerializedSizeInWords(segments); + KJ_REQUIRE(words <= static_cast(PY_SSIZE_T_MAX) / sizeof(capnp::word)); + PyObject* result = PyBytes_FromStringAndSize(nullptr, words * sizeof(capnp::word)); + if (result == nullptr) return nullptr; + char* data = PyBytes_AS_STRING(result); + auto write32 = [data](size_t index, uint32_t value) { + for (unsigned int byte = 0; byte < 4; ++byte) { + data[index * 4 + byte] = static_cast(value >> (byte * 8)); + } + }; + write32(0, segments.size() - 1); + for (size_t index = 0; index < segments.size(); ++index) { + write32(index + 1, segments[index].size()); + } + if (segments.size() % 2 == 0) write32(segments.size() + 1, 0); + char* output = data + (segments.size() / 2 + 1) * sizeof(capnp::word); + for (auto segment : segments) { + auto size = segment.size() * sizeof(capnp::word); + std::memcpy(output, segment.begin(), size); + output += size; + } + return result; +} diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index 75b91a9..cae27e8 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -13,6 +13,7 @@ from capnp.helpers.helpers cimport init_capnp_api from builtins import memoryview as BuiltinsMemoryview from cpython cimport Py_buffer, PyObject_CheckBuffer from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_CONTIG_RO +from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING from cpython.exc cimport PyErr_Clear from cython.operator cimport dereference as deref from libc.stdlib cimport malloc, free @@ -28,6 +29,11 @@ import warnings as _warnings from types import ModuleType as _ModuleType +cdef object _numpy = None + +cdef extern from "capnp/helpers/serialize.h": + object messageToPythonBytes(schema_cpp.MessageBuilder&) except +reraise_kj_exception + _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR _CAPNP_VERSION_MICRO = capnp.CAPNP_VERSION_MICRO @@ -243,6 +249,49 @@ cdef class _DynamicListReader: ptr = self.thisptr[index] return to_python_reader(ptr, self._parent) + def to_float64_bytes(self): + cdef Py_ssize_t size = self.thisptr.size() + cdef bytes result = PyBytes_FromStringAndSize(NULL, size * sizeof(double)) + cdef char* target = PyBytes_AS_STRING(result) + cdef Py_ssize_t index + cdef double value + cdef C_DynamicValue.Reader item + cdef int kind + if size: + item = self.thisptr[0] + kind = item.getType() + if kind != capnp.TYPE_FLOAT and kind != capnp.TYPE_INT and kind != capnp.TYPE_UINT: + raise TypeError("to_float64_bytes requires a numeric list") + for index in range(size): + item = self.thisptr[index] + value = item.asDouble() + memcpy(target + index * sizeof(double), &value, sizeof(double)) + return result + + def __array__(self, dtype=None, copy=None): + global _numpy + if _numpy is None: + _numpy = __import__("numpy") + np = _numpy + if copy is False: + raise ValueError("Cap'n Proto lists require a copy for NumPy conversion") + cdef Py_ssize_t size = self.thisptr.size() + cdef Py_ssize_t index + cdef C_DynamicValue.Reader item + cdef int kind + cdef double[::1] values64 + if size: + item = self.thisptr[0] + kind = item.getType() + if kind != capnp.TYPE_FLOAT: + return np.array(list(self), dtype=dtype) + result = np.empty(size, dtype=np.float64) + values64 = result + for index in range(size): + item = self.thisptr[index] + values64[index] = item.asDouble() + return result if dtype is None else result.astype(dtype, copy=False) + def __getitem__(self, int64_t index): cdef uint size = self.thisptr.size() if index >= size: @@ -289,6 +338,9 @@ cdef class _DynamicListBuilder: ptr = self.thisptr[index] return to_python_builder(ptr, self._parent) + def __array__(self, dtype=None, copy=None): + return _DynamicListReader()._init(self.thisptr.asReader(), self._parent).__array__(dtype=dtype, copy=copy) + def __getitem__(self, int64_t index): cdef uint size = self.thisptr.size() if index >= size: @@ -888,17 +940,13 @@ cdef class _DynamicStructBuilder: cpdef to_bytes(_DynamicStructBuilder self): """Returns the struct's containing message as a Python bytes object in the unpacked binary format. - This is inefficient; it makes several copies. - :rtype: bytes :Raises: :exc:`KjException` if this isn't the message's root struct. """ self._check_write() cdef _MessageBuilder builder = self._parent - array = schema_cpp.messageToFlatArray(deref(builder.thisptr)) - cdef const char* ptr = array.begin() - cdef bytes ret = ptr[:8*array.size()] + cdef bytes ret = messageToPythonBytes(deref(builder.thisptr)) self._is_written = True return ret diff --git a/pyproject.toml b/pyproject.toml index b8d5a23..0054e72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [] [dependency-groups] dev = ["cython>=3.0", "setuptools", "wheel", "build", {include-group = "test"}, {include-group = "lint"}] -test = ["pytest"] +test = ["pytest", "numpy>=2"] lint = ["ruff==0.16.8"] [tool.pytest.ini_options] diff --git a/setup.py b/setup.py index 5c5033e..55de2bd 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ from pathlib import Path MAJOR = 2 MINOR = 2 MICRO = 4 -TAG = "" +TAG = ".post1" VERSION = "%d.%d.%d%s" % (MAJOR, MINOR, MICRO, TAG) @@ -155,8 +155,8 @@ setup( # (setup.py only supports 1 author...) author="Jacob Alexander", # <- Current maintainer; Original author -> Jason Paryani author_email="haata@kiibohd.com", - url="https://github.com/commaai/pycapnp", - download_url="https://github.com/commaai/pycapnp/archive/v%s.zip" % VERSION, + url="https://git.konn3kt.com/IQ.Lvbs/pycapnp", + download_url="https://git.konn3kt.com/IQ.Lvbs/pycapnp/archive/v%s.zip" % VERSION, keywords=["capnp", "capnproto", "Cap'n Proto", "pycapnp"], classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/test/serialization-reference.json b/test/serialization-reference.json new file mode 100644 index 0000000..0ea2ade --- /dev/null +++ b/test/serialization-reference.json @@ -0,0 +1,61 @@ +{ + "source_commit": "caee18aeef0a9a54ad8f98292a325a1675a43bec", + "cases": [ + { + "size": 0, + "text_size": 0, + "sha256": "5d1e945d489c17931e0bd82f6daba3f8657795d6abbe81bded5dd34e3c4bab6d", + "bytes": 256, + "segments": 1 + }, + { + "size": 1, + "text_size": 1, + "sha256": "c5f683aec8691d1049acf8913bfdea5d35a126ee4c2745814aebbeea070bde16", + "bytes": 264, + "segments": 1 + }, + { + "size": 7, + "text_size": 7, + "sha256": "18478024be5786d8d07e68355a5f81f66dbd1ee48d328e4e375d04ca2bda9682", + "bytes": 272, + "segments": 1 + }, + { + "size": 1024, + "text_size": 1024, + "sha256": "fc40c903202d5c6eee2ffa7bd3be3e651badbc8623dbebe3fbc7361010bed360", + "bytes": 3328, + "segments": 1 + }, + { + "size": 8192, + "text_size": 8192, + "sha256": "c4eefda6030b879873ff39dc6a0d17cdc8f059af8ba24869a8b8f3802928dd89", + "bytes": 24856, + "segments": 3 + }, + { + "size": 65536, + "text_size": 65536, + "sha256": "a846392964ce60df5e15ff8c50d400b9783ca2d8b07216f85e9dc24c0706f7fa", + "bytes": 196888, + "segments": 3 + }, + { + "size": 1048576, + "text_size": 1048576, + "sha256": "ef3d77a4e1f80f2cf839f73c04688410957d6948f284ae0fd46f29a679f973fe", + "bytes": 3146008, + "segments": 3 + }, + { + "size": 8192, + "text_size": 0, + "sha256": "bf2e44b6eaa518a054ab8a260fcb03e154b0e07866096fa7c222c018abb7baaf", + "bytes": 8464, + "segments": 2 + } + ] +} diff --git a/test/test_bulk_numeric.py b/test/test_bulk_numeric.py new file mode 100644 index 0000000..c75c22a --- /dev/null +++ b/test/test_bulk_numeric.py @@ -0,0 +1,127 @@ +# Copyright (c) 2026 IQ.Lvbs. All rights reserved. +import gc + +import capnp +import numpy as np +import pytest + + +@pytest.fixture(scope="module") +def numeric_schema(tmp_path_factory): + path = tmp_path_factory.mktemp("bulk-schema") / "numeric.capnp" + path.write_text("""# Copyright (c) 2026 IQ.Lvbs. All rights reserved. +@0xe7ab11f278d8b04a; +struct Numbers { + f32 @0 :List(Float32); + f64 @1 :List(Float64); + i64 @2 :List(Int64); + u64 @3 :List(UInt64); + text @4 :List(Text); + nested @5 :List(List(Float32)); +} +""") + return capnp.load(str(path)).Numbers + + +@pytest.mark.parametrize( + ("field", "values"), + [ + ("f32", []), + ("f32", [0.0, -0.0, 1.25, -2.5, float("inf"), -float("inf"), float("nan")]), + ("f64", [1e-300, 1e300, 1.23456789012345, -0.0]), + ("i64", [-(2**63), -1, 0, 2**63 - 1]), + ("u64", [0, 2**53 + 1, 2**64 - 1]), + ], +) +def test_bulk_values_and_lifetime(numeric_schema, field, values): + builder = numeric_schema.new_message(**{field: values}) + encoded = builder.to_bytes() + with numeric_schema.from_bytes(encoded) as reader: + data = getattr(reader, field) + expected = np.array(data, dtype=np.float64) + buffer = data.to_float64_bytes() + actual = np.frombuffer(buffer, dtype=np.float64) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(np.signbit(actual), np.signbit(expected)) + assert not actual.flags.writeable + del reader, data, encoded, builder, buffer + gc.collect() + np.testing.assert_array_equal(actual, expected) + + +def test_bulk_rejects_text(numeric_schema): + builder = numeric_schema.new_message(text=["not numeric"]) + with numeric_schema.from_bytes(builder.to_bytes()) as reader: + with pytest.raises(TypeError): + reader.text.to_float64_bytes() + + +def test_bulk_is_an_owned_snapshot(numeric_schema): + builder = numeric_schema.new_message(f32=[1.0, 2.0]) + reader = builder.as_reader() + array = np.frombuffer(reader.f32.to_float64_bytes(), dtype=np.float64) + builder.f32[0] = 9.0 + np.testing.assert_array_equal(array, [1.0, 2.0]) + np.testing.assert_array_equal(np.frombuffer(reader.f32.to_float64_bytes(), dtype=np.float64), [9.0, 2.0]) + + +@pytest.mark.parametrize( + ("field", "values"), + [ + ("f32", []), + ("f32", [0.0, -0.0, 1.25, float("inf"), float("nan")]), + ("f64", [1e-300, 1e300, -0.0]), + ("i64", [-(2**63), 0, 2**63 - 1]), + ("u64", [0, 2**64 - 1]), + ("text", ["a", "bc"]), + ], +) +@pytest.mark.parametrize("dtype", [None, np.float64, np.complex128, "U32"]) +@pytest.mark.parametrize("reader", [False, True]) +def test_array_protocol(numeric_schema, field, values, dtype, reader): + builder = numeric_schema.new_message(**{field: values}) + data = getattr(builder.as_reader() if reader else builder, field) + if field == "text" and dtype in (np.float64, np.complex128): + with pytest.raises(ValueError): + np.array(data, dtype=dtype) + return + expected = np.array(list(data), dtype=dtype) + actual = np.array(data, dtype=dtype) + np.testing.assert_array_equal(actual, expected) + assert actual.dtype == expected.dtype + assert actual.flags.writeable + assert actual.flags.owndata + with pytest.raises(ValueError): + np.asarray(data, copy=False) + + +def test_float32_array_ownership(numeric_schema): + builder = numeric_schema.new_message(f32=[1.25, -0.0, float("nan")]) + actual = np.asarray(builder.as_reader().f32, dtype=np.float32) + expected = np.asarray(list(builder.f32), dtype=np.float32) + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal(np.signbit(actual), np.signbit(expected)) + builder.f32[0] = 99.0 + del builder + gc.collect() + actual[2] = 7.0 + assert actual[0] == 1.25 + assert actual[2] == 7.0 + + +def test_array_cast_overflow(numeric_schema): + builder = numeric_schema.new_message(f64=[1e300]) + with pytest.warns(RuntimeWarning, match="overflow"): + expected = np.array(list(builder.f64), dtype=np.float32) + with pytest.warns(RuntimeWarning, match="overflow"): + actual = np.array(builder.as_reader().f64, dtype=np.float32) + np.testing.assert_array_equal(actual, expected) + + +def test_nested_arrays(numeric_schema): + builder = numeric_schema.new_message(nested=[[1.0, 2.0], [3.0, 4.0]]) + for message in (builder, builder.as_reader()): + actual = np.array(message.nested) + expected = np.array([list(row) for row in message.nested]) + np.testing.assert_array_equal(actual, expected) + assert actual.dtype == expected.dtype diff --git a/test/test_serialization_identity.py b/test/test_serialization_identity.py new file mode 100644 index 0000000..3e1e91a --- /dev/null +++ b/test/test_serialization_identity.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 IQ.Lvbs. All rights reserved. +import hashlib +import json +import struct +from pathlib import Path + +import capnp +import pytest + +ROOT = Path(__file__).parent +CASES = json.loads((ROOT / "serialization-reference.json").read_text())["cases"] + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: str(case["size"])) +def test_serialization_identity(case): + schema = capnp.load(str(ROOT / "all_types.capnp")).TestAllTypes + size = case["size"] + text_size = case["text_size"] + message = schema.new_message(dataField=b"Z" * size, textField="iq" * text_size, float64List=[1.25, -0.0, 123.0]) + data = message.to_bytes() + assert len(data) == case["bytes"] + assert struct.unpack_from("