Support python custom message builder and make Data field's type return MemoryView (#380)

This PR is for resolving the following issue:
[issue](https://github.com/capnproto/pycapnp/issues/379)

1. Created `_PyCustomMessageBuilder` extends `MessageBuilder`, enabling the ability to customise the `SegmentAllocate` method in Python. This allows allocation and data population within shared memory, and supports zero-copy inter-process data transfer by passing segment offsets.

2. Fields of type `Data` now support being set with a `memoryview`. When retrieving a `Data` field from a `DynamicStructBuilder`, it will return a writable `memoryview`, allowing users to modify the data directly. This enables memory to be pre-allocated and content to be modified in later, eliminating an extra copy. When retrieving a `Data` field from a `DynamicStructReader`, it will return a read-only `memoryview`, allowing user to read data without memory copy.

* add memoryview and custom builder

* support set dynamic field

* add curSize

* add initialSize and lastSize

* change StringPtr name

* add test case

* refine test case

* convert func to py callable object

* add initial value

* refine example

* add copy as_reader and new_message, make structReader's data field return RO memoryView

* rebase master and bugfix

* reformat flake8

* refine test case

* refine test cases for blob

* remove unused import for flake8

* run black .

---------

Co-authored-by: Brian Xu <brian.xu1@bytedance.com>
This commit is contained in:
Brian Xu
2025-09-10 23:48:58 +10:00
committed by GitHub
parent 62682977cb
commit 84674909a2
10 changed files with 304 additions and 17 deletions

View File

@@ -1,6 +1,5 @@
import os
import capnp
import base64
import pytest
this_dir = os.path.dirname(__file__)
@@ -15,7 +14,7 @@ def test_blob_to_dict(blob_schema):
blob_value = b"hello world"
blob = blob_schema.BlobTest(blob=blob_value)
blob_dict = blob.to_dict(encode_bytes_as_base64=True)
assert base64.b64decode(blob_dict["blob"]) == blob_value
assert blob_dict["blob"].tobytes() == blob_value
msg = blob_schema.BlobTest.new_message()
msg.from_dict(blob_dict)
assert blob.blob == blob_value

View File

@@ -0,0 +1,51 @@
#!/usr/bin/env python3
import pytest
import capnp # noqa: F401
import os
this_dir = os.path.dirname(__file__)
@pytest.fixture
def all_types():
return capnp.load(os.path.join(this_dir, "all_types.capnp"))
def test_addressbook(all_types):
class Allocator:
def __init__(self):
self.cur_size = 0
self.last_size = 0
def __call__(self, minimum_size: int) -> bytearray:
actual_size = max(minimum_size, self.cur_size)
print(
f"minimum_size: {minimum_size}, last_size: {self.last_size}, "
f"actual_size: {actual_size}, cur_size: {self.cur_size}"
)
self.last_size = actual_size
self.cur_size += actual_size
WORD_SIZE = 8
byte_count = actual_size * WORD_SIZE
return bytearray(byte_count)
allocator = Allocator()
assert allocator.cur_size == 0
assert allocator.last_size == 0
msg_builder = capnp._PyCustomMessageBuilder(allocator, 1024)
struct_builder = msg_builder.init_root(all_types.TestAllTypes)
assert allocator.cur_size == 1024
assert allocator.last_size == 1024
struct_builder.init("dataField", 5)
assert struct_builder._get("dataField") == b"\x00\x00\x00\x00\x00"
struct_builder._get("dataField")[1] = 0xFF
assert struct_builder._get("dataField") == b"\x00\xff\x00\x00\x00"
struct_builder.dataField = b"hello"
assert struct_builder._get("dataField") == b"hello"
struct_builder = struct_builder.as_reader()
assert struct_builder._get("dataField") == b"hello"