From 5befba15f389662fe188cd9759bb7de416f186eb Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 21 Sep 2026 18:57:37 -0700 Subject: [PATCH] Minimal pycapnp --- .github/workflows/docs.yml | 66 - .github/workflows/wheels.yml | 119 +- .gitignore | 4 + CHANGELOG.md | 408 --- MANIFEST.in | 24 +- README.md | 299 +-- benchmark/addressbook.capnp | 24 - benchmark/addressbook.capnp.orphan.py | 50 - benchmark/addressbook.capnp.py | 95 - benchmark/addressbook.proto | 26 - benchmark/addressbook.proto.py | 47 - benchmark/addressbook_pb2.py | 278 -- benchmark/bin/README.md | 13 - benchmark/bin/pycapnp-carsales | 10 - benchmark/bin/pycapnp-catrank | 10 - benchmark/bin/pycapnp-eval | 10 - benchmark/bin/pyproto-carsales | 10 - benchmark/bin/pyproto-catrank | 10 - benchmark/bin/pyproto-eval | 10 - benchmark/bin/pyproto_cpp-carsales | 4 - benchmark/bin/pyproto_cpp-catrank | 4 - benchmark/bin/pyproto_cpp-eval | 4 - benchmark/bin/requirements.txt | 1 - benchmark/bin/run_all.py | 129 - benchmark/bin/runner.py | 90 - benchmark/carsales.capnp | 82 - benchmark/carsales.proto | 81 - benchmark/carsales_pb2.py | 729 ----- benchmark/carsales_proto.py | 110 - benchmark/carsales_pycapnp.py | 114 - benchmark/catrank.capnp | 37 - benchmark/catrank.proto | 36 - benchmark/catrank_pb2.py | 165 -- benchmark/catrank_proto.py | 74 - benchmark/catrank_pycapnp.py | 82 - benchmark/common.py | 94 - benchmark/common_fast.pyx | 20 - benchmark/eval.capnp | 53 - benchmark/eval.proto | 48 - benchmark/eval_pb2.py | 239 -- benchmark/eval_proto.py | 112 - benchmark/eval_pycapnp.py | 120 - buildutils/build.py | 23 +- capnp/__init__.py | 48 +- capnp/_gen.py | 66 - capnp/helpers/capabilityHelper.cpp | 199 -- capnp/helpers/capabilityHelper.h | 139 - capnp/helpers/checkCompiler.h | 5 - capnp/helpers/deserialize.h | 12 - capnp/helpers/exception.cpp | 31 + capnp/helpers/exception.h | 18 + capnp/helpers/helpers.pxd | 29 +- capnp/helpers/non_circular.pxd | 8 +- capnp/helpers/rpcHelper.h | 21 - capnp/helpers/serialize.h | 14 - capnp/includes/PyCustomMessageBuilder.cpp | 50 - capnp/includes/PyCustomMessageBuilder.h | 28 - capnp/includes/capnp_cpp.pxd | 260 +- capnp/includes/schema_cpp.pxd | 802 +----- capnp/lib/capnp.pxd | 65 +- capnp/lib/capnp.pyx | 2929 +-------------------- capnp/lib/pickle_helper.py | 7 - capnp/templates/module.pyx | 249 -- capnp/templates/setup.py.tmpl | 38 - docs/Makefile | 153 -- docs/_templates/versioning.html | 8 - docs/capnp.rst | 178 -- docs/conf.py | 313 --- docs/index.rst | 19 - docs/install.rst | 88 - docs/quickstart.rst | 597 ----- examples/addressbook.capnp | 40 - examples/addressbook.py | 62 - examples/async_calculator_client.py | 306 --- examples/async_calculator_server.py | 129 - examples/async_client.py | 55 - examples/async_reconnecting_ssl_client.py | 107 - examples/async_server.py | 46 - examples/async_socket_message_client.py | 53 - examples/async_socket_message_server.py | 60 - examples/async_ssl_calculator_client.py | 323 --- examples/async_ssl_calculator_server.py | 149 -- examples/async_ssl_client.py | 69 - examples/async_ssl_server.py | 67 - examples/calculator.capnp | 97 - examples/py_custom_message_builder.py | 63 - examples/selfsigned.cert | 29 - examples/selfsigned.key | 52 - examples/thread.capnp | 12 - pyproject.toml | 47 +- scripts/capnp-json.py | 54 - scripts/capnp_test_pycapnp.py | 28 - scripts/release-pypi.sh | 170 -- setup.py | 42 +- test/addressbook with spaces.capnp | 39 - test/addressbook-with-dashes.capnp | 39 - test/all-types.packed | Bin 831 -> 0 bytes test/annotations.capnp | 17 - test/schemas/child.capnp | 5 - test/schemas/parent.capnp | 7 - test/test_async_write_large_payload.py | 108 - test/test_capability.capnp | 97 - test/test_capability.py | 411 --- test/test_capability_context.py | 258 -- test/test_context_manager.py | 241 -- test/test_examples.py | 161 -- test/test_get_data_view.py | 292 -- test/test_large_read.py | 18 +- test/test_lifetime.py | 60 + test/test_load.py | 91 - test/test_memory_handling.py | 32 - test/test_object.py | 51 - test/test_openpilot.py | 97 + test/test_py_custom_message_builder.py | 79 - test/test_regression.py | 124 +- test/test_response.capnp | 13 - test/test_response.py | 48 - test/test_rpc.py | 64 - test/test_rpc_calculator.py | 50 - test/test_schema.py | 29 - test/test_serialization.py | 239 +- test/test_struct.py | 9 +- test/test_structs_sequence.capnp | 27 - test/test_structs_sequence.py | 93 - tox.ini | 19 - 125 files changed, 423 insertions(+), 14793 deletions(-) delete mode 100644 .github/workflows/docs.yml delete mode 100644 CHANGELOG.md delete mode 100644 benchmark/addressbook.capnp delete mode 100644 benchmark/addressbook.capnp.orphan.py delete mode 100644 benchmark/addressbook.capnp.py delete mode 100644 benchmark/addressbook.proto delete mode 100644 benchmark/addressbook.proto.py delete mode 100644 benchmark/addressbook_pb2.py delete mode 100644 benchmark/bin/README.md delete mode 100755 benchmark/bin/pycapnp-carsales delete mode 100755 benchmark/bin/pycapnp-catrank delete mode 100755 benchmark/bin/pycapnp-eval delete mode 100755 benchmark/bin/pyproto-carsales delete mode 100755 benchmark/bin/pyproto-catrank delete mode 100755 benchmark/bin/pyproto-eval delete mode 100755 benchmark/bin/pyproto_cpp-carsales delete mode 100755 benchmark/bin/pyproto_cpp-catrank delete mode 100755 benchmark/bin/pyproto_cpp-eval delete mode 100644 benchmark/bin/requirements.txt delete mode 100755 benchmark/bin/run_all.py delete mode 100755 benchmark/bin/runner.py delete mode 100644 benchmark/carsales.capnp delete mode 100644 benchmark/carsales.proto delete mode 100644 benchmark/carsales_pb2.py delete mode 100755 benchmark/carsales_proto.py delete mode 100755 benchmark/carsales_pycapnp.py delete mode 100644 benchmark/catrank.capnp delete mode 100644 benchmark/catrank.proto delete mode 100644 benchmark/catrank_pb2.py delete mode 100755 benchmark/catrank_proto.py delete mode 100755 benchmark/catrank_pycapnp.py delete mode 100644 benchmark/common.py delete mode 100644 benchmark/common_fast.pyx delete mode 100644 benchmark/eval.capnp delete mode 100644 benchmark/eval.proto delete mode 100644 benchmark/eval_pb2.py delete mode 100755 benchmark/eval_proto.py delete mode 100755 benchmark/eval_pycapnp.py delete mode 100644 capnp/_gen.py delete mode 100644 capnp/helpers/capabilityHelper.cpp delete mode 100644 capnp/helpers/capabilityHelper.h delete mode 100644 capnp/helpers/deserialize.h create mode 100644 capnp/helpers/exception.cpp create mode 100644 capnp/helpers/exception.h delete mode 100644 capnp/helpers/rpcHelper.h delete mode 100644 capnp/helpers/serialize.h delete mode 100644 capnp/includes/PyCustomMessageBuilder.cpp delete mode 100644 capnp/includes/PyCustomMessageBuilder.h delete mode 100644 capnp/lib/pickle_helper.py delete mode 100644 capnp/templates/module.pyx delete mode 100644 capnp/templates/setup.py.tmpl delete mode 100644 docs/Makefile delete mode 100644 docs/_templates/versioning.html delete mode 100644 docs/capnp.rst delete mode 100644 docs/conf.py delete mode 100644 docs/index.rst delete mode 100644 docs/install.rst delete mode 100644 docs/quickstart.rst delete mode 100644 examples/addressbook.capnp delete mode 100755 examples/addressbook.py delete mode 100755 examples/async_calculator_client.py delete mode 100755 examples/async_calculator_server.py delete mode 100755 examples/async_client.py delete mode 100755 examples/async_reconnecting_ssl_client.py delete mode 100755 examples/async_server.py delete mode 100644 examples/async_socket_message_client.py delete mode 100644 examples/async_socket_message_server.py delete mode 100755 examples/async_ssl_calculator_client.py delete mode 100755 examples/async_ssl_calculator_server.py delete mode 100755 examples/async_ssl_client.py delete mode 100755 examples/async_ssl_server.py delete mode 100644 examples/calculator.capnp delete mode 100644 examples/py_custom_message_builder.py delete mode 100644 examples/selfsigned.cert delete mode 100644 examples/selfsigned.key delete mode 100644 examples/thread.capnp delete mode 100755 scripts/capnp-json.py delete mode 100755 scripts/capnp_test_pycapnp.py delete mode 100755 scripts/release-pypi.sh delete mode 100644 test/addressbook with spaces.capnp delete mode 100644 test/addressbook-with-dashes.capnp delete mode 100644 test/all-types.packed delete mode 100644 test/annotations.capnp delete mode 100644 test/schemas/child.capnp delete mode 100644 test/schemas/parent.capnp delete mode 100644 test/test_async_write_large_payload.py delete mode 100644 test/test_capability.capnp delete mode 100644 test/test_capability.py delete mode 100644 test/test_capability_context.py delete mode 100644 test/test_context_manager.py delete mode 100644 test/test_examples.py delete mode 100644 test/test_get_data_view.py create mode 100644 test/test_lifetime.py delete mode 100644 test/test_memory_handling.py delete mode 100644 test/test_object.py create mode 100644 test/test_openpilot.py delete mode 100644 test/test_py_custom_message_builder.py delete mode 100644 test/test_response.capnp delete mode 100644 test/test_response.py delete mode 100644 test/test_rpc.py delete mode 100644 test/test_rpc_calculator.py delete mode 100644 test/test_structs_sequence.capnp delete mode 100644 test/test_structs_sequence.py delete mode 100644 tox.ini diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 452f0aa..0000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Docs - -on: - push: - branches: [master, main] - tags: - - 'v*' - pull_request: - -jobs: - docs: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-glob: "pyproject.toml" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install project and dependencies - run: | - # This replaces requirements.txt, setup.py build, and pip install . - # It reads pyproject.toml, builds pycapnp, and installs it into a fast .venv - uv sync --all-extras - - - name: Build documentation - run: | - # uv run executes sphinx within the isolated virtual environment - uv run sphinx-build docs build/html - - - name: Upload documentation artifact - uses: actions/upload-artifact@v4 - with: - name: html-docs - path: build/html - - deploy: - needs: docs - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Download documentation artifact - uses: actions/download-artifact@v4 - with: - name: html-docs - path: build/html - - - name: Add .nojekyll file - run: touch build/html/.nojekyll - - - name: Deploy to gh-pages - uses: peaceiris/actions-gh-pages@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./build/html - publish_branch: gh-pages diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 60e01c7..741629f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -3,135 +3,48 @@ name: Build on: [push, pull_request] jobs: - build_wheels: - name: Build wheels on ${{ matrix.os }} ${{ matrix.arch }} - runs-on: ${{ matrix.os }} + wheels: strategy: - max-parallel: 99 fail-fast: false matrix: include: - os: ubuntu-latest arch: x86_64 - - os: ubuntu-latest - arch: i686 - # Native ARM runner — no QEMU, drastically faster than emulated aarch64 - os: ubuntu-24.04-arm arch: aarch64 - - - os: macos-15 - arch: x86_64 - os: macos-15 arch: arm64 - # Disabled until someone figures out how to build capnproto for arm64 and x86_64 simultaneously - # - os: macOS-latest - # arch: universal2 - - - os: windows-2022 - arch: AMD64 - - os: windows-2022 - arch: x86 - # Does not build currently - # - os: windows-2019 - # arch: ARM64 - + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 - - - name: Build wheels - uses: pypa/cibuildwheel@v3.4.1 - with: - output-dir: wheelhouse + - uses: pypa/cibuildwheel@v3.4.1 env: + CIBW_BUILD: cp312-* CIBW_ARCHS: ${{ matrix.arch }} - # Drop EOL CPython 3.8 (also avoids the macOS x86_64-only-installer warning) - CIBW_SKIP: "cp38-*" - CIBW_TEST_REQUIRES: pytest pytest-asyncio - CIBW_TEST_COMMAND: pytest {project} - # Force libcapnp to build for the target arch on macOS (the runner - # is arm64, so without this an x86_64 wheel ends up linking against - # an arm64 libkj/libcapnp and failing to load at test time). + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: python -m pytest {project}/test + CIBW_CONFIG_SETTINGS: force-bundled-libcapnp=true CMAKE_OSX_ARCHITECTURES: "${{ runner.os == 'macOS' && matrix.arch || '' }}" - - uses: actions/upload-artifact@v6 with: - name: cibw-wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ strategy.job-index }} - path: ./wheelhouse/*.whl + name: wheels-${{ matrix.arch }}-${{ matrix.os }} + path: wheelhouse/*.whl - # Exotic arches built via QEMU emulation — very slow (often >1h), so only run - # on release tags rather than every push/PR. - build_wheels_exotic: - name: Build wheels (exotic) ${{ matrix.arch }} - if: startsWith(github.ref, 'refs/tags/v') - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - arch: [ppc64le, s390x] - steps: - - uses: actions/checkout@v5 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - - name: Build wheels - uses: pypa/cibuildwheel@v3.4.1 - with: - output-dir: wheelhouse - env: - CIBW_ARCHS: ${{ matrix.arch }} - CIBW_SKIP: "cp38-*" - # Tests under QEMU are extremely slow; skip them for these arches. - CIBW_TEST_SKIP: "*" - - - uses: actions/upload-artifact@v6 - with: - name: cibw-wheels-${{ matrix.arch }} - path: ./wheelhouse/*.whl - - build_sdist: - name: Build source distribution + source: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - - name: Build sdist - run: pipx run build --sdist - + - uses: astral-sh/setup-uv@v3 + - run: uv build --sdist - uses: actions/upload-artifact@v6 with: - name: cibw-sdist + name: sdist path: dist/*.tar.gz lint: - name: Lint and format with ruff runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - - name: Lint and format with ruff - run: | - pip install --upgrade pip - pip install --group lint - ruff check . - ruff format --check . - - # upload_pypi: - # needs: [build_wheels, build_sdist] - # runs-on: ubuntu-latest - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - # steps: - # - uses: actions/download-artifact@v3 - # with: - # # unpacks default artifact into dist/ - # # if `name: artifact` is omitted, the action will create extra parent dir - # name: artifact - # path: dist - - # - uses: pypa/gh-action-pypi-publish@v1.5.0 - # with: - # user: __token__ - # password: ${{ secrets.PYPI_PASSWORD_RELEASE }} - - # # password: ${{ secrets.PYPI_PASSWORD }} - # # repository_url: https://test.pypi.org/legacy/ + - uses: astral-sh/setup-uv@v3 + - run: uvx ruff@0.16.8 check . + - run: uvx ruff@0.16.8 format --check . diff --git a/.gitignore b/.gitignore index 428343e..2a6d478 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ example # capnp files *.capnp + +.venv/ +.pytest_cache/ +.ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 8a0e578..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,408 +0,0 @@ -## v2.2.4 (2026-07-03) -- Fix memory leak in `_DynamicCapabilityClient._send_helper()` (#398) -- Fix SIGSEGV (NULL pointer dereference) on malformed Text field -- Update default bundled capnproto to 1.4.0 -- Migrate project dependency management, testing, and CI to `uv` -- Consolidate configurations into `pyproject.toml` (removed `Pipfile` and `requirements.txt`) -- Apply small ruff linting fixes - -## v2.2.3 (2026-05-30) -- Fix test failures on Python 3.14 (#394) -- Refine documentation for PyCustomMessageBuilder (#395) -- Replace black and flake8 with ruff for linting and formatting -- ci: deploy docs to gh-pages on tagged releases -- Add scripts/release-pypi.sh for downloading CI artifacts and uploading to PyPI - -## v2.2.2 (2026-01-16) -- Revert Data fields to bytes and add get_data_as_view for zero-copy access (#390) -- Fix use-after-free in async write causing corruption with large payloads (#392) -- Update macOS builds from 13 to 15 (#393) - -## v2.2.1 (2025-10-21) -- Make `message.to_dict()` return bytes for DATA type field (#386) - -## v2.2.0 (2025-09-12) -- Add binary support in dictionaries via base64 encoding (#351) -- Add structure-free read_multiple_bytes_packed (#378) -- Support python custom message builder and make Data field's type return MemoryView (#380) - -## v2.1.0 (2025-09-04) -- Add Python 3.13 support - * Python 3.8 is still enabled but will be disabled if changes are needed that prevent compatibility with newer versions of Python (e.g. 3.14) - * Disabling experimental Python 3.14 build as it currently causes build issues -- Upgrade to Cython version 3 -- Include \_custom_build in sdist -- Remove usage of deprecated kj::mvCapture functions -- Make license information SPDX-compatible -- Generate a new certificate that is compatible with strict x509 checking -- Avoid storm of 'warning: moving a temporary object prevents copy elision' -- cibuildwheel updates -- Fix deprecation warning when importing a schema - -## v2.0.0 (2024-01-19) -- Updated link for mailing list in README - -## v2.0.0b2 (2023-11-25) -- Fix broken test in test_load (#329) -- Update README example to async code (#331) -- Fix 'AttributeError: '\_UnixSelectorEventLoop' object has no attribute 'call_soon' -- Delete and update some Python 3.7-specific todo notes -- Make a server fail early when the KJ loop is not running -- Update documentation to async code (#331) (#332) -- Fix retransmit bug for large messages causing message corruption -- Unlock the GIL for all capnp functions that do IO -- Handle exceptions from server callbacks -- Disable the use of ninja for windows builds -- DynamicCapabilityClient fix -- Make `reraise_kj_exception` available to downstream -- Support `_DynamicListReader` in `_setDynamicField` -- Fix re-raising of KjException -- Allow cancellation of all capability contexts -- Corner case for cancelled server methods that raise exceptions -- Some fixes to the magic import system - -## v2.0.0b1 (2023-10-03) -- Update to bundled capnproto-1.0.1 -- Remove support for Python 3.7 -- Use custom build backend to support build args (#328) -- Update Cython version and Python to 3.12 (#320) -- Wrap all capnp code in a context-manager to avoid segfaults (#317) -- Schema loading from the wire (#307) -- Make pycapnp more GIL friendly (#308) -- Use cibuildwheel in ci (#309) -- Integrate the KJ event loop into Python's asyncio event loop (#310) -- Allow capability implementation methods to be `async` (#312) -- Allow reading and writing messages from sockets in `async` mode (#313) -- Remove the synchronous RPC mode (#315) - -## v1.3.0 (2023-01-26) -- Update to bundled capnproto-0.10.3 -- Add Python 3.11 to Github Actions builds (#306) -- Prevent race condition in example code (#305) - -## v1.2.2 (2022-12-01) -- Update bundled bundled capnp to 0.8.1 due to CVE-2022-46149 -- Bundle lib/capnp_api.h and helpers/capabilityHelper.cpp (#301) -- Avoid reading random values for reader options from dangling reference (#300) - -## v1.2.1 (2022-09-11) -- Fix packaging for Apple Silicon - -## v1.2.0 (2022-08-29) -- Added support for Apple Silicon - -## v1.1.1 (2022-05-23) -- Added Python 3.10 support -- aarch64 wheel support -- Fix doc string for `_DynamicResizableListBuilder` -- fix for unreleased buffers under mmap (issue 280) - -## v1.1.0 (2021-06-09) -- Validated compatibility with Python 3.10.0b2 -- Remove all bare except -- Improve `_StructModuleWhich` to inherit from `enum.Enum` -- Add Union on top level union messages -- Fixed memory leak in `_SegmentArrayMessageReader` -- Removed many pycodestyle warnings -- Avoid crash if `__file__` is not set by importer -- Fixed module.pyx `_set_` for boolean fields -- Fixed setup.py.tmpl support for `*.c++` files -- Fixed `_gen.py` for python3 as `dict_keys` object are not indexable. -- Add test data to sdist -- Add `pyproject.yaml` -- Add missing inheritance to `_Schema` for `_StructSchema` - -## v1.0.0 (2020-11-20) -- Validated Python 3.9 (3.7 and 3.8 are also supported) -- Updated package to include LICENSE file -- Updated examples to avoid run_forever() as ctrl+c will not work -- Adding xfail to pytest cases which fail sometimes due to network port oddities (please use asyncio, as Python handles things more gracefully) - -## v1.0.0b2 (2020-06-14) -- Minimum capnproto version is now 0.8.0 -- Added asyncio ssl calculator test -- Added poll_once to TwoPartyServer API -- More cleanup -- Fix absolute and circular imports -- Fix Promise aliasing issue (Promise to \_Promise) -- Documentation update -- Updated installation instructions -- Added RPC documentation for asyncio - -## v1.0.0b1 (2019-12-26) -- Python 3.7+ required (asyncio support) -- TLS/SSL support using asyncio -- Windows support -- General cleanup -- May be incompatible with code written for pycapnp 0.6.4 and lower -- Removing pypandoc/pandoc packaging requirement -- Minimum capnproto version is now 0.7.0 - -## v0.6.4 (2019-01-31) -- Fix bugs in `read_multiple_bytes` (thanks to @tsh56) -- Remove end-of-life Python versions 2.6, 3.2, and 3.3. Add CI tests for 3.6 -- Expose SchemaParser in Cython header - -## v0.6.3 (2018-01-14) -- Bump bundled capnp version to v0.6.1 (thanks to @E8Yuval) -- Fix a memleak in RemotePromise (thanks to @E8Yuval) - -## v0.6.2 (2017-11-30) -- Add support for buffers/memoryviews in `from_bytes` (thanks to @aldanor) - -## v0.6.1 (2017-07-27) -- Fixed upload to PyPi (forgot to cythonize) - -## v0.6.0 (2017-07-27) -- Update bundled capnp version to v0.6.0 and fix related problems (thanks to @benmoran) -- Fix memleak with KjException (thanks to @tsh56) - -## v0.5.12 (2017-04-18) -- Bump bundled capnp version to v0.5.3.1 - -## v0.5.11 (2017-04-10) -- Make enums hashable (thanks to @madeleine-empirical) -- Rework logic on when to build bundled libcapnp. Fixes cross-compilation (thanks to @benizl) -- Add traversal_limit_in_words and nesting_limit to RPC classes (thanks to @asilversempirical) -- Include class attributes in __dir__. This allows for code completion of class methods (thanks to @chaoflow ) -- Allow setting lists with python tuples (thanks to @chaoflow) -- Fix traversal_limit_in_words and nesting_limit being ignored by `from_bytes` (thanks to @plesner) - -## v0.5.10 (2016-11-28) -- Fix bug that prevented event loop from actually being lazy initialized -- Fix possible recursive loop in KjException -- Add `clear_write_flag` method to builder classes - -## v0.5.9 (2016-07-07) -- Make the event loop be lazy initialized -- Add support for segment (de)serialization (thanks to @gcv). See to_segments/from_segments methods. -- Fix response objects not referencing parents correctly -- Add test for large reads - -## v0.5.8 (2016-05-27) -- Fix build problem with Cython v0.24 -- Include the changelog in the manifest (should fix install problems if pandoc is present) -- Include the traceback in exceptions -- Make sure to encode to utf-8, not the default encoding (thanks to @novas0x2a) -- Add --libcapnp-url option in installer to allow installing arbitrary libcapnp versions -- Support mmap objects for reading with from_bytes (thanks to @bpiwowar) -- Change read_multiple and read_multiple_packed to copy by default -- Fix mistakenly discarding the file parameter on reads -- Add reraise_kj_exception to the prettyPrint functions. (thanks to @kdienes) -- Fix KjException init (missing wrapper). (thanks to @E8-Storage) -- Add `result_type` to InterfaceMethodSchema - - -## v0.5.7 (2015-06-16) -- Update bundled libcapnp to v0.5.2 -- Add warnings for using old restorer methods. You should use `bootstrap` instead -- Fix warning from PyEventPort -- Handle AnyPointers better as arguments to RPC functions -- Add support for using keyword arguments with a named struct in an RPC -- Add bootstrap method to TwoPartyServer -- Add `init` method to lists -- Add support for unix sockets in RPC - -## v0.5.6 (2015-04-13) -- Fix a serious bug in TwoPartyServer that was preventing it from working when passed a string address. -- Fix bugs that were exposed by defining KJDEBUG (thanks @davidcarne for finding this) - - -## v0.5.5 (2015-03-06) -- Update bundled C++ libcapnp to v0.5.1.2 security release - - -## v0.5.4 (2015-03-02) -- Update bundled C++ libcapnp to v0.5.1.1 security release -- Add bootstrap RPC methods -- Fix possible segfault when importing multiple schemas - - -## v0.5.3 (2015-02-23) -- Fix possible crash due to bad destructor ordering in MessageReader (by @JohnEmhoff) -- Default to no longer using cython - - -## v0.5.2 (2015-02-20) -- Add read\_multiple\_bytes/read\_multiple\_bytes\_packed methods -- Added Python 3.4 to the travis build matrix -- Bump version for bundled C++ libcapnp to v0.5.1 - - -## v0.5.1 (2014-12-27) -- Remove installation dependency on cython. We now have no dependencies since libcapnp will automatically build as well. - - -## v0.5.0 (2014-12-15) -- Timer class `capnp.getTimer()` -- pycapnp is now thread-safe and allows an event loop to be run in each thread - - You must destroy and re-create the event loop to get this functionality (see `test_threads.py`) -- Inheritance now works correctly for interfaces (previously inherited methods were inaccessible from pycapnp) -- Add ability to import modules with dashes or spaces. Use underscores in place of them -- `from_bytes` with builder=True is no longer zero copy. It never worked correctly, and is much safer now -- Add `num_first_segment_words` argument wherever message creation can occur -- Allow restoring a null objectId by passing None to restore -- Support ordered dictionary in `to_dict` -- Add ListSchema class and schemas for native types under `capnp.types` which completes all the Schemas needed to be wrapped. See `test_schema.py` for examples using it -- Add automatic build of C++ libcapnp if it's not detected on the system. Also add flags --force-bundled-libcapnp and --force-system-libcapnp respectively - - -## v0.4.6 (2014-9-10) -- Fix build for new 0.21 release of Cython. 0.21 is now the minimum supported version of Cython. - - -## v0.4.5 (2014-6-26) -- Fix `to_dict` not converting enums to strings - - -## v0.4.4 (2014-04-25) -- Fix compilation problem with gcc 4.8 - - -## v0.4.3 (2014-02-18) -- Fix problem with uninitialized unions in \_from\_dict -- Add accesible version numbers for C++ libcapnp - - -## v0.4.2 (2014-02-13) -- Remove onDrained since it was removed upstream -- Replace usage of strings as enum type with custom `_DynamicEnum` class. -- Also change `Struct.which()` method to be a property `Struct.which` and return an enum type (`_DynamicEnumField`, which behaves much like `_DynamicEnum`). -- TwoPartyServer.run_forever() now will handle more than 1 simulataneous connection. -- Change exception wrapper to detect and raise AttributeError for field lookup exceptions (Fixes problem in Python3.x `__dir__`) -- Allow setting of fields with python dicts. - - -## 0.4.1 (2013-12-18) -- Remove python 3.2 from travis tests. Python 3.2 still should work fine, but it's more trouble than it's worth to write unicode tests that work in both it and Python2. -- Fix problems with null characters in Text/Data fields. Fixes #19 - - -## 0.4.0 (2013-12-12) -- Initial working version of RPC -- Add get_root_as_any to _MessageReader -- Add capnp.pxd for public declarations of cython classes -- Fix problems compiling with gcc4.7 - - -## v0.3.18 (2013-11-05) -- Change naming of ReaderOption parameters to be pep8 compliant - - -## v0.3.17 (2013-11-05) -- Add ReaderOptions to read/read_packed/from_bytes - - -## v0.3.16 (2013-10-28) -- Add defaults flag to capnp-json. Also remove 'which' field -- Add capnp-json serializer script. Also fix bugs in from_dict -- Fix build for clang/python3. Also remove -fpermissive -- Add `as_builder` method to Struct Reader -- Add warning when writing the same message more than once -- First working version of capability interfaces -- Wrap InterfaceSchema -- Fix setting string fields to support all types of strings -- Fix changed API for DynamicObject/ObjectPointer - - -## v0.3.15 (2013-09-19) -- Add not having installed the C++ libcapnp library to 'Common Problems' -- Add _short_str function for use in capnp_test_pycapnp.py -- Add test script for testing with https://github.com/kaos/capnp_test -- Add handling of DynamicObject -- Fix lists of lists or dicts for from_dict - - -## v0.3.14 (2013-09-04) -- Fix problem with to_dict - - -## v0.3.13 (2013-09-04) -- Add _DynamicStructBuilder.to_bytes() and .from_bytes() -- Change == on StructSchema to return cbool -- Add Builder and Reader ABCs for each struct type - -## v0.3.12 (2013-09-03) -- Fix handling of empty path '' in load_module -- Add from_dict -- Fix bug in exception handling for which(). Also standardize exceptions. -- Change import hook to require modules to end in '_capnp' -- Add import monkey patch function. -- Change naming for functions to conform to PEP 8. Also deprecate old read/write API -- Update preferred method for reading/writing messages from files - -## v0.3.11 (2013-09-01) -- Forgot to change project name in setup.py - -## v0.3.10 (2013-09-01) -- Change all references to old project name (change from capnpc-python-cpp to pycapnp) -- Change DynamicValue.Reader lists to be returned as _DynamicListReader -- Unify setters for DynamicList and DynamicStruct -- Add shortcuts for reading from / writing to files. In Python, it doesn't make much sense to force people to muck around with MessageReaders and MessageBuilders since everything is landing on the heap anyway. Instead, let's make it easy: MyType.read[Packed]From(file) reads a file and returns a MyType reader. MyType.newMessage() returns a MyType builder representing the root of a new message. You can call this builder's write[Packed]To(file) method to write it to a file. -- Store Builders by value rather than allocate them separately on the heap (matches treatment of Readers). v0.3 fixes the bug that made this not work. -- Wrap MessageBuilder::setRoot(). -- Add tests based on TestAllTypes from the C++ test.capnp. Fix problems uncovered in capnp.pyx. -- Implement __str__ and __repr__ for struct and list builders. __str__ uses prettyPrint while __repr__ shows the type name and the low-whitespace stringification. Also implement __repr__ for StructSchema, just because why not? - - -## v0.3.9 (2013-08-30) -- Change load to use a global SchemaParser. Make structs settable as field -- Add docstrings for new functions and _DynamicResizableListBuilder - - -## v0.3.8 (2013-08-29) -- Add initial tests -- Add _capnp for original Cython module. Meant for testing. -- Lowercase schema so it conforms to member naming conventions -- Expose _StructSchema's raw node -- Add some useful _StructSchema, reader, and builder methods -- Add full orphan functionality. Also, allow special orphan lists -- Finish up adding docstrings to all public classes/methods - - -## v0.3.7 (2013-08-26) -- Add a ton of docstrings and add to official docs -- Add DynamicOrphan - - -## v0.3.6 (2013-08-26) -- Add intersphinx for linking to python docs -- Add C++ library version check - - -## v0.3.5 (2013-08-25) -- Add handling of constants in schemas -- Fix new error with DynamicValue.Builder no longer being copyable - - -## v0.3.4 (2013-08-22) -- Fix Void namespace change -- Updated capnp schema to conform with new union rules - - -## v0.3.3 (2013-08-22) -- Fix for the removal of DynamicUnion from the C++ API - - -## v0.3.2 (2013-08-21) -- Add MANIFEST.in to include README - - -## v0.3.1 (2013-08-21) -- Update docs with lines about upgrading setuptools - - -## 0.3.0 (2013-08-21) -- Initial commit of docs -- Add querying unnamed enums to structs - - -## 0.2.1 (2013-08-13) -- Fix enum interface change for benchmark -- Random formatting cleanup -- Allow import paths in the schema loader -- Add travis CI - - -## 0.2.0 (2013-08-12) -- Initial working version diff --git a/MANIFEST.in b/MANIFEST.in index ce627de..dabd5c2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,18 +1,6 @@ -include README.md -include LICENSE.md -include CHANGELOG.md -include requirements.txt -include buildutils/* -include _custom_build/* -include capnp/lib/capnp_api.h -include Pipfile -include tox.ini -recursive-include examples *.capnp -recursive-include examples *.cert -recursive-include examples *.key -recursive-include examples *.py -recursive-include test *.binary -recursive-include test *.capnp -recursive-include test *.packed -recursive-include test *.txt -recursive-include test *.py +include README.md LICENSE.md +include buildutils/*.py +include _custom_build/*.py +recursive-include capnp *.py *.pyx *.pxd *.h *.cpp +recursive-include test *.py *.capnp *.binary *.txt +exclude capnp/lib/capnp.cpp capnp/lib/capnp.h capnp/lib/capnp_api.h diff --git a/README.md b/README.md index 44839f1..cdf388c 100644 --- a/README.md +++ b/README.md @@ -1,265 +1,62 @@ -# pycapnp +# pycapnp for openpilot -[![Packaging Status](https://github.com/capnproto/pycapnp/workflows/Packaging%20Test/badge.svg)](https://github.com/capnproto/pycapnp/actions) -[![manylinux2014 Status](https://github.com/capnproto/pycapnp/workflows/manylinux2014/badge.svg)](https://github.com/capnproto/pycapnp/actions) -[![PyPI version](https://badge.fury.io/py/pycapnp.svg)](https://badge.fury.io/py/pycapnp) +A serialization-only fork of [pycapnp](https://github.com/capnproto/pycapnp). +The `minimal` branch starts at upstream commit +`a0cb5cdf0673481f2f9850541f4d42b89698c476`, including the `from_dict` reference-cycle +fix in upstream PR #407. -[Cap'n'proto Mailing List](https://github.com/capnproto/capnproto/discussions) [Documentation](https://capnproto.github.io/pycapnp) +The supported surface is based on openpilot at +`7f6f13997c3c9b8e27e1581583f61e3fcabc5151`, including its opendbc checkout: +- Explicit schema loading with `capnp.load()` and schema imports. +- Dynamic structs, lists, enums, unions, nested groups, and constants. +- Message construction, field access (including cached `_get_by_field` and + `_set_by_field`), dictionaries, reader/builder copies, and pickling. +- Unpacked `to_bytes()`, `from_bytes()`, and `read_multiple_bytes()`, including + traversal/nesting limits and readers that retain their underlying message. +- Schema reflection for cereal, CAN conversion, WebRTC, fuzzing, and replay tools. -## Requirements +Removed: RPC/capabilities, promises, KJ event loops, asyncio/network streams, +packed serialization, file-descriptor I/O, segment APIs, borrowed Data views, +custom allocators, orphans/resizable lists, AnyPointer wrappers, type registration, +the Python schema import hook, and the Cython code generator. Their examples, +tests, docs, dependencies, and unsupported-platform CI were removed too. +`remove_import_hook()` remains a no-op for cereal/opendbc compatibility. -* C++14 supported compiler - - gcc 6.1+ (5+ may work) - - clang 6 (3.4+ may work) - - Visual Studio 2017+ -* cmake (needed for bundled capnproto) - - ninja (macOS + Linux) - - Visual Studio 2017+ -* capnproto-1.0 (>=0.8.0 will also work if linking to system libraries) - - Not necessary if using bundled capnproto -* Python development headers (i.e. Python.h) - - Distributables from python.org include these, however they are usually in a separate package on Linux distributions +This is intentionally not a full upstream API replacement. The import and +package names remain `capnp` and `pycapnp`. It must replace the installed pycapnp, +not be installed alongside another distribution providing `capnp`. -32-bit Linux requires that capnproto be compiled with `-fPIC`. This is usually set correctly unless you are compiling canproto yourself. This is also called `-DCMAKE_POSITION_INDEPENDENT_CODE=1` for cmake. +## Build and test -pycapnp has additional development dependencies, including cython and pytest. See requirements.txt for them all. +Targets: CPython 3.12, Linux x86_64/aarch64, and macOS arm64. A C++14 compiler and +CMake are required for a bundled build. - -## Building and installation - -Install with `pip install pycapnp`. You can set the CC environment variable to control which compiler is used, ie `CC=gcc-8.2 pip install pycapnp`. - -Or you can clone the repo like so: - -```bash -git clone https://github.com/capnproto/pycapnp.git -cd pycapnp -pip install . +```sh +uv venv --python 3.12 +uv pip install cython setuptools wheel pkgconfig pytest build +.venv/bin/python setup.py build_ext --inplace --force-bundled-libcapnp +.venv/bin/python -m pytest +.venv/bin/python -m build -Cforce-bundled-libcapnp=true ``` -By default, the setup script will automatically use the locally installed Cap'n Proto. -If Cap'n Proto is not installed, it will bundle and build the matching Cap'n Proto library. +The existing build fallback downloads Cap'n Proto 1.4.0. The extension links only +`capnpc`, `capnp`, and `kj`; it does not link `capnp-rpc` or `kj-async`. `capnpc` is +needed for runtime schema parsing. Owning/vendoring the C++ source itself is a +separate step. To use a system installation, pass `--force-system-libcapnp` to +`build_ext` (or `-Cforce-system-libcapnp=true` to the wheel build). -To enforce bundling, the Cap'n Proto library: +The retained upstream tests cover message construction, schema loading, +reflection, binary fixtures, serialization, and exceptions. Added lifetime tests +check kwargs construction with GC disabled and readers surviving their input or +iterator. Optional integration tests use real openpilot schemas and exercise +messaging, CAN conversion, WebRTC reflection, LogReader, pickling, and replay: -```bash -pip install . -C force-bundled-libcapnp=True +```sh +# Run in an environment with openpilot's dependencies and this fork installed. +OPENPILOT_PATH=/path/to/openpilot python -m pytest test/test_openpilot.py +python -m pytest /path/to/openpilot/openpilot/cereal/messaging/tests \ + /path/to/openpilot/openpilot/tools/lib/tests/test_logreader.py ``` -If you wish to install using the latest upstream C++ Cap'n Proto: - -```bash -pip install . \ - -C force-bundled-libcapnp=True \ - -C libcapnp-url="https://github.com/capnproto/capnproto/archive/master.tar.gz" -``` - -To enforce using the installed Cap'n Proto from the system: - -```bash -pip install . -C force-system-libcapnp=True -``` - -The bundling system isn't that smart so it might be necessary to clean up the bundled build when changing versions: - -```bash -python setup.py clean -``` - - -## Stub-file generation - -While not directly supported by pycapnp, a tool has been created to help generate pycapnp stubfile to assist with development (this is very helpful if you're new to pypcapnp!). See [#289](https://github.com/capnproto/pycapnp/pull/289#event-9078216721) for more details. - -[Python Capnp Stub Generator](https://gitlab.com/mic_public/tools/python-helpers/capnp-stub-generator) - - -## Python Versions - -Python 3.9+ is supported. - - -## Development - -Git flow has been abandoned, use master. - -To test, use a pipenv (or install requirements.txt and run pytest manually). -```bash -pip install pipenv -pipenv install -pipenv run pytest -``` - - -### Binary Packages - -Building a Python wheel distributiion - -```bash -pip wheel . -``` - - -### Releasing to PyPI - -Wheels and the sdist are built by the `Build` GitHub Actions workflow -(`.github/workflows/wheels.yml`) for every push, including tag pushes. The -`scripts/release-pypi.sh` helper downloads those artifacts for a given tag (or -explicit run ID) and uploads them to PyPI via `twine`. - -Typical release flow: - -```bash -git tag v2.2.1 -git push origin v2.2.1 -# wait for the "Build" workflow run to finish successfully on GitHub - -# Download artifacts and upload to PyPI (creates dist_221/ by default). -scripts/release-pypi.sh v2.2.1 - -# Or, target a specific Actions run id: -scripts/release-pypi.sh 1234567890 - -# Dry run: upload to TestPyPI (https://test.pypi.org) instead of real PyPI. -# Useful for validating the release flow end-to-end before pushing to -# production. Requires a TestPyPI account + API token configured in -# ~/.pypirc under a [testpypi] section. See -# https://packaging.python.org/en/latest/guides/using-testpypi/ . -scripts/release-pypi.sh v2.2.1 --test -``` - -Requirements on the release machine: - -- `gh` CLI, authenticated (`gh auth login`) -- `python3` (the script creates `.venv-release/` and installs `twine` into it) -- PyPI credentials available to `twine`, e.g. `TWINE_USERNAME=__token__` and - `TWINE_PASSWORD=`, or a configured `~/.pypirc` - -The script: - -1. Resolves the latest successful `wheels.yml` run for the tag (or uses the - given run ID). -2. Downloads `cibw-*` artifacts and flattens all `*.whl` / `*.tar.gz` files - into the output directory (default `dist_` for tags, - `dist_run_` for run IDs; pass a second arg to override, and `--force` - to reuse a non-empty directory). -3. Runs `twine check`, prints the file list, and prompts before running - `twine upload`. - -## Documentation/Example - -There is some basic documentation [here](http://capnproto.github.io/pycapnp/). - -Make sure to look at the [examples](examples). The examples are generally kept up to date with the recommended usage of the library. - -The examples directory has one example that shows off pycapnp quite nicely. Here it is, reproduced: - -```python -import os -import capnp - -import addressbook_capnp - -def writeAddressBook(file): - addresses = addressbook_capnp.AddressBook.new_message() - people = addresses.init('people', 2) - - alice = people[0] - alice.id = 123 - alice.name = 'Alice' - alice.email = 'alice@example.com' - alicePhones = alice.init('phones', 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = 'mobile' - alice.employment.school = "MIT" - - bob = people[1] - bob.id = 456 - bob.name = 'Bob' - bob.email = 'bob@example.com' - bobPhones = bob.init('phones', 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = 'home' - bobPhones[1].number = "555-7654" - bobPhones[1].type = 'work' - bob.employment.unemployed = None - - addresses.write(file) - - -def printAddressBook(file): - addresses = addressbook_capnp.AddressBook.read(file) - - for person in addresses.people: - print(person.name, ':', person.email) - for phone in person.phones: - print(phone.type, ':', phone.number) - - which = person.employment.which() - print(which) - - if which == 'unemployed': - print('unemployed') - elif which == 'employer': - print('employer:', person.employment.employer) - elif which == 'school': - print('student at:', person.employment.school) - elif which == 'selfEmployed': - print('self employed') - print() - - -if __name__ == '__main__': - f = open('example', 'w') - writeAddressBook(f) - - f = open('example', 'r') - printAddressBook(f) -``` - -Also, pycapnp has gained RPC features that include pipelining and a promise style API. Refer to the calculator example in the examples directory for a much better demonstration: - -```python -import asyncio -import capnp -import socket - -import test_capability_capnp - - -class Server(test_capability_capnp.TestInterface.Server): - - def __init__(self, val=1): - self.val = val - - async def foo(self, i, j, **kwargs): - return str(i * 5 + self.val) - - -async def client(read_end): - client = capnp.TwoPartyClient(read_end) - - cap = client.bootstrap() - cap = cap.cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - response = await remote - - assert response.x == '125' - -async def main(): - client_end, server_end = socket.socketpair(socket.AF_UNIX) - # This is a toy example using socketpair. - # In real situations, you can use any socket. - - client_end = await capnp.AsyncIoStream.create_connection(sock=client_end) - server_end = await capnp.AsyncIoStream.create_connection(sock=server_end) - - _ = capnp.TwoPartyServer(server_end, bootstrap=Server(100)) - await client(client_end) - - -if __name__ == '__main__': - asyncio.run(capnp.run(main())) -``` +See [LICENSE.md](LICENSE.md) for the upstream BSD license and attribution. diff --git a/benchmark/addressbook.capnp b/benchmark/addressbook.capnp deleted file mode 100644 index 862d51c..0000000 --- a/benchmark/addressbook.capnp +++ /dev/null @@ -1,24 +0,0 @@ -@0x934efea7f017fff0; - -struct Person { - id @0 :UInt32; - name @1 :Text; - email @2 :Text; - phones @3 :List(PhoneNumber); - - struct PhoneNumber { - number @0 :Text; - type @1 :Type; - - enum Type { - mobile @0; - home @1; - work @2; - } - } -} - -struct AddressBook { - people @0 :List(Person); -} - diff --git a/benchmark/addressbook.capnp.orphan.py b/benchmark/addressbook.capnp.orphan.py deleted file mode 100644 index 4ac6e59..0000000 --- a/benchmark/addressbook.capnp.orphan.py +++ /dev/null @@ -1,50 +0,0 @@ -import os -import capnp - -this_dir = os.path.dirname(__file__) -addressbook = capnp.load(os.path.join(this_dir, "addressbook.capnp")) - -print = lambda *x: x - - -def writeAddressBook(): - addressBook = addressbook.AddressBook.new_message() - people = addressBook.init_resizable_list("people") - - alice = people.add() - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - - bob = people.add() - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - - people.finish() - msg_bytes = addressBook.to_bytes() - return msg_bytes - - -def printAddressBook(msg_bytes): - with addressbook.AddressBook.from_bytes(msg_bytes) as addressBook: - for person in addressBook.people: - print(person.name, ":", person.email) - for phone in person.phones: - print(phone.type, ":", phone.number) - print() - - -if __name__ == "__main__": - for i in range(10000): - msg_bytes = writeAddressBook() - - printAddressBook(msg_bytes) diff --git a/benchmark/addressbook.capnp.py b/benchmark/addressbook.capnp.py deleted file mode 100644 index 83c6ba1..0000000 --- a/benchmark/addressbook.capnp.py +++ /dev/null @@ -1,95 +0,0 @@ -import os -import capnp - -try: - profile -except: - profile = lambda func: func -this_dir = os.path.dirname(__file__) -addressbook = capnp.load(os.path.join(this_dir, "addressbook.capnp")) - -print = lambda *x: x - - -@profile -def writeAddressBook(): - addressBook = addressbook.AddressBook.new_message() - people = addressBook.init("people", 2) - - alice = people[0] - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - - bob = people[1] - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - - msg_bytes = addressBook.to_bytes() - return msg_bytes - - -@profile -def printAddressBook(msg_bytes): - with addressbook.AddressBook.from_bytes(msg_bytes) as addressBook: - for person in addressBook.people: - person.name, person.email - for phone in person.phones: - phone.type, phone.number - - -@profile -def writeAddressBookDict(): - addressBook = addressbook.AddressBook.new_message() - people = addressBook.init("people", 2) - - alice = people[0] - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - - bob = people[1] - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - - msg = addressBook.to_dict() - return msg - - -@profile -def printAddressBookDict(msg): - addressBook = addressbook.AddressBook.new_message(**msg) - - for person in addressBook.people: - person.name, person.email - for phone in person.phones: - phone.type, phone.number - - -if __name__ == "__main__": - # for i in range(10000): - # msg_bytes = writeAddressBook() - - # printAddressBook(msg_bytes) - for i in range(10000): - msg = writeAddressBookDict() - - printAddressBookDict(msg) diff --git a/benchmark/addressbook.proto b/benchmark/addressbook.proto deleted file mode 100644 index 1c08c6b..0000000 --- a/benchmark/addressbook.proto +++ /dev/null @@ -1,26 +0,0 @@ -syntax = "proto2"; - -package tutorial; - -message Person { - required string name = 1; - required int32 id = 2; - required string email = 3; - - enum PhoneType { - MOBILE = 0; - HOME = 1; - WORK = 2; - } - - message PhoneNumber { - required string number = 1; - optional PhoneType type = 2 [default = HOME]; - } - - repeated PhoneNumber phone = 4; -} - -message AddressBook { - repeated Person person = 1; -} diff --git a/benchmark/addressbook.proto.py b/benchmark/addressbook.proto.py deleted file mode 100644 index d060ca4..0000000 --- a/benchmark/addressbook.proto.py +++ /dev/null @@ -1,47 +0,0 @@ -import addressbook_pb2 as addressbook -import os - -print = lambda *x: x - - -def writeAddressBook(): - addressBook = addressbook.AddressBook() - - alice = addressBook.person.add() - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = [alice.phone.add()] - alicePhones[0].number = "555-1212" - alicePhones[0].type = addressbook.Person.MOBILE - - bob = addressBook.person.add() - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = [bob.phone.add(), bob.phone.add()] - bobPhones[0].number = "555-4567" - bobPhones[0].type = addressbook.Person.HOME - bobPhones[1].number = "555-7654" - bobPhones[1].type = addressbook.Person.WORK - - message_string = addressBook.SerializeToString() - return message_string - - -def printAddressBook(message_string): - addressBook = addressbook.AddressBook() - addressBook.ParseFromString(message_string) - - for person in addressBook.person: - print(person.name, ":", person.email) - for phone in person.phone: - print(phone.type, ":", phone.number) - print() - - -if __name__ == "__main__": - for i in range(10000): - message_string = writeAddressBook() - - printAddressBook(message_string) diff --git a/benchmark/addressbook_pb2.py b/benchmark/addressbook_pb2.py deleted file mode 100644 index b410841..0000000 --- a/benchmark/addressbook_pb2.py +++ /dev/null @@ -1,278 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: addressbook.proto - -import sys - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="addressbook.proto", - package="tutorial", - syntax="proto2", - serialized_pb=_b( - '\n\x11\x61\x64\x64ressbook.proto\x12\x08tutorial"\xda\x01\n\x06Person\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\n\n\x02id\x18\x02 \x02(\x05\x12\r\n\x05\x65mail\x18\x03 \x02(\t\x12+\n\x05phone\x18\x04 \x03(\x0b\x32\x1c.tutorial.Person.PhoneNumber\x1aM\n\x0bPhoneNumber\x12\x0e\n\x06number\x18\x01 \x02(\t\x12.\n\x04type\x18\x02 \x01(\x0e\x32\x1a.tutorial.Person.PhoneType:\x04HOME"+\n\tPhoneType\x12\n\n\x06MOBILE\x10\x00\x12\x08\n\x04HOME\x10\x01\x12\x08\n\x04WORK\x10\x02"/\n\x0b\x41\x64\x64ressBook\x12 \n\x06person\x18\x01 \x03(\x0b\x32\x10.tutorial.Person' - ), -) - - -_PERSON_PHONETYPE = _descriptor.EnumDescriptor( - name="PhoneType", - full_name="tutorial.Person.PhoneType", - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name="MOBILE", index=0, number=0, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="HOME", index=1, number=1, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="WORK", index=2, number=2, options=None, type=None - ), - ], - containing_type=None, - options=None, - serialized_start=207, - serialized_end=250, -) -_sym_db.RegisterEnumDescriptor(_PERSON_PHONETYPE) - - -_PERSON_PHONENUMBER = _descriptor.Descriptor( - name="PhoneNumber", - full_name="tutorial.Person.PhoneNumber", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="number", - full_name="tutorial.Person.PhoneNumber.number", - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="type", - full_name="tutorial.Person.PhoneNumber.type", - index=1, - number=2, - type=14, - cpp_type=8, - label=1, - has_default_value=True, - default_value=1, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=128, - serialized_end=205, -) - -_PERSON = _descriptor.Descriptor( - name="Person", - full_name="tutorial.Person", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="name", - full_name="tutorial.Person.name", - index=0, - number=1, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="id", - full_name="tutorial.Person.id", - index=1, - number=2, - type=5, - cpp_type=1, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="email", - full_name="tutorial.Person.email", - index=2, - number=3, - type=9, - cpp_type=9, - label=2, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="phone", - full_name="tutorial.Person.phone", - index=3, - number=4, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[ - _PERSON_PHONENUMBER, - ], - enum_types=[ - _PERSON_PHONETYPE, - ], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=32, - serialized_end=250, -) - - -_ADDRESSBOOK = _descriptor.Descriptor( - name="AddressBook", - full_name="tutorial.AddressBook", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="person", - full_name="tutorial.AddressBook.person", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=252, - serialized_end=299, -) - -_PERSON_PHONENUMBER.fields_by_name["type"].enum_type = _PERSON_PHONETYPE -_PERSON_PHONENUMBER.containing_type = _PERSON -_PERSON.fields_by_name["phone"].message_type = _PERSON_PHONENUMBER -_PERSON_PHONETYPE.containing_type = _PERSON -_ADDRESSBOOK.fields_by_name["person"].message_type = _PERSON -DESCRIPTOR.message_types_by_name["Person"] = _PERSON -DESCRIPTOR.message_types_by_name["AddressBook"] = _ADDRESSBOOK -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Person = _reflection.GeneratedProtocolMessageType( - "Person", - (_message.Message,), - dict( - PhoneNumber=_reflection.GeneratedProtocolMessageType( - "PhoneNumber", - (_message.Message,), - dict( - DESCRIPTOR=_PERSON_PHONENUMBER, - __module__="addressbook_pb2", - # @@protoc_insertion_point(class_scope:tutorial.Person.PhoneNumber) - ), - ), - DESCRIPTOR=_PERSON, - __module__="addressbook_pb2", - # @@protoc_insertion_point(class_scope:tutorial.Person) - ), -) -_sym_db.RegisterMessage(Person) -_sym_db.RegisterMessage(Person.PhoneNumber) - -AddressBook = _reflection.GeneratedProtocolMessageType( - "AddressBook", - (_message.Message,), - dict( - DESCRIPTOR=_ADDRESSBOOK, - __module__="addressbook_pb2", - # @@protoc_insertion_point(class_scope:tutorial.AddressBook) - ), -) -_sym_db.RegisterMessage(AddressBook) - - -# @@protoc_insertion_point(module_scope) diff --git a/benchmark/bin/README.md b/benchmark/bin/README.md deleted file mode 100644 index 6d8c6b9..0000000 --- a/benchmark/bin/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Benchmarks - -You'll need to install the protobuf dependencies if you want to profile against protobufs. - -```bash -pip install -r requirements.txt -``` - -To run all the benchmarks: - -```bash -./run_all -l pyproto -l pyproto_cpp -``` diff --git a/benchmark/bin/pycapnp-carsales b/benchmark/bin/pycapnp-carsales deleted file mode 100755 index 670720b..0000000 --- a/benchmark/bin/pycapnp-carsales +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='carsales', suffix='pycapnp', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pycapnp-catrank b/benchmark/bin/pycapnp-catrank deleted file mode 100755 index a11cb27..0000000 --- a/benchmark/bin/pycapnp-catrank +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='catrank', suffix='pycapnp', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pycapnp-eval b/benchmark/bin/pycapnp-eval deleted file mode 100755 index 8bb759a..0000000 --- a/benchmark/bin/pycapnp-eval +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='eval', suffix='pycapnp', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-carsales b/benchmark/bin/pyproto-carsales deleted file mode 100755 index 5fb8e54..0000000 --- a/benchmark/bin/pyproto-carsales +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='carsales', suffix='proto', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-catrank b/benchmark/bin/pyproto-catrank deleted file mode 100755 index e82456c..0000000 --- a/benchmark/bin/pyproto-catrank +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='catrank', suffix='proto', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pyproto-eval b/benchmark/bin/pyproto-eval deleted file mode 100755 index 270b615..0000000 --- a/benchmark/bin/pyproto-eval +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python - -from runner import parse_args_simple, run_test - -def main(): - args = parse_args_simple() - run_test(name='eval', suffix='proto', **vars(args)) - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/benchmark/bin/pyproto_cpp-carsales b/benchmark/bin/pyproto_cpp-carsales deleted file mode 100755 index c9a2788..0000000 --- a/benchmark/bin/pyproto_cpp-carsales +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -pyproto-carsales $@ diff --git a/benchmark/bin/pyproto_cpp-catrank b/benchmark/bin/pyproto_cpp-catrank deleted file mode 100755 index 5297843..0000000 --- a/benchmark/bin/pyproto_cpp-catrank +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -pyproto-catrank $@ diff --git a/benchmark/bin/pyproto_cpp-eval b/benchmark/bin/pyproto_cpp-eval deleted file mode 100755 index 88361fc..0000000 --- a/benchmark/bin/pyproto_cpp-eval +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -pyproto-eval $@ diff --git a/benchmark/bin/requirements.txt b/benchmark/bin/requirements.txt deleted file mode 100644 index b0c79cc..0000000 --- a/benchmark/bin/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -protobuf diff --git a/benchmark/bin/run_all.py b/benchmark/bin/run_all.py deleted file mode 100755 index 36a7930..0000000 --- a/benchmark/bin/run_all.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -from subprocess import Popen, PIPE -import sys -import os -import json -import argparse -import time - -_this_dir = os.path.dirname(__file__) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "-l", - "--langs", - help="Add languages to test, ie: -l pyproto -l pyproto_cpp", - action="append", - default=["pycapnp"], - ) - parser.add_argument( - "-r", - "--reuse", - help="If this flag is passed, re-use tests will be run", - action="store_true", - ) - parser.add_argument( - "-c", - "--compression", - help="If this flag is passed, compression tests will be run", - action="store_true", - ) - parser.add_argument( - "-i", - "--scale_iters", - help="Scaling factor to multiply the default iters by", - type=float, - default=1.0, - ) - - return parser.parse_args() - - -def run_one(prefix, name, mode, iters, faster, compression): - res_type = prefix - reuse = "no-reuse" - - if faster: - reuse = "reuse" - res_type += "_reuse" - if compression != "none": - res_type += "_" + compression - - command = [ - os.path.join(_this_dir, prefix + "-" + name), - mode, - reuse, - compression, - str(iters), - ] - start = time.time() - print("running: " + " ".join(command), file=sys.stderr) - p = Popen(command, stdout=PIPE, stderr=PIPE) - res = p.wait() - end = time.time() - - data = {} - - if p.returncode != 0: - sys.stderr.write( - " ".join(command) - + " failed to run with errors: \n" - + p.stderr.read().decode(sys.stdout.encoding) - + "\n" - ) - sys.stderr.flush() - - data["type"] = res_type - data["mode"] = mode - data["name"] = name - data["iters"] = iters - data["time"] = end - start - - return data - - -def run_each(name, langs, reuse, compression, iters): - ret = [] - - for lang_name in langs: - ret.append(run_one(lang_name, name, "object", iters, False, "none")) - ret.append(run_one(lang_name, name, "bytes", iters, False, "none")) - if reuse: - ret.append(run_one(lang_name, name, "object", iters, True, "none")) - ret.append(run_one(lang_name, name, "bytes", iters, True, "none")) - if compression: - ret.append(run_one(lang_name, name, "bytes", iters, True, "packed")) - if compression: - ret.append(run_one(lang_name, name, "bytes", iters, False, "packed")) - - return ret - - -def main(): - args = parse_args() - - os.environ["PATH"] += ":." - - data = [] - data += run_each( - "carsales", - args.langs, - args.reuse, - args.compression, - int(2000 * args.scale_iters), - ) - data += run_each( - "catrank", args.langs, args.reuse, args.compression, int(100 * args.scale_iters) - ) - data += run_each( - "eval", args.langs, args.reuse, args.compression, int(10000 * args.scale_iters) - ) - json.dump(data, sys.stdout, sort_keys=True, indent=4, separators=(",", ": ")) - - -if __name__ == "__main__": - main() diff --git a/benchmark/bin/runner.py b/benchmark/bin/runner.py deleted file mode 100755 index d4654b8..0000000 --- a/benchmark/bin/runner.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -import os -from importlib import import_module -from timeit import default_timer -import random - -_this_dir = os.path.dirname(__file__) -sys.path.append(os.path.join(_this_dir, "..")) -from common import do_benchmark - - -def parse_args_simple(): - parser = argparse.ArgumentParser() - parser.add_argument( - "mode", help="Mode to use for serialization, ie. object or bytes" - ) - parser.add_argument("reuse", help="Currently ignored") - parser.add_argument("compression", help="Valid values are none or packed") - parser.add_argument("iters", help="Number of iterations to run for", type=int) - parser.add_argument( - "-I", - "--includes", - help="Directories to add to PYTHONPATH", - default="/usr/local/include", - ) - - return parser.parse_args() - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "name", - help="Name of the benchmark to run, eg. carsales", - nargs="?", - default="carsales", - ) - parser.add_argument( - "-c", "--compression", help="Specify the compression type", default=None - ) - parser.add_argument( - "-s", "--suffix", help="Choose the protocol type.", default="pycapnp" - ) - parser.add_argument("-m", "--mode", help="Specify the mode", default="object") - parser.add_argument( - "-i", - "--iters", - help="Specify the number of iterations manually. By default, it will be looked up in preset table", - default=10, - type=int, - ) - parser.add_argument( - "-r", - "--reuse", - help="If this flag is passed, objects will be re-used", - action="store_true", - ) - parser.add_argument( - "-I", - "--includes", - help="Directories to add to PYTHONPATH", - default="/usr/local/include", - ) - - return parser.parse_args() - - -def run_test(name, mode, reuse, compression, iters, suffix, includes): - tic = default_timer() - - name = name - sys.path.append(includes) - module = import_module(name + "_" + suffix) - benchmark = module.Benchmark(compression=compression) - - do_benchmark(mode=mode, benchmark=benchmark, iters=iters, reuse=reuse) - toc = default_timer() - return toc - tic - - -def main(): - args = parse_args() - run_test(**vars(args)) - - -if __name__ == "__main__": - main() diff --git a/benchmark/carsales.capnp b/benchmark/carsales.capnp deleted file mode 100644 index e8e7fbb..0000000 --- a/benchmark/carsales.capnp +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -using Cxx = import "/capnp/c++.capnp"; - -@0xff75ddc6a36723c9; -$Cxx.namespace("capnp::benchmark::capnp"); - -struct ParkingLot { - cars@0: List(Car); -} - -struct TotalValue { - amount@0: UInt64; -} - -struct Car { - make@0: Text; - model@1: Text; - color@2: Color; - seats@3: UInt8; - doors@4: UInt8; - wheels@5: List(Wheel); - length@6: UInt16; - width@7: UInt16; - height@8: UInt16; - weight@9: UInt32; - engine@10: Engine; - fuelCapacity@11: Float32; - fuelLevel@12: Float32; - hasPowerWindows@13: Bool; - hasPowerSteering@14: Bool; - hasCruiseControl@15: Bool; - cupHolders@16: UInt8; - hasNavSystem@17: Bool; -} - -enum Color { - black @0; - white @1; - red @2; - green @3; - blue @4; - cyan @5; - magenta @6; - yellow @7; - silver @8; -} - -struct Wheel { - diameter@0: UInt16; - airPressure@1: Float32; - snowTires@2: Bool; -} - -struct Engine { - horsepower@0: UInt16; - cylinders@1: UInt8; - cc@2: UInt32; - usesGas@3: Bool; - usesElectric@4: Bool; -} diff --git a/benchmark/carsales.proto b/benchmark/carsales.proto deleted file mode 100644 index 0b6a975..0000000 --- a/benchmark/carsales.proto +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2013, Kenton Varda -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package capnp.benchmark.protobuf; - -message ParkingLot { - repeated Car car = 1; -} - -message TotalValue { - required uint64 amount = 1; -} - -message Car { - optional string make = 1; - optional string model = 2; - optional Color color = 3; - optional uint32 seats = 4; - optional uint32 doors = 5; - repeated Wheel wheel = 6; - optional uint32 length = 7; - optional uint32 width = 8; - optional uint32 height = 9; - optional uint32 weight = 10; - optional Engine engine = 11; - optional float fuel_capacity = 12; - optional float fuel_level = 13; - optional bool has_power_windows = 14; - optional bool has_power_steering = 15; - optional bool has_cruise_control = 16; - optional uint32 cup_holders = 17; - optional bool has_nav_system = 18; -} - -enum Color { - BLACK = 0; - WHITE = 1; - RED = 2; - GREEN = 3; - BLUE = 4; - CYAN = 5; - MAGENTA = 6; - YELLOW = 7; - SILVER = 8; -} - -message Wheel { - optional uint32 diameter = 1; - optional float air_pressure = 2; - optional bool snow_tires = 3; -} - -message Engine { - optional uint32 horsepower = 1; - optional uint32 cylinders = 2; - optional uint32 cc = 3; - optional bool uses_gas = 4; - optional bool uses_electric = 5; -} diff --git a/benchmark/carsales_pb2.py b/benchmark/carsales_pb2.py deleted file mode 100644 index 3e9dd5d..0000000 --- a/benchmark/carsales_pb2.py +++ /dev/null @@ -1,729 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: carsales.proto - -import sys - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="carsales.proto", - package="capnp.benchmark.protobuf", - syntax="proto2", - serialized_pb=_b( - '\n\x0e\x63\x61rsales.proto\x12\x18\x63\x61pnp.benchmark.protobuf"8\n\nParkingLot\x12*\n\x03\x63\x61r\x18\x01 \x03(\x0b\x32\x1d.capnp.benchmark.protobuf.Car"\x1c\n\nTotalValue\x12\x0e\n\x06\x61mount\x18\x01 \x02(\x04"\xbc\x03\n\x03\x43\x61r\x12\x0c\n\x04make\x18\x01 \x01(\t\x12\r\n\x05model\x18\x02 \x01(\t\x12.\n\x05\x63olor\x18\x03 \x01(\x0e\x32\x1f.capnp.benchmark.protobuf.Color\x12\r\n\x05seats\x18\x04 \x01(\r\x12\r\n\x05\x64oors\x18\x05 \x01(\r\x12.\n\x05wheel\x18\x06 \x03(\x0b\x32\x1f.capnp.benchmark.protobuf.Wheel\x12\x0e\n\x06length\x18\x07 \x01(\r\x12\r\n\x05width\x18\x08 \x01(\r\x12\x0e\n\x06height\x18\t \x01(\r\x12\x0e\n\x06weight\x18\n \x01(\r\x12\x30\n\x06\x65ngine\x18\x0b \x01(\x0b\x32 .capnp.benchmark.protobuf.Engine\x12\x15\n\rfuel_capacity\x18\x0c \x01(\x02\x12\x12\n\nfuel_level\x18\r \x01(\x02\x12\x19\n\x11has_power_windows\x18\x0e \x01(\x08\x12\x1a\n\x12has_power_steering\x18\x0f \x01(\x08\x12\x1a\n\x12has_cruise_control\x18\x10 \x01(\x08\x12\x13\n\x0b\x63up_holders\x18\x11 \x01(\r\x12\x16\n\x0ehas_nav_system\x18\x12 \x01(\x08"C\n\x05Wheel\x12\x10\n\x08\x64iameter\x18\x01 \x01(\r\x12\x14\n\x0c\x61ir_pressure\x18\x02 \x01(\x02\x12\x12\n\nsnow_tires\x18\x03 \x01(\x08"d\n\x06\x45ngine\x12\x12\n\nhorsepower\x18\x01 \x01(\r\x12\x11\n\tcylinders\x18\x02 \x01(\r\x12\n\n\x02\x63\x63\x18\x03 \x01(\r\x12\x10\n\x08uses_gas\x18\x04 \x01(\x08\x12\x15\n\ruses_electric\x18\x05 \x01(\x08*j\n\x05\x43olor\x12\t\n\x05\x42LACK\x10\x00\x12\t\n\x05WHITE\x10\x01\x12\x07\n\x03RED\x10\x02\x12\t\n\x05GREEN\x10\x03\x12\x08\n\x04\x42LUE\x10\x04\x12\x08\n\x04\x43YAN\x10\x05\x12\x0b\n\x07MAGENTA\x10\x06\x12\n\n\x06YELLOW\x10\x07\x12\n\n\x06SILVER\x10\x08' - ), -) - -_COLOR = _descriptor.EnumDescriptor( - name="Color", - full_name="capnp.benchmark.protobuf.Color", - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name="BLACK", index=0, number=0, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="WHITE", index=1, number=1, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="RED", index=2, number=2, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="GREEN", index=3, number=3, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="BLUE", index=4, number=4, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="CYAN", index=5, number=5, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="MAGENTA", index=6, number=6, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="YELLOW", index=7, number=7, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="SILVER", index=8, number=8, options=None, type=None - ), - ], - containing_type=None, - options=None, - serialized_start=750, - serialized_end=856, -) -_sym_db.RegisterEnumDescriptor(_COLOR) - -Color = enum_type_wrapper.EnumTypeWrapper(_COLOR) -BLACK = 0 -WHITE = 1 -RED = 2 -GREEN = 3 -BLUE = 4 -CYAN = 5 -MAGENTA = 6 -YELLOW = 7 -SILVER = 8 - - -_PARKINGLOT = _descriptor.Descriptor( - name="ParkingLot", - full_name="capnp.benchmark.protobuf.ParkingLot", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="car", - full_name="capnp.benchmark.protobuf.ParkingLot.car", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=44, - serialized_end=100, -) - - -_TOTALVALUE = _descriptor.Descriptor( - name="TotalValue", - full_name="capnp.benchmark.protobuf.TotalValue", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="amount", - full_name="capnp.benchmark.protobuf.TotalValue.amount", - index=0, - number=1, - type=4, - cpp_type=4, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=102, - serialized_end=130, -) - - -_CAR = _descriptor.Descriptor( - name="Car", - full_name="capnp.benchmark.protobuf.Car", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="make", - full_name="capnp.benchmark.protobuf.Car.make", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="model", - full_name="capnp.benchmark.protobuf.Car.model", - index=1, - number=2, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="color", - full_name="capnp.benchmark.protobuf.Car.color", - index=2, - number=3, - type=14, - cpp_type=8, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="seats", - full_name="capnp.benchmark.protobuf.Car.seats", - index=3, - number=4, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="doors", - full_name="capnp.benchmark.protobuf.Car.doors", - index=4, - number=5, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="wheel", - full_name="capnp.benchmark.protobuf.Car.wheel", - index=5, - number=6, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="length", - full_name="capnp.benchmark.protobuf.Car.length", - index=6, - number=7, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="width", - full_name="capnp.benchmark.protobuf.Car.width", - index=7, - number=8, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="height", - full_name="capnp.benchmark.protobuf.Car.height", - index=8, - number=9, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="weight", - full_name="capnp.benchmark.protobuf.Car.weight", - index=9, - number=10, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="engine", - full_name="capnp.benchmark.protobuf.Car.engine", - index=10, - number=11, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="fuel_capacity", - full_name="capnp.benchmark.protobuf.Car.fuel_capacity", - index=11, - number=12, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="fuel_level", - full_name="capnp.benchmark.protobuf.Car.fuel_level", - index=12, - number=13, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="has_power_windows", - full_name="capnp.benchmark.protobuf.Car.has_power_windows", - index=13, - number=14, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="has_power_steering", - full_name="capnp.benchmark.protobuf.Car.has_power_steering", - index=14, - number=15, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="has_cruise_control", - full_name="capnp.benchmark.protobuf.Car.has_cruise_control", - index=15, - number=16, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="cup_holders", - full_name="capnp.benchmark.protobuf.Car.cup_holders", - index=16, - number=17, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="has_nav_system", - full_name="capnp.benchmark.protobuf.Car.has_nav_system", - index=17, - number=18, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=133, - serialized_end=577, -) - - -_WHEEL = _descriptor.Descriptor( - name="Wheel", - full_name="capnp.benchmark.protobuf.Wheel", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="diameter", - full_name="capnp.benchmark.protobuf.Wheel.diameter", - index=0, - number=1, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="air_pressure", - full_name="capnp.benchmark.protobuf.Wheel.air_pressure", - index=1, - number=2, - type=2, - cpp_type=6, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="snow_tires", - full_name="capnp.benchmark.protobuf.Wheel.snow_tires", - index=2, - number=3, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=579, - serialized_end=646, -) - - -_ENGINE = _descriptor.Descriptor( - name="Engine", - full_name="capnp.benchmark.protobuf.Engine", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="horsepower", - full_name="capnp.benchmark.protobuf.Engine.horsepower", - index=0, - number=1, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="cylinders", - full_name="capnp.benchmark.protobuf.Engine.cylinders", - index=1, - number=2, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="cc", - full_name="capnp.benchmark.protobuf.Engine.cc", - index=2, - number=3, - type=13, - cpp_type=3, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="uses_gas", - full_name="capnp.benchmark.protobuf.Engine.uses_gas", - index=3, - number=4, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="uses_electric", - full_name="capnp.benchmark.protobuf.Engine.uses_electric", - index=4, - number=5, - type=8, - cpp_type=7, - label=1, - has_default_value=False, - default_value=False, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=648, - serialized_end=748, -) - -_PARKINGLOT.fields_by_name["car"].message_type = _CAR -_CAR.fields_by_name["color"].enum_type = _COLOR -_CAR.fields_by_name["wheel"].message_type = _WHEEL -_CAR.fields_by_name["engine"].message_type = _ENGINE -DESCRIPTOR.message_types_by_name["ParkingLot"] = _PARKINGLOT -DESCRIPTOR.message_types_by_name["TotalValue"] = _TOTALVALUE -DESCRIPTOR.message_types_by_name["Car"] = _CAR -DESCRIPTOR.message_types_by_name["Wheel"] = _WHEEL -DESCRIPTOR.message_types_by_name["Engine"] = _ENGINE -DESCRIPTOR.enum_types_by_name["Color"] = _COLOR -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -ParkingLot = _reflection.GeneratedProtocolMessageType( - "ParkingLot", - (_message.Message,), - dict( - DESCRIPTOR=_PARKINGLOT, - __module__="carsales_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.ParkingLot) - ), -) -_sym_db.RegisterMessage(ParkingLot) - -TotalValue = _reflection.GeneratedProtocolMessageType( - "TotalValue", - (_message.Message,), - dict( - DESCRIPTOR=_TOTALVALUE, - __module__="carsales_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.TotalValue) - ), -) -_sym_db.RegisterMessage(TotalValue) - -Car = _reflection.GeneratedProtocolMessageType( - "Car", - (_message.Message,), - dict( - DESCRIPTOR=_CAR, - __module__="carsales_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Car) - ), -) -_sym_db.RegisterMessage(Car) - -Wheel = _reflection.GeneratedProtocolMessageType( - "Wheel", - (_message.Message,), - dict( - DESCRIPTOR=_WHEEL, - __module__="carsales_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Wheel) - ), -) -_sym_db.RegisterMessage(Wheel) - -Engine = _reflection.GeneratedProtocolMessageType( - "Engine", - (_message.Message,), - dict( - DESCRIPTOR=_ENGINE, - __module__="carsales_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Engine) - ), -) -_sym_db.RegisterMessage(Engine) - - -# @@protoc_insertion_point(module_scope) diff --git a/benchmark/carsales_proto.py b/benchmark/carsales_proto.py deleted file mode 100755 index e98939d..0000000 --- a/benchmark/carsales_proto.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -import carsales_pb2 -from common import rand_int, rand_double, rand_bool, from_bytes_helper -from random import choice - -MAKES = ["Toyota", "GM", "Ford", "Honda", "Tesla"] -MODELS = ["Camry", "Prius", "Volt", "Accord", "Leaf", "Model S"] -COLORS = [ - "black", - "white", - "red", - "green", - "blue", - "cyan", - "magenta", - "yellow", - "silver", -] - - -def random_car(car): - car.make = choice(MAKES) - car.model = choice(MODELS) - car.color = rand_int(len(COLORS)) - - car.seats = 2 + rand_int(6) - car.doors = 2 + rand_int(3) - - for _ in range(4): - wheel = car.wheel.add() - wheel.diameter = 25 + rand_int(15) - wheel.air_pressure = 30 + rand_double(20) - wheel.snow_tires = rand_int(16) == 0 - - car.length = 170 + rand_int(150) - car.width = 48 + rand_int(36) - car.height = 54 + rand_int(48) - car.weight = car.length * car.width * car.height // 200 - - engine = car.engine - engine.horsepower = 100 * rand_int(400) - engine.cylinders = 4 + 2 * rand_int(3) - engine.cc = 800 + rand_int(10000) - engine.uses_gas = True - engine.uses_electric = rand_bool() - - car.fuel_capacity = 10.0 + rand_double(30.0) - car.fuel_level = rand_double(car.fuel_capacity) - car.has_power_windows = rand_bool() - car.has_power_steering = rand_bool() - car.has_cruise_control = rand_bool() - car.cup_holders = rand_int(12) - car.has_nav_system = rand_bool() - - -def calc_value(car): - result = 0 - - result += car.seats * 200 - result += car.doors * 350 - for wheel in car.wheel: - result += wheel.diameter * wheel.diameter - result += 100 if wheel.snow_tires else 0 - - result += car.length * car.width * car.height // 50 - - engine = car.engine - result += engine.horsepower * 40 - if engine.uses_electric: - if engine.uses_gas: - result += 5000 - else: - result += 3000 - - result += 100 if car.has_power_windows else 0 - result += 200 if car.has_power_steering else 0 - result += 400 if car.has_cruise_control else 0 - result += 2000 if car.has_nav_system else 0 - - result += car.cup_holders * 25 - - return result - - -class Benchmark: - def __init__(self, compression): - self.Request = carsales_pb2.ParkingLot - self.Response = carsales_pb2.TotalValue - self.from_bytes_request = from_bytes_helper(carsales_pb2.ParkingLot) - self.from_bytes_response = from_bytes_helper(carsales_pb2.TotalValue) - self.to_bytes = lambda x: x.SerializeToString() - - def setup(self, request): - result = 0 - for _ in range(rand_int(200)): - car = request.car.add() - random_car(car) - result += calc_value(car) - return result - - def handle(self, request, response): - result = 0 - for car in request.car: - result += calc_value(car) - - response.amount = result - - def check(self, response, expected): - return response.amount == expected diff --git a/benchmark/carsales_pycapnp.py b/benchmark/carsales_pycapnp.py deleted file mode 100755 index bd724d7..0000000 --- a/benchmark/carsales_pycapnp.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python - -import capnp -import carsales_capnp -from common import rand_int, rand_double, rand_bool -from random import choice - -MAKES = ["Toyota", "GM", "Ford", "Honda", "Tesla"] -MODELS = ["Camry", "Prius", "Volt", "Accord", "Leaf", "Model S"] -COLORS = [ - "black", - "white", - "red", - "green", - "blue", - "cyan", - "magenta", - "yellow", - "silver", -] - - -def random_car(car): - car.make = choice(MAKES) - car.model = choice(MODELS) - car.color = choice(COLORS) - - car.seats = 2 + rand_int(6) - car.doors = 2 + rand_int(3) - - for wheel in car.init("wheels", 4): - wheel.diameter = 25 + rand_int(15) - wheel.airPressure = 30 + rand_double(20) - wheel.snowTires = rand_int(16) == 0 - - car.length = 170 + rand_int(150) - car.width = 48 + rand_int(36) - car.height = 54 + rand_int(48) - car.weight = car.length * car.width * car.height // 200 - - engine = car.init("engine") - engine.horsepower = 100 * rand_int(400) - engine.cylinders = 4 + 2 * rand_int(3) - engine.cc = 800 + rand_int(10000) - engine.usesGas = True - engine.usesElectric = rand_bool() - - car.fuelCapacity = 10.0 + rand_double(30.0) - car.fuelLevel = rand_double(car.fuelCapacity) - car.hasPowerWindows = rand_bool() - car.hasPowerSteering = rand_bool() - car.hasCruiseControl = rand_bool() - car.cupHolders = rand_int(12) - car.hasNavSystem = rand_bool() - - -def calc_value(car): - result = 0 - - result += car.seats * 200 - result += car.doors * 350 - for wheel in car.wheels: - result += wheel.diameter * wheel.diameter - result += 100 if wheel.snowTires else 0 - - result += car.length * car.width * car.height // 50 - - engine = car.engine - result += engine.horsepower * 40 - if engine.usesElectric: - if engine.usesGas: - result += 5000 - else: - result += 3000 - - result += 100 if car.hasPowerWindows else 0 - result += 200 if car.hasPowerSteering else 0 - result += 400 if car.hasCruiseControl else 0 - result += 2000 if car.hasNavSystem else 0 - - result += car.cupHolders * 25 - - return result - - -class Benchmark: - def __init__(self, compression): - self.Request = carsales_capnp.ParkingLot.new_message - self.Response = carsales_capnp.TotalValue.new_message - if compression == "packed": - self.from_bytes_request = carsales_capnp.ParkingLot.from_bytes_packed - self.from_bytes_response = carsales_capnp.TotalValue.from_bytes_packed - self.to_bytes = lambda x: x.to_bytes_packed() - else: - self.from_bytes_request = carsales_capnp.ParkingLot.from_bytes - self.from_bytes_response = carsales_capnp.TotalValue.from_bytes - self.to_bytes = lambda x: x.to_bytes() - - def setup(self, request): - result = 0 - for car in request.init("cars", rand_int(200)): - random_car(car) - result += calc_value(car) - return result - - def handle(self, request, response): - result = 0 - for car in request.cars: - result += calc_value(car) - - response.amount = result - - def check(self, response, expected): - return response.amount == expected diff --git a/benchmark/catrank.capnp b/benchmark/catrank.capnp deleted file mode 100644 index 15fda47..0000000 --- a/benchmark/catrank.capnp +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -using Cxx = import "/capnp/c++.capnp"; - -@0x82beb8e37ff79aba; -$Cxx.namespace("capnp::benchmark::capnp"); - -struct SearchResultList { - results@0: List(SearchResult); -} - -struct SearchResult { - url@0: Text; - score@1: Float64; - snippet@2: Text; -} diff --git a/benchmark/catrank.proto b/benchmark/catrank.proto deleted file mode 100644 index c3f55ae..0000000 --- a/benchmark/catrank.proto +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) 2013, Kenton Varda -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package capnp.benchmark.protobuf; - -message SearchResultList { - repeated SearchResult result = 1; -} - -message SearchResult { - optional string url = 1; - optional double score = 2; - optional string snippet = 3; -} diff --git a/benchmark/catrank_pb2.py b/benchmark/catrank_pb2.py deleted file mode 100644 index 86d1b05..0000000 --- a/benchmark/catrank_pb2.py +++ /dev/null @@ -1,165 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: catrank.proto - -import sys - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="catrank.proto", - package="capnp.benchmark.protobuf", - syntax="proto2", - serialized_pb=_b( - '\n\rcatrank.proto\x12\x18\x63\x61pnp.benchmark.protobuf"J\n\x10SearchResultList\x12\x36\n\x06result\x18\x01 \x03(\x0b\x32&.capnp.benchmark.protobuf.SearchResult";\n\x0cSearchResult\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x01\x12\x0f\n\x07snippet\x18\x03 \x01(\t' - ), -) - - -_SEARCHRESULTLIST = _descriptor.Descriptor( - name="SearchResultList", - full_name="capnp.benchmark.protobuf.SearchResultList", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="result", - full_name="capnp.benchmark.protobuf.SearchResultList.result", - index=0, - number=1, - type=11, - cpp_type=10, - label=3, - has_default_value=False, - default_value=[], - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=43, - serialized_end=117, -) - - -_SEARCHRESULT = _descriptor.Descriptor( - name="SearchResult", - full_name="capnp.benchmark.protobuf.SearchResult", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="url", - full_name="capnp.benchmark.protobuf.SearchResult.url", - index=0, - number=1, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="score", - full_name="capnp.benchmark.protobuf.SearchResult.score", - index=1, - number=2, - type=1, - cpp_type=5, - label=1, - has_default_value=False, - default_value=float(0), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="snippet", - full_name="capnp.benchmark.protobuf.SearchResult.snippet", - index=2, - number=3, - type=9, - cpp_type=9, - label=1, - has_default_value=False, - default_value=_b("").decode("utf-8"), - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=119, - serialized_end=178, -) - -_SEARCHRESULTLIST.fields_by_name["result"].message_type = _SEARCHRESULT -DESCRIPTOR.message_types_by_name["SearchResultList"] = _SEARCHRESULTLIST -DESCRIPTOR.message_types_by_name["SearchResult"] = _SEARCHRESULT -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -SearchResultList = _reflection.GeneratedProtocolMessageType( - "SearchResultList", - (_message.Message,), - dict( - DESCRIPTOR=_SEARCHRESULTLIST, - __module__="catrank_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.SearchResultList) - ), -) -_sym_db.RegisterMessage(SearchResultList) - -SearchResult = _reflection.GeneratedProtocolMessageType( - "SearchResult", - (_message.Message,), - dict( - DESCRIPTOR=_SEARCHRESULT, - __module__="catrank_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.SearchResult) - ), -) -_sym_db.RegisterMessage(SearchResult) - - -# @@protoc_insertion_point(module_scope) diff --git a/benchmark/catrank_proto.py b/benchmark/catrank_proto.py deleted file mode 100755 index 886139f..0000000 --- a/benchmark/catrank_proto.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python - -from common import rand_int, rand_double, rand_bool, WORDS, from_bytes_helper -from random import choice -from string import ascii_letters - -try: - # Python 2 - from itertools import izip -except ImportError: - izip = zip -import catrank_pb2 - - -class Benchmark: - def __init__(self, compression): - self.Request = catrank_pb2.SearchResultList - self.Response = catrank_pb2.SearchResultList - self.from_bytes_request = from_bytes_helper(catrank_pb2.SearchResultList) - self.from_bytes_response = from_bytes_helper(catrank_pb2.SearchResultList) - self.to_bytes = lambda x: x.SerializeToString() - - def setup(self, request): - goodCount = 0 - count = rand_int(1000) - - for i in range(count): - result = request.result.add() - result.score = 1000 - i - url_size = rand_int(100) - result.url = "http://example.com/" + "".join( - [choice(ascii_letters) for _ in range(url_size)] - ) - - isCat = rand_bool() - isDog = rand_bool() - if isCat and not isDog: - goodCount += 1 - - snippet = [choice(WORDS) for i in range(rand_int(20))] - - if isCat: - snippet.append(" cat ") - if isDog: - snippet.append(" dog ") - - snippet += [choice(WORDS) for i in range(rand_int(20))] - - result.snippet = "".join(snippet) - - return goodCount - - def handle(self, request, response): - for req in request.result: - resp = response.result.add() - score = req.score - - if " cat " in req.snippet: - score *= 10000 - if " dog " in req.snippet: - score /= 10000 - - resp.score = score - resp.url = req.url - resp.snippet = req.snippet - - def check(self, response, expected): - goodCount = 0 - - for result in response.result: - if result.score > 1001: - goodCount += 1 - - return goodCount == expected diff --git a/benchmark/catrank_pycapnp.py b/benchmark/catrank_pycapnp.py deleted file mode 100755 index e1e9365..0000000 --- a/benchmark/catrank_pycapnp.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -import capnp -import catrank_capnp -from common import rand_int, rand_double, rand_bool, WORDS -from random import choice -from string import ascii_letters - -try: - # Python 2 - from itertools import izip -except ImportError: - izip = zip - - -class Benchmark: - def __init__(self, compression): - self.Request = catrank_capnp.SearchResultList.new_message - self.Response = catrank_capnp.SearchResultList.new_message - if compression == "packed": - self.from_bytes_request = catrank_capnp.SearchResultList.from_bytes_packed - self.from_bytes_response = catrank_capnp.SearchResultList.from_bytes_packed - self.to_bytes = lambda x: x.to_bytes_packed() - else: - self.from_bytes_request = catrank_capnp.SearchResultList.from_bytes - self.from_bytes_response = catrank_capnp.SearchResultList.from_bytes - self.to_bytes = lambda x: x.to_bytes() - - def setup(self, request): - goodCount = 0 - count = rand_int(1000) - - results = request.init("results", count) - - for i, result in enumerate(results): - result.score = 1000 - i - url_size = rand_int(100) - result.url = "http://example.com/" + "".join( - [choice(ascii_letters) for _ in range(url_size)] - ) - - isCat = rand_bool() - isDog = rand_bool() - if isCat and not isDog: - goodCount += 1 - - snippet = [choice(WORDS) for i in range(rand_int(20))] - - if isCat: - snippet.append(" cat ") - if isDog: - snippet.append(" dog ") - - snippet += [choice(WORDS) for i in range(rand_int(20))] - - result.snippet = "".join(snippet) - - return goodCount - - def handle(self, request, response): - results = response.init("results", len(request.results)) - - for req, resp in izip(request.results, results): - score = req.score - - if " cat " in req.snippet: - score *= 10000 - if " dog " in req.snippet: - score /= 10000 - - resp.score = score - resp.url = req.url - resp.snippet = req.snippet - - def check(self, response, expected): - goodCount = 0 - - for result in response.results: - if result.score > 1001: - goodCount += 1 - - return goodCount == expected diff --git a/benchmark/common.py b/benchmark/common.py deleted file mode 100644 index 1ecd76a..0000000 --- a/benchmark/common.py +++ /dev/null @@ -1,94 +0,0 @@ -from random import random -import pyximport - -importers = pyximport.install() -from common_fast import rand_int, rand_double, rand_bool - -pyximport.uninstall(*importers) - -WORDS = [ - "foo ", - "bar ", - "baz ", - "qux ", - "quux ", - "corge ", - "grault ", - "garply ", - "waldo ", - "fred ", - "plugh ", - "xyzzy ", - "thud ", -] - - -def from_bytes_helper(klass): - def helper(text): - obj = klass() - obj.ParseFromString(text) - return obj - - return helper - - -def pass_by_object(reuse, iters, benchmark): - for _ in range(iters): - request = benchmark.Request() - expected = benchmark.setup(request) - - response = benchmark.Response() - benchmark.handle(request, response) - - if not benchmark.check(response, expected): - raise ValueError("Expected {}".format(expected)) - - -def pass_by_bytes(reuse, iters, benchmark): - for _ in range(iters): - request = benchmark.Request() - expected = benchmark.setup(request) - req_bytes = benchmark.to_bytes(request) - - request2 = benchmark.from_bytes_request(req_bytes) - response = benchmark.Response() - benchmark.handle(request2, response) - resp_bytes = benchmark.to_bytes(response) - - response2 = benchmark.from_bytes_response(resp_bytes) - if not benchmark.check(response2, expected): - raise ValueError("Expected {}".format(expected)) - - -def do_benchmark(mode, *args, **kwargs): - if mode == "client": - pass - elif mode == "object": - return pass_by_object(*args, **kwargs) - elif mode == "bytes": - return pass_by_bytes(*args, **kwargs) - else: - raise ValueError("Unknown mode: " + str(mode)) - - -# typedef typename BenchmarkTypes::template BenchmarkMethods -# BenchmarkMethods; -# if (mode == "client") { -# return BenchmarkMethods::syncClient(STDIN_FILENO, STDOUT_FILENO, iters); -# } else if (mode == "server") { -# return BenchmarkMethods::server(STDIN_FILENO, STDOUT_FILENO, iters); -# } else if (mode == "object") { -# return BenchmarkMethods::passByObject(iters, false); -# } else if (mode == "object-size") { -# return BenchmarkMethods::passByObject(iters, true); -# } else if (mode == "bytes") { -# return BenchmarkMethods::passByBytes(iters); -# } else if (mode == "pipe") { -# return passByPipe(BenchmarkMethods::syncClient, iters); -# } else if (mode == "pipe-async") { -# return passByPipe(BenchmarkMethods::asyncClient, iters); -# } else { -# fprintf(stderr, "Unknown mode: %s\n", mode.c_str()); -# exit(1); -# } -# } diff --git a/benchmark/common_fast.pyx b/benchmark/common_fast.pyx deleted file mode 100644 index 8185741..0000000 --- a/benchmark/common_fast.pyx +++ /dev/null @@ -1,20 +0,0 @@ -from libc.stdint cimport * - -cdef uint32_t A = 1664525 -cdef uint32_t C = 1013904223 -cdef uint32_t state = C -cdef int32_t MAX_INT = 2**31 - 1 - -cpdef uint32_t nextFastRand(): - global state - state = A * state + C - return state - -cpdef uint32_t rand_int(uint32_t range): - return nextFastRand() % range - -cpdef double rand_double(double range): - return nextFastRand() * range / MAX_INT - -cpdef bint rand_bool(): - return nextFastRand() % 2 diff --git a/benchmark/eval.capnp b/benchmark/eval.capnp deleted file mode 100644 index 8e008ec..0000000 --- a/benchmark/eval.capnp +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -using Cxx = import "/capnp/c++.capnp"; - -@0xe12dc4c3e70e9eda; -$Cxx.namespace("capnp::benchmark::capnp"); - -enum Operation { - add @0; - subtract @1; - multiply @2; - divide @3; - modulus @4; -} - -struct Expression { - op@0: Operation; - - left :union { - value@1: Int32; - expression@2: Expression; - } - - right :union { - value@3: Int32; - expression@4: Expression; - } -} - -struct EvaluationResult { - value@0: Int32; -} diff --git a/benchmark/eval.proto b/benchmark/eval.proto deleted file mode 100644 index 204a081..0000000 --- a/benchmark/eval.proto +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2013, Kenton Varda -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; - -package capnp.benchmark.protobuf; - -enum Operation { - ADD = 0; - SUBTRACT = 1; - MULTIPLY = 2; - DIVIDE = 3; - MODULUS = 4; -} - -message Expression { - required Operation op = 1; - - optional int32 left_value = 2; - optional Expression left_expression = 3; - - optional int32 right_value = 4; - optional Expression right_expression = 5; -} - -message EvaluationResult { - required sint32 value = 1; -} diff --git a/benchmark/eval_pb2.py b/benchmark/eval_pb2.py deleted file mode 100644 index d1f31d3..0000000 --- a/benchmark/eval_pb2.py +++ /dev/null @@ -1,239 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: eval.proto - -import sys - -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor.FileDescriptor( - name="eval.proto", - package="capnp.benchmark.protobuf", - syntax="proto2", - serialized_pb=_b( - '\n\neval.proto\x12\x18\x63\x61pnp.benchmark.protobuf"\xe5\x01\n\nExpression\x12/\n\x02op\x18\x01 \x02(\x0e\x32#.capnp.benchmark.protobuf.Operation\x12\x12\n\nleft_value\x18\x02 \x01(\x05\x12=\n\x0fleft_expression\x18\x03 \x01(\x0b\x32$.capnp.benchmark.protobuf.Expression\x12\x13\n\x0bright_value\x18\x04 \x01(\x05\x12>\n\x10right_expression\x18\x05 \x01(\x0b\x32$.capnp.benchmark.protobuf.Expression"!\n\x10\x45valuationResult\x12\r\n\x05value\x18\x01 \x02(\x11*I\n\tOperation\x12\x07\n\x03\x41\x44\x44\x10\x00\x12\x0c\n\x08SUBTRACT\x10\x01\x12\x0c\n\x08MULTIPLY\x10\x02\x12\n\n\x06\x44IVIDE\x10\x03\x12\x0b\n\x07MODULUS\x10\x04' - ), -) - -_OPERATION = _descriptor.EnumDescriptor( - name="Operation", - full_name="capnp.benchmark.protobuf.Operation", - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name="ADD", index=0, number=0, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="SUBTRACT", index=1, number=1, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="MULTIPLY", index=2, number=2, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="DIVIDE", index=3, number=3, options=None, type=None - ), - _descriptor.EnumValueDescriptor( - name="MODULUS", index=4, number=4, options=None, type=None - ), - ], - containing_type=None, - options=None, - serialized_start=307, - serialized_end=380, -) -_sym_db.RegisterEnumDescriptor(_OPERATION) - -Operation = enum_type_wrapper.EnumTypeWrapper(_OPERATION) -ADD = 0 -SUBTRACT = 1 -MULTIPLY = 2 -DIVIDE = 3 -MODULUS = 4 - - -_EXPRESSION = _descriptor.Descriptor( - name="Expression", - full_name="capnp.benchmark.protobuf.Expression", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="op", - full_name="capnp.benchmark.protobuf.Expression.op", - index=0, - number=1, - type=14, - cpp_type=8, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="left_value", - full_name="capnp.benchmark.protobuf.Expression.left_value", - index=1, - number=2, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="left_expression", - full_name="capnp.benchmark.protobuf.Expression.left_expression", - index=2, - number=3, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="right_value", - full_name="capnp.benchmark.protobuf.Expression.right_value", - index=3, - number=4, - type=5, - cpp_type=1, - label=1, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - _descriptor.FieldDescriptor( - name="right_expression", - full_name="capnp.benchmark.protobuf.Expression.right_expression", - index=4, - number=5, - type=11, - cpp_type=10, - label=1, - has_default_value=False, - default_value=None, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=41, - serialized_end=270, -) - - -_EVALUATIONRESULT = _descriptor.Descriptor( - name="EvaluationResult", - full_name="capnp.benchmark.protobuf.EvaluationResult", - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name="value", - full_name="capnp.benchmark.protobuf.EvaluationResult.value", - index=0, - number=1, - type=17, - cpp_type=1, - label=2, - has_default_value=False, - default_value=0, - message_type=None, - enum_type=None, - containing_type=None, - is_extension=False, - extension_scope=None, - options=None, - ), - ], - extensions=[], - nested_types=[], - enum_types=[], - options=None, - is_extendable=False, - syntax="proto2", - extension_ranges=[], - oneofs=[], - serialized_start=272, - serialized_end=305, -) - -_EXPRESSION.fields_by_name["op"].enum_type = _OPERATION -_EXPRESSION.fields_by_name["left_expression"].message_type = _EXPRESSION -_EXPRESSION.fields_by_name["right_expression"].message_type = _EXPRESSION -DESCRIPTOR.message_types_by_name["Expression"] = _EXPRESSION -DESCRIPTOR.message_types_by_name["EvaluationResult"] = _EVALUATIONRESULT -DESCRIPTOR.enum_types_by_name["Operation"] = _OPERATION -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - -Expression = _reflection.GeneratedProtocolMessageType( - "Expression", - (_message.Message,), - dict( - DESCRIPTOR=_EXPRESSION, - __module__="eval_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.Expression) - ), -) -_sym_db.RegisterMessage(Expression) - -EvaluationResult = _reflection.GeneratedProtocolMessageType( - "EvaluationResult", - (_message.Message,), - dict( - DESCRIPTOR=_EVALUATIONRESULT, - __module__="eval_pb2", - # @@protoc_insertion_point(class_scope:capnp.benchmark.protobuf.EvaluationResult) - ), -) -_sym_db.RegisterMessage(EvaluationResult) - - -# @@protoc_insertion_point(module_scope) diff --git a/benchmark/eval_proto.py b/benchmark/eval_proto.py deleted file mode 100755 index cfea137..0000000 --- a/benchmark/eval_proto.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python - -from common import rand_int, rand_double, rand_bool, from_bytes_helper -from random import choice -import eval_pb2 - -MAX_INT = 2**31 - 1 -MIN_INT = -(2**31) - -OPERATIONS = ["add", "subtract", "multiply", "divide", "modulus"] - - -def clamp(res): - if res > MAX_INT: - return MAX_INT - elif res < MIN_INT: - return MIN_INT - else: - return res - - -def div(a, b): - if b == 0: - return MAX_INT - if a == MIN_INT and b == -1: - return MAX_INT - - return a // b - - -def mod(a, b): - if b == 0: - return MAX_INT - if a == MIN_INT and b == -1: - return MAX_INT - - return a % b - - -def make_expression(exp, depth): - exp.op = rand_int(len(OPERATIONS)) - - if rand_int(8) < depth: - left = rand_int(128) + 1 - exp.left_value = left - else: - left = make_expression(exp.left_expression, depth + 1) - - if rand_int(8) < depth: - right = rand_int(128) + 1 - exp.right_value = right - else: - right = make_expression(exp.right_expression, depth + 1) - - op = exp.op - if op == 0: - return clamp(left + right) - elif op == 1: - return clamp(left - right) - elif op == 2: - return clamp(left * right) - elif op == 3: - return div(left, right) - elif op == 4: - return mod(left, right) - raise RuntimeError("op wasn't a valid value: " + str(op)) - - -def evaluate_expression(exp): - left = 0 - right = 0 - - if exp.HasField("left_value"): - left = exp.left_value - else: - left = evaluate_expression(exp.left_expression) - - if exp.HasField("right_value"): - right = exp.right_value - else: - right = evaluate_expression(exp.right_expression) - - op = exp.op - if op == 0: - return clamp(left + right) - elif op == 1: - return clamp(left - right) - elif op == 2: - return clamp(left * right) - elif op == 3: - return div(left, right) - elif op == 4: - return mod(left, right) - raise RuntimeError("op wasn't a valid value: " + str(op)) - - -class Benchmark: - def __init__(self, compression): - self.Request = eval_pb2.Expression - self.Response = eval_pb2.EvaluationResult - self.from_bytes_request = from_bytes_helper(eval_pb2.Expression) - self.from_bytes_response = from_bytes_helper(eval_pb2.EvaluationResult) - self.to_bytes = lambda x: x.SerializeToString() - - def setup(self, request): - return make_expression(request, 0) - - def handle(self, request, response): - response.value = evaluate_expression(request) - - def check(self, response, expected): - return response.value == expected diff --git a/benchmark/eval_pycapnp.py b/benchmark/eval_pycapnp.py deleted file mode 100755 index 61c92a7..0000000 --- a/benchmark/eval_pycapnp.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python - -import capnp -import eval_capnp -from common import rand_int, rand_double, rand_bool -from random import choice - -MAX_INT = 2**31 - 1 -MIN_INT = -(2**31) - -OPERATIONS = ["add", "subtract", "multiply", "divide", "modulus"] - - -def clamp(res): - if res > MAX_INT: - return MAX_INT - elif res < MIN_INT: - return MIN_INT - else: - return res - - -def div(a, b): - if b == 0: - return MAX_INT - if a == MIN_INT and b == -1: - return MAX_INT - - return a // b - - -def mod(a, b): - if b == 0: - return MAX_INT - if a == MIN_INT and b == -1: - return MAX_INT - - return a % b - - -def make_expression(exp, depth): - exp.op = choice(OPERATIONS) - - if rand_int(8) < depth: - left = rand_int(128) + 1 - exp.left.value = left - else: - left = make_expression(exp.left.init("expression"), depth + 1) - - if rand_int(8) < depth: - right = rand_int(128) + 1 - exp.right.value = right - else: - right = make_expression(exp.right.init("expression"), depth + 1) - - op = exp.op - if op == "add": - return clamp(left + right) - elif op == "subtract": - return clamp(left - right) - elif op == "multiply": - return clamp(left * right) - elif op == "divide": - return div(left, right) - elif op == "modulus": - return mod(left, right) - raise RuntimeError("op wasn't a valid value: " + str(op)) - - -def evaluate_expression(exp): - left = 0 - right = 0 - - which = exp.left.which() - if which == "value": - left = exp.left.value - elif which == "expression": - left = evaluate_expression(exp.left.expression) - - which = exp.right.which() - if which == "value": - right = exp.right.value - elif which == "expression": - right = evaluate_expression(exp.right.expression) - - op = exp.op - if op == "add": - return clamp(left + right) - elif op == "subtract": - return clamp(left - right) - elif op == "multiply": - return clamp(left * right) - elif op == "divide": - return div(left, right) - elif op == "modulus": - return mod(left, right) - raise RuntimeError("op wasn't a valid value: " + str(op)) - - -class Benchmark: - def __init__(self, compression): - self.Request = eval_capnp.Expression.new_message - self.Response = eval_capnp.EvaluationResult.new_message - if compression == "packed": - self.from_bytes_request = eval_capnp.Expression.from_bytes_packed - self.from_bytes_response = eval_capnp.EvaluationResult.from_bytes_packed - self.to_bytes = lambda x: x.to_bytes_packed() - else: - self.from_bytes_request = eval_capnp.Expression.from_bytes - self.from_bytes_response = eval_capnp.EvaluationResult.from_bytes - self.to_bytes = lambda x: x.to_bytes() - - def setup(self, request): - return make_expression(request, 0) - - def handle(self, request, response): - response.value = evaluate_expression(request) - - def check(self, response, expected): - return response.value == expected diff --git a/buildutils/build.py b/buildutils/build.py index e79bdf0..6bd8d80 100644 --- a/buildutils/build.py +++ b/buildutils/build.py @@ -3,7 +3,6 @@ import subprocess import os import shutil -import struct import sys @@ -14,7 +13,7 @@ def build_libcapnp(bundle_dir, build_dir): # noqa: C901 bundle_dir = os.path.abspath(bundle_dir) capnp_dir = os.path.join(bundle_dir, "capnproto-c++") build_dir = os.path.abspath(build_dir) - tmp_dir = os.path.join(capnp_dir, "build{}".format(8 * struct.calcsize("P"))) + tmp_dir = os.path.join(capnp_dir, "build") # Clean the tmp build directory every time if os.path.exists(tmp_dir): @@ -28,24 +27,9 @@ def build_libcapnp(bundle_dir, build_dir): # noqa: C901 # Enable ninja for compilation if available build_type = [] - if shutil.which("ninja") and os.name != "nt": + if shutil.which("ninja"): build_type = ["-G", "Ninja"] - # Determine python shell architecture for Windows - python_arch = 8 * struct.calcsize("P") - build_arch = [] - build_flags = [] - if os.name == "nt": - if python_arch == 64: - build_arch_flag = "x64" - elif python_arch == 32: - build_arch_flag = "Win32" - else: - raise RuntimeError("Unknown windows build arch") - build_arch = ["-A", build_arch_flag] - build_flags = ["--config", "Release"] - print("Building module for {}".format(python_arch)) - if not shutil.which("cmake"): raise RuntimeError("Could not find cmake in your path!") @@ -54,11 +38,11 @@ def build_libcapnp(bundle_dir, build_dir): # noqa: C901 "-DCMAKE_POSITION_INDEPENDENT_CODE=1", "-DBUILD_TESTING=OFF", "-DBUILD_SHARED_LIBS=OFF", + "-DWITH_OPENSSL=OFF", "-DCMAKE_INSTALL_PREFIX:PATH={}".format(build_dir), capnp_dir, ] args.extend(build_type) - args.extend(build_arch) conf = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = conf.wait() if returncode != 0: @@ -72,7 +56,6 @@ def build_libcapnp(bundle_dir, build_dir): # noqa: C901 "--target", "install", ] - args.extend(build_flags) build = subprocess.Popen(args, cwd=tmp_dir, stdout=sys.stdout) returncode = build.wait() if cxxflags is None: diff --git a/capnp/__init__.py b/capnp/__init__.py index 727ccb9..88e6021 100644 --- a/capnp/__init__.py +++ b/capnp/__init__.py @@ -1,61 +1,17 @@ -"""A python library wrapping the Cap'n Proto C++ library - -Example Usage:: - - import capnp - - addressbook = capnp.load('addressbook.capnp') - - # Building - addresses = addressbook.AddressBook.newMessage() - people = addresses.init('people', 1) - - alice = people[0] - alice.id = 123 - alice.name = 'Alice' - alice.email = 'alice@example.com' - alicePhone = alice.init('phones', 1)[0] - alicePhone.type = 'mobile' - - f = open('example.bin', 'w') - addresses.write(f) - f.close() - - # Reading - f = open('example.bin') - - addresses = addressbook.AddressBook.read(f) - - for person in addresses.people: - print(person.name, ':', person.email) - for phone in person.phones: - print(phone.type, ':', phone.number) -""" +"""Dynamic Cap'n Proto serialization for openpilot.""" from .version import version as __version__ from .lib.capnp import * from .lib.capnp import ( - _CapabilityClient, - _DynamicCapabilityClient, + _DynamicEnum, _DynamicListBuilder, _DynamicListReader, - _DynamicOrphan, - _DynamicResizableListBuilder, _DynamicStructBuilder, _DynamicStructReader, - _EventLoop, - _InterfaceModule, _ListSchema, _MallocMessageBuilder, - _PyCustomMessageBuilder, - _PackedFdMessageReader, - _StreamFdMessageReader, _StructModule, - _write_message_to_fd, - _write_packed_message_to_fd, - _AsyncIoStream as AsyncIoStream, _init_capnp_api, ) _init_capnp_api() -add_import_hook() # enable import hook by default diff --git a/capnp/_gen.py b/capnp/_gen.py deleted file mode 100644 index ac1cfda..0000000 --- a/capnp/_gen.py +++ /dev/null @@ -1,66 +0,0 @@ -import os -import sys - -from jinja2 import Environment, PackageLoader - -import capnp -import schema_capnp - - -def find_type(code, id): - for node in code["nodes"]: - if node["id"] == id: - return node - - return None - - -def main(): - env = Environment(loader=PackageLoader("capnp", "templates")) - env.filters["format_name"] = lambda name: name[name.find(":") + 1 :] - - code = schema_capnp.CodeGeneratorRequest.read(sys.stdin) - code = code.to_dict() - code["nodes"] = [node for node in code["nodes"] if "struct" in node and node["scopeId"] != 0] - for node in code["nodes"]: - displayName = node["displayName"] - parent, path = displayName.split(":") - node["module_path"] = parent.replace(".", "_") + "." + ".".join([x[0].upper() + x[1:] for x in path.split(".")]) - node["module_name"] = path.replace(".", "_") - node["c_module_path"] = "::".join([x[0].upper() + x[1:] for x in path.split(".")]) - node["schema"] = "_{}_Schema".format(node["module_name"]) - is_union = False - for field in node["struct"]["fields"]: - if field["discriminantValue"] != 65535: - is_union = True - field["c_name"] = field["name"][0].upper() + field["name"][1:] - if "slot" in field: - field["type"] = list(field["slot"]["type"].keys())[0] - if not isinstance(field["slot"]["type"][field["type"]], dict): - continue - sub_type = field["slot"]["type"][field["type"]].get("typeId", None) - if sub_type: - field["sub_type"] = find_type(code, sub_type) - sub_type = field["slot"]["type"][field["type"]].get("elementType", None) - if sub_type: - field["sub_type"] = sub_type - else: - field["type"] = find_type(code, field["group"]["typeId"]) - node["is_union"] = is_union - - include_dir = os.path.abspath(os.path.join(os.path.dirname(capnp.__file__), "..")) - module = env.get_template("module.pyx") - - for f in code["requestedFiles"]: - filename = f["filename"].replace(".", "_") + "_cython.pyx" - - file_code = dict(code) - file_code["nodes"] = [node for node in file_code["nodes"] if node["displayName"].startswith(f["filename"])] - with open(filename, "w") as out: - out.write(module.render(code=file_code, file=f, include_dir=include_dir)) - - setup = env.get_template("setup.py.tmpl") - with open("setup_capnp.py", "w") as out: - out.write(setup.render(code=code)) - print("You now need to build the cython module by running `python setup_capnp.py build_ext --inplace`.") - print() diff --git a/capnp/helpers/capabilityHelper.cpp b/capnp/helpers/capabilityHelper.cpp deleted file mode 100644 index 5a0a3db..0000000 --- a/capnp/helpers/capabilityHelper.cpp +++ /dev/null @@ -1,199 +0,0 @@ -#include "capnp/helpers/capabilityHelper.h" -#include "capnp/lib/capnp_api.h" - -::kj::Promise> convert_to_pypromise(capnp::RemotePromise promise) { - return promise.then([](capnp::Response&& response) { - return stealPyRef(wrap_dynamic_struct_reader(response)); } ); -} - -void c_reraise_kj_exception() { - GILAcquire gil; - try { - if (PyErr_Occurred()) - ; // let the latest Python exn pass through and ignore the current one - else - throw; - } - catch (kj::Exception& exn) { - auto obj = wrap_kj_exception_for_reraise(exn); - if (obj == nullptr) { - return; - } - PyErr_SetObject((PyObject*)obj->ob_type, obj); - Py_DECREF(obj); - } - catch (const std::exception& exn) { - PyErr_SetString(PyExc_RuntimeError, exn.what()); - } - catch (...) - { - PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); - } -} - -void check_py_error() { - GILAcquire gil; - PyObject * err = PyErr_Occurred(); - if(err) { - PyObject * ptype, *pvalue, *ptraceback; - PyErr_Fetch(&ptype, &pvalue, &ptraceback); - if(ptype == NULL || pvalue == NULL || ptraceback == NULL) - throw kj::Exception(kj::Exception::Type::FAILED, kj::heapString("capabilityHelper.h"), 44, kj::heapString("Unknown error occurred")); - - PyObject * info = get_exception_info(ptype, pvalue, ptraceback); - - PyObject * py_filename = PyTuple_GetItem(info, 0); - kj::String filename(kj::heapString(PyBytes_AsString(py_filename))); - - PyObject * py_line = PyTuple_GetItem(info, 1); - int line = PyLong_AsLong(py_line); - - PyObject * py_description = PyTuple_GetItem(info, 2); - kj::String description(kj::heapString(PyBytes_AsString(py_description))); - - Py_DECREF(ptype); - Py_DECREF(pvalue); - Py_DECREF(ptraceback); - Py_DECREF(info); - PyErr_Clear(); - - throw kj::Exception(kj::Exception::Type::FAILED, kj::mv(filename), line, kj::mv(description)); - } -} - -kj::Promise> wrapPyFunc(kj::Own func, kj::Own arg) { - GILAcquire gil; - PyObject * result = PyObject_CallFunctionObjArgs(func->obj, arg->obj, NULL); - check_py_error(); - return stealPyRef(result); -} - -::kj::Promise> then(kj::Promise> promise, - kj::Own func, kj::Own error_func) { - if(error_func->obj == Py_None) - return promise.then([func=kj::mv(func)](kj::Own arg) mutable { - return wrapPyFunc(kj::mv(func), kj::mv(arg)); } ); - else - return promise.then - ([func=kj::mv(func)](kj::Own arg) mutable { - return wrapPyFunc(kj::mv(func), kj::mv(arg)); }, - [error_func=kj::mv(error_func)](kj::Exception arg) mutable { - return wrapPyFunc(kj::mv(error_func), stealPyRef(wrap_kj_exception(arg))); } ); -} - -kj::Promise PythonInterfaceDynamicImpl::call(capnp::InterfaceSchema::Method method, - capnp::CallContext< capnp::DynamicStruct, - capnp::DynamicStruct> context) { - auto methodName = method.getProto().getName(); - - kj::Promise * promise = call_server_method(this->py_server->obj, - const_cast(methodName.cStr()), - context, - this->kj_loop->obj); - - check_py_error(); - - if(promise == nullptr) - return kj::READY_NOW; - - kj::Promise ret(kj::mv(*promise)); - delete promise; - return ret; -}; - - -class ReadPromiseAdapter { -public: - ReadPromiseAdapter(kj::PromiseFulfiller& fulfiller, PyObject* protocol, - void* buffer, size_t minBytes, size_t maxBytes) - : protocol(protocol) { - _asyncio_stream_read_start(protocol, buffer, minBytes, maxBytes, fulfiller); - } - - ~ReadPromiseAdapter() { - _asyncio_stream_read_stop(protocol); - } - -private: - PyObject* protocol; -}; - - -class WritePromiseAdapter { -public: - WritePromiseAdapter(kj::PromiseFulfiller& fulfiller, PyObject* protocol, - kj::ArrayPtr> pieces) - : protocol(protocol) { - _asyncio_stream_write_start(protocol, pieces, fulfiller); - } - - ~WritePromiseAdapter() { - _asyncio_stream_write_stop(protocol); - } - -private: - PyObject* protocol; - -}; - -PyAsyncIoStream::~PyAsyncIoStream() { - _asyncio_stream_close(protocol->obj); -} - -kj::Promise PyAsyncIoStream::tryRead(void* buffer, size_t minBytes, size_t maxBytes) { - return kj::newAdaptedPromise(protocol->obj, buffer, minBytes, maxBytes); -} - -kj::Promise PyAsyncIoStream::write(const void* buffer, size_t size) { - KJ_UNIMPLEMENTED("No use-case AsyncIoStream::write was found yet."); -} - -kj::Promise PyAsyncIoStream::write(kj::ArrayPtr> pieces) { - return kj::newAdaptedPromise(protocol->obj, pieces); -} - -kj::Promise PyAsyncIoStream::whenWriteDisconnected() { - // TODO: Possibly connect this to protocol.connection_lost? - return kj::NEVER_DONE; -} - -void PyAsyncIoStream::shutdownWrite() { - _asyncio_stream_shutdown_write(protocol->obj); -} - -class TaskToPromiseAdapter { -public: - TaskToPromiseAdapter(kj::PromiseFulfiller& fulfiller, - kj::Own task, PyObject* callback) - : task(kj::mv(task)) { - promise_task_add_done_callback(this->task->obj, callback, fulfiller); - } - - ~TaskToPromiseAdapter() { - promise_task_cancel(this->task->obj); - } - -private: - kj::Own task; -}; - -kj::Promise taskToPromise(kj::Own task, PyObject* callback) { - return kj::newAdaptedPromise(kj::mv(task), callback); -} - -::kj::Promise> tryReadMessage(kj::AsyncIoStream& stream, capnp::ReaderOptions opts) { - return capnp::tryReadMessage(stream, opts) - .then([](kj::Maybe> maybeReader) -> kj::Promise> { - KJ_IF_MAYBE(reader, maybeReader) { - PyObject* pyreader = make_async_message_reader(kj::mv(*reader)); - check_py_error(); - return kj::heap(pyreader); - } else { - return kj::heap(Py_None); - } - }); -} - -void init_capnp_api() { - import_capnp__lib__capnp(); -} diff --git a/capnp/helpers/capabilityHelper.h b/capnp/helpers/capabilityHelper.h deleted file mode 100644 index 4dcc9b4..0000000 --- a/capnp/helpers/capabilityHelper.h +++ /dev/null @@ -1,139 +0,0 @@ -#pragma once - -#include "capnp/dynamic.h" -#include -#include -#include -#include "Python.h" - -class GILAcquire { -public: - GILAcquire() : gstate(PyGILState_Ensure()) {} - ~GILAcquire() { - PyGILState_Release(gstate); - } - - PyGILState_STATE gstate; -}; - -class GILRelease { -public: - GILRelease() { - Py_UNBLOCK_THREADS - } - ~GILRelease() { - Py_BLOCK_THREADS - } - - PyThreadState *_save; // The macros above read/write from this variable -}; - -class PyRefCounter { -public: - PyObject * obj; - - PyRefCounter(PyObject * o) : obj(o) { - GILAcquire gil; - Py_INCREF(obj); - } - - PyRefCounter(const PyRefCounter & ref) : obj(ref.obj) { - GILAcquire gil; - Py_INCREF(obj); - } - - ~PyRefCounter() { - GILAcquire gil; - Py_DECREF(obj); - } -}; - -inline kj::Own stealPyRef(PyObject* o) { - auto ret = kj::heap(o); - Py_DECREF(o); - return ret; -} - -::kj::Promise> convert_to_pypromise(capnp::RemotePromise promise); - -inline ::kj::Promise> convert_to_pypromise(kj::Promise promise) { - return promise.then([]() { - GILAcquire gil; - return kj::heap(Py_None); - }); -} - -void c_reraise_kj_exception(); - -void check_py_error(); - -::kj::Promise> then(kj::Promise> promise, - kj::Own func, kj::Own error_func); - -class PythonInterfaceDynamicImpl final: public capnp::DynamicCapability::Server { -public: - kj::Own py_server; - kj::Own kj_loop; - -#if (CAPNP_VERSION_MAJOR < 1) - PythonInterfaceDynamicImpl(capnp::InterfaceSchema & schema, - kj::Own _py_server, - kj::Own kj_loop) - : capnp::DynamicCapability::Server(schema), - py_server(kj::mv(_py_server)), kj_loop(kj::mv(kj_loop)) { } -#else - PythonInterfaceDynamicImpl(capnp::InterfaceSchema & schema, - kj::Own _py_server, - kj::Own kj_loop) - : capnp::DynamicCapability::Server(schema, { true }), - py_server(kj::mv(_py_server)), kj_loop(kj::mv(kj_loop)) { } -#endif - - ~PythonInterfaceDynamicImpl() { - } - - kj::Promise call(capnp::InterfaceSchema::Method method, - capnp::CallContext< capnp::DynamicStruct, capnp::DynamicStruct> context); -}; - -inline void allowCancellation(capnp::CallContext context) { -#if (CAPNP_VERSION_MAJOR < 1) - context.allowCancellation(); -#endif -} - -class PyAsyncIoStream: public kj::AsyncIoStream { -public: - kj::Own protocol; - - PyAsyncIoStream(kj::Own protocol) : protocol(kj::mv(protocol)) {} - ~PyAsyncIoStream(); - - kj::Promise tryRead(void* buffer, size_t minBytes, size_t maxBytes); - - kj::Promise write(const void* buffer, size_t size); - - kj::Promise write(kj::ArrayPtr> pieces); - - kj::Promise whenWriteDisconnected(); - - void shutdownWrite(); -}; - -template -inline void rejectDisconnected(kj::PromiseFulfiller& fulfiller, kj::StringPtr message) { - fulfiller.reject(KJ_EXCEPTION(DISCONNECTED, message)); -} -inline void rejectVoidDisconnected(kj::PromiseFulfiller& fulfiller, kj::StringPtr message) { - fulfiller.reject(KJ_EXCEPTION(DISCONNECTED, message)); -} - -inline kj::Exception makeException(kj::StringPtr message) { - return KJ_EXCEPTION(FAILED, message); -} - -kj::Promise taskToPromise(kj::Own coroutine, PyObject* callback); - -::kj::Promise> tryReadMessage(kj::AsyncIoStream& stream, capnp::ReaderOptions opts); - -void init_capnp_api(); diff --git a/capnp/helpers/checkCompiler.h b/capnp/helpers/checkCompiler.h index e0e03c4..66381ca 100644 --- a/capnp/helpers/checkCompiler.h +++ b/capnp/helpers/checkCompiler.h @@ -1,8 +1,3 @@ -#ifdef _MSC_VER -#pragma comment(lib, "Ws2_32.lib") -#pragma comment(lib, "advapi32.lib") -#endif - #include "capnp/dynamic.h" static_assert(CAPNP_VERSION >= 8000, "Version of Cap'n Proto C++ Library is too old. Please upgrade to a version >= 0.8 and then re-install this python library"); diff --git a/capnp/helpers/deserialize.h b/capnp/helpers/deserialize.h deleted file mode 100644 index d5ed120..0000000 --- a/capnp/helpers/deserialize.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include "capnp/dynamic.h" -#include "capnp/schema.capnp.h" - -/// @brief Convert the dynamic struct to a Node::Reader -::capnp::schema::Node::Reader toReader(capnp::DynamicStruct::Reader reader) -{ - // requires an intermediate step to AnyStruct before going directly to Node::Reader, - // since there exists no direct conversion from DynamicStruct::Reader to Node::Reader. - return reader.as().as(); -} diff --git a/capnp/helpers/exception.cpp b/capnp/helpers/exception.cpp new file mode 100644 index 0000000..a48bd5d --- /dev/null +++ b/capnp/helpers/exception.cpp @@ -0,0 +1,31 @@ +#include "capnp/helpers/exception.h" +#include "capnp/lib/capnp_api.h" + +void c_reraise_kj_exception() { + GILAcquire gil; + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } + catch (kj::Exception& exn) { + auto obj = wrap_kj_exception_for_reraise(exn); + if (obj == nullptr) { + return; + } + PyErr_SetObject((PyObject*)obj->ob_type, obj); + Py_DECREF(obj); + } + catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} + +void init_capnp_api() { + import_capnp__lib__capnp(); +} diff --git a/capnp/helpers/exception.h b/capnp/helpers/exception.h new file mode 100644 index 0000000..c33d16f --- /dev/null +++ b/capnp/helpers/exception.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include +#include + +class GILAcquire { +public: + GILAcquire() : gstate(PyGILState_Ensure()) {} + ~GILAcquire() { + PyGILState_Release(gstate); + } + + PyGILState_STATE gstate; +}; + +void c_reraise_kj_exception(); +void init_capnp_api(); diff --git a/capnp/helpers/helpers.pxd b/capnp/helpers/helpers.pxd index e476c99..d717cbf 100644 --- a/capnp/helpers/helpers.pxd +++ b/capnp/helpers/helpers.pxd @@ -1,34 +1,9 @@ -from capnp.includes.capnp_cpp cimport ( - Maybe, PyPromise, VoidPromise, RemotePromise, - DynamicCapability, InterfaceSchema, EnumSchema, StructSchema, DynamicValue, Capability, - RpcSystem, MessageBuilder, Own, PyRefCounter, Node, DynamicStruct, CallContext -) - -from capnp.includes.schema_cpp cimport ByteArray - +from capnp.includes.capnp_cpp cimport Maybe, EnumSchema, StructSchema from non_circular cimport c_reraise_kj_exception as reraise_kj_exception -from cpython.ref cimport PyObject - cdef extern from "capnp/helpers/fixMaybe.h": EnumSchema.Enumerant fixMaybe(Maybe[EnumSchema.Enumerant]) except +reraise_kj_exception StructSchema.Field fixMaybe(Maybe[StructSchema.Field]) except +reraise_kj_exception -cdef extern from "capnp/helpers/capabilityHelper.h": - PyPromise then(PyPromise promise, Own[PyRefCounter] func, Own[PyRefCounter] error_func) - PyPromise convert_to_pypromise(RemotePromise) - PyPromise convert_to_pypromise(VoidPromise) - VoidPromise taskToPromise(Own[PyRefCounter] coroutine, PyObject* callback) - void allowCancellation(CallContext context) except +reraise_kj_exception nogil +cdef extern from "capnp/helpers/exception.h": void init_capnp_api() - -cdef extern from "capnp/helpers/rpcHelper.h": - Own[Capability.Client] bootstrapHelper(RpcSystem&) except +reraise_kj_exception - Own[Capability.Client] bootstrapHelperServer(RpcSystem&) except +reraise_kj_exception - -cdef extern from "capnp/helpers/serialize.h": - ByteArray messageToPackedBytes(MessageBuilder &, size_t wordCount) except +reraise_kj_exception - -cdef extern from "capnp/helpers/deserialize.h": - Node.Reader toReader(DynamicStruct.Reader reader) except +reraise_kj_exception - diff --git a/capnp/helpers/non_circular.pxd b/capnp/helpers/non_circular.pxd index 000c32b..354cadf 100644 --- a/capnp/helpers/non_circular.pxd +++ b/capnp/helpers/non_circular.pxd @@ -1,8 +1,2 @@ -from cpython.ref cimport PyObject -from libcpp cimport bool - -cdef extern from "capnp/helpers/capabilityHelper.h": +cdef extern from "capnp/helpers/exception.h": void c_reraise_kj_exception() - cdef cppclass PyRefCounter: - PyRefCounter(PyObject *) - PyObject * obj diff --git a/capnp/helpers/rpcHelper.h b/capnp/helpers/rpcHelper.h deleted file mode 100644 index c017ba3..0000000 --- a/capnp/helpers/rpcHelper.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -#include "capnp/dynamic.h" -#include -#include "capnp/rpc-twoparty.h" -#include "Python.h" -#include "capabilityHelper.h" - -kj::Own bootstrapHelper(capnp::RpcSystem& client) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::SERVER); - return kj::heap(client.bootstrap(hostId)); -} - -kj::Own bootstrapHelperServer(capnp::RpcSystem& client) { - capnp::MallocMessageBuilder hostIdMessage(8); - auto hostId = hostIdMessage.initRoot(); - hostId.setSide(capnp::rpc::twoparty::Side::CLIENT); - return kj::heap(client.bootstrap(hostId)); -} diff --git a/capnp/helpers/serialize.h b/capnp/helpers/serialize.h deleted file mode 100644 index ddf36e5..0000000 --- a/capnp/helpers/serialize.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "kj/io.h" -#include "capnp/dynamic.h" -#include "capnp/serialize-packed.h" - -kj::Array< ::capnp::byte> messageToPackedBytes(capnp::MessageBuilder & message, size_t wordCount) -{ - - kj::Array result = kj::heapArray(wordCount * 8); - kj::ArrayOutputStream out(result.asPtr()); - capnp::writePackedMessage(out, message); - return heapArray(out.getArray()); // TODO: make this non-copying somehow -} diff --git a/capnp/includes/PyCustomMessageBuilder.cpp b/capnp/includes/PyCustomMessageBuilder.cpp deleted file mode 100644 index 9a08833..0000000 --- a/capnp/includes/PyCustomMessageBuilder.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "PyCustomMessageBuilder.h" -#include - -namespace capnp { - -PyCustomMessageBuilder::PyCustomMessageBuilder( - PyObject* allocateSegmentCallable, uint firstSegmentWords) - : allocateSegmentCallable(allocateSegmentCallable), firstSize(firstSegmentWords) -{ - KJ_REQUIRE(PyCallable_Check(allocateSegmentCallable), - "allocateSegmentCallable must be callable"); - Py_INCREF(allocateSegmentCallable); -} - -PyCustomMessageBuilder::~PyCustomMessageBuilder() noexcept(false) { - PyGILState_STATE gstate = PyGILState_Ensure(); - - for (auto* obj : allocatedBuffers) { - Py_DECREF(obj); - } - allocatedBuffers.clear(); - - Py_DECREF(allocateSegmentCallable); - PyGILState_Release(gstate); -} - -kj::ArrayPtr PyCustomMessageBuilder::allocateSegment(capnp::uint minimumSize) { - PyGILState_STATE gstate = PyGILState_Ensure(); - KJ_DEFER({ PyGILState_Release(gstate); }); - if (curSize == 0) { - minimumSize = kj::max(minimumSize, firstSize); - } - PyObject* pyBufObj = PyObject_CallFunction(allocateSegmentCallable, "I", minimumSize); - KJ_REQUIRE(pyBufObj, "PyCustomMessageBuilder: allocateSegment failed"); - allocatedBuffers.push_back(pyBufObj); - - - Py_buffer view; - int bufRes = PyObject_GetBuffer(pyBufObj, &view, PyBUF_SIMPLE); - KJ_REQUIRE(bufRes == 0, "PyCustomMessageBuilder: object does not support buffer protocol"); - KJ_DEFER({ PyBuffer_Release(&view); }); - - size_t byteCount = view.len; - size_t wordCount = byteCount / sizeof(capnp::word); - KJ_REQUIRE(wordCount >= minimumSize, "PyCustomMessageBuilder: buffer too small for minimumSize"); - curSize += wordCount; - return kj::arrayPtr(reinterpret_cast(view.buf), wordCount); -} - -} \ No newline at end of file diff --git a/capnp/includes/PyCustomMessageBuilder.h b/capnp/includes/PyCustomMessageBuilder.h deleted file mode 100644 index 0aba128..0000000 --- a/capnp/includes/PyCustomMessageBuilder.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include "Python.h" -#include -#include -#include - -namespace capnp { - -class PyCustomMessageBuilder : public capnp::MessageBuilder { -public: - explicit PyCustomMessageBuilder(PyObject* allocateSegmentCallable, - uint firstSegmentWords = capnp::SUGGESTED_FIRST_SEGMENT_WORDS); - - ~PyCustomMessageBuilder() noexcept(false) override; - - kj::ArrayPtr allocateSegment(capnp::uint minimumSize) override; - -private: - PyObject* allocateSegmentCallable; - - uint firstSize; - uint curSize = 0; - - std::vector allocatedBuffers; -}; - -} diff --git a/capnp/includes/capnp_cpp.pxd b/capnp/includes/capnp_cpp.pxd index a5bc62c..2d1da39 100644 --- a/capnp/includes/capnp_cpp.pxd +++ b/capnp/includes/capnp_cpp.pxd @@ -3,12 +3,11 @@ cdef extern from "capnp/helpers/checkCompiler.h": pass -from libcpp cimport bool from capnp.helpers.non_circular cimport ( - c_reraise_kj_exception as reraise_kj_exception, PyRefCounter, + c_reraise_kj_exception as reraise_kj_exception, ) from capnp.includes.schema_cpp cimport ( - Node, Data, StructNode, EnumNode, InterfaceNode, MessageBuilder, MessageReader, ReaderOptions, + Node, Data, Field as SchemaField, Enumerant as SchemaEnumerant, MessageBuilder, MessageReader, ReaderOptions, ) from capnp.includes.types cimport * @@ -46,26 +45,6 @@ cdef extern from "kj/exception.h" namespace " ::kj": int getType() StringPtr getDescription() -cdef extern from "kj/memory.h" namespace " ::kj": - cdef cppclass Own[T] nogil: - Own() - T& operator*() - T* get() - Own[T] heap[T](...) - -cdef extern from "kj/async.h" namespace " ::kj": - cdef cppclass Promise[T] nogil: - Promise(Promise) - Promise(T) - String trace() - Promise[T] attach(Own[PyRefCounter] &) - Promise[T] attach(Own[PyRefCounter] &, Own[PyRefCounter] &) - Promise[T] attach(Own[PyRefCounter] &, Own[PyRefCounter] &, Own[PyRefCounter] &) - Promise[T] attach(Own[PyRefCounter] &, Own[PyRefCounter] &, Own[PyRefCounter] &, Own[PyRefCounter] &) - -ctypedef Promise[Own[PyRefCounter]] PyPromise -ctypedef Promise[void] VoidPromise - cdef extern from "kj/string-tree.h" namespace " ::kj": cdef cppclass StringTree nogil: String flatten() @@ -80,59 +59,15 @@ cdef extern from "kj/common.h" namespace " ::kj": size_t size() T& operator[](size_t index) -cdef extern from "kj/array.h" namespace " ::kj": - cdef cppclass Array[T] nogil: - T* begin() - size_t size() - T& operator[](size_t index) - cdef cppclass ArrayBuilder[T] nogil: - T* begin() - size_t size() - T& operator[](size_t index) - T& add(T&) - Array[T] finish() - - -cdef extern from "kj/async-io.h" namespace " ::kj": - cdef cppclass AsyncIoStream nogil: - Promise[size_t] read(void*, size_t, size_t) except +reraise_kj_exception - Promise[void] write(const void*, size_t) except +reraise_kj_exception - -cdef extern from "capnp/schema.capnp.h" namespace " ::capnp": - enum TypeWhich" ::capnp::schema::Type::Which": - TypeWhichVOID " ::capnp::schema::Type::Which::VOID" - TypeWhichBOOL " ::capnp::schema::Type::Which::BOOL" - TypeWhichINT8 " ::capnp::schema::Type::Which::INT8" - TypeWhichINT16 " ::capnp::schema::Type::Which::INT16" - TypeWhichINT32 " ::capnp::schema::Type::Which::INT32" - TypeWhichINT64 " ::capnp::schema::Type::Which::INT64" - TypeWhichUINT8 " ::capnp::schema::Type::Which::UINT8" - TypeWhichUINT16 " ::capnp::schema::Type::Which::UINT16" - TypeWhichUINT32 " ::capnp::schema::Type::Which::UINT32" - TypeWhichUINT64 " ::capnp::schema::Type::Which::UINT64" - TypeWhichFLOAT32 " ::capnp::schema::Type::Which::FLOAT32" - TypeWhichFLOAT64 " ::capnp::schema::Type::Which::FLOAT64" - TypeWhichTEXT " ::capnp::schema::Type::Which::TEXT" - TypeWhichDATA " ::capnp::schema::Type::Which::DATA" - TypeWhichLIST " ::capnp::schema::Type::Which::LIST" - TypeWhichENUM " ::capnp::schema::Type::Which::ENUM" - TypeWhichSTRUCT " ::capnp::schema::Type::Which::STRUCT" - TypeWhichINTERFACE " ::capnp::schema::Type::Which::INTERFACE" - TypeWhichANY_POINTER " ::capnp::schema::Type::Which::ANY_POINTER" - cdef extern from "capnp/schema.h" namespace " ::capnp": cdef cppclass SchemaType" ::capnp::Type" nogil: - SchemaType() - SchemaType(TypeWhich) cbool isList() cbool isEnum() cbool isStruct() - cbool isInterface() cbool isData() StructSchema asStruct() except +reraise_kj_exception EnumSchema asEnum() except +reraise_kj_exception - InterfaceSchema asInterface() except +reraise_kj_exception ListSchema asList() except +reraise_kj_exception cdef cppclass Schema nogil: @@ -141,35 +76,10 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": EnumSchema asEnum() except +reraise_kj_exception ConstSchema asConst() except +reraise_kj_exception Schema getDependency(uint64_t id) except +reraise_kj_exception - InterfaceSchema asInterface() except +reraise_kj_exception - - cdef cppclass InterfaceSchema(Schema) nogil: - cppclass SuperclassList nogil: - uint size() - InterfaceSchema operator[](uint index) - - cppclass Method nogil: - InterfaceNode.Method.Reader getProto() - InterfaceSchema getContainingInterface() - uint16_t getOrdinal() - uint getIndex() - StructSchema getParamType() - StructSchema getResultType() - - cppclass MethodList nogil: - uint size() - Method operator[](uint index) - - MethodList getMethods() - Maybe[Method] findMethodByName(StringPtr name) - Method getMethodByName(StringPtr name) - bint extends(InterfaceSchema other) - SuperclassList getSuperclasses() - # kj::Maybe findSuperclass(uint64_t typeId) const; cdef cppclass StructSchema(Schema) nogil: cppclass Field nogil: - StructNode.Member.Reader getProto() + SchemaField.Reader getProto() StructSchema getContainingStruct() uint getIndex() SchemaType getType() @@ -192,7 +102,7 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": cdef cppclass EnumSchema nogil: cppclass Enumerant nogil: - EnumNode.Enumerant.Reader getProto() + SchemaEnumerant.Reader getProto() EnumSchema getContainingEnum() uint16_t getOrdinal() @@ -207,11 +117,6 @@ cdef extern from "capnp/schema.h" namespace " ::capnp": cdef cppclass ListSchema nogil: SchemaType getElementType() - ListSchema listSchemaOfStruct" ::capnp::ListSchema::of"(StructSchema) nogil - ListSchema listSchemaOfEnum" ::capnp::ListSchema::of"(EnumSchema) nogil - ListSchema listSchemaOfInterface" ::capnp::ListSchema::of"(InterfaceSchema) nogil - ListSchema listSchemaOfList" ::capnp::ListSchema::of"(ListSchema) nogil - ListSchema listSchemaOfType" ::capnp::ListSchema::of"(SchemaType) nogil cdef cppclass ConstSchema: pass @@ -222,8 +127,6 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": pass cppclass Builder nogil: pass - cppclass Pipeline nogil: - pass enum Type: TYPE_UNKNOWN " ::capnp::DynamicValue::UNKNOWN" @@ -237,8 +140,6 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": TYPE_LIST " ::capnp::DynamicValue::LIST" TYPE_ENUM " ::capnp::DynamicValue::ENUM" TYPE_STRUCT " ::capnp::DynamicValue::STRUCT" - TYPE_CAPABILITY " ::capnp::DynamicValue::CAPABILITY" - TYPE_ANY_POINTER " ::capnp::DynamicValue::ANY_POINTER" cdef cppclass DynamicStruct nogil: cppclass Reader nogil: @@ -250,11 +151,6 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": uint64_t getId"getSchema().getProto().getId"() Maybe[StructSchema.Field] which() MessageSize totalSize() - cppclass Pipeline nogil: - Pipeline() - Pipeline(Pipeline &) - DynamicValueForward.Pipeline get(char *) - StructSchema getSchema() cdef cppclass DynamicStruct_Builder" ::capnp::DynamicStruct::Builder" nogil: # Need to flatten this class out, since nested C++ classes cause havoc with cython fused types @@ -273,63 +169,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": StructSchema getSchema() uint64_t getId"getSchema().getProto().getId"() Maybe[StructSchema.Field] which() - void adopt(char *, DynamicOrphan) except +reraise_kj_exception - DynamicOrphan disown(char *) DynamicStruct.Reader asReader() MessageSize totalSize() -cdef extern from "capnp/dynamic.h" namespace " ::capnp": - cdef cppclass DynamicCapability nogil: - cppclass Client nogil: - Client() - Client(Client&) - Client(Own[PythonInterfaceDynamicImpl]) - Client upcast(InterfaceSchema requestedSchema) except +reraise_kj_exception - DynamicCapability.Client castAs"castAs< ::capnp::DynamicCapability>"(InterfaceSchema) - InterfaceSchema getSchema() - Request newRequest(char * methodName) - # Request newRequest(char * methodName, MessageSize) - -cdef extern from "capnp/capability.h" namespace " ::capnp": - cdef cppclass Response" ::capnp::Response< ::capnp::DynamicStruct>"(DynamicStruct.Reader) nogil: - Response(Response) - cdef cppclass RemotePromise" ::capnp::RemotePromise< ::capnp::DynamicStruct>"( - Promise[Response], DynamicStruct.Pipeline) nogil: - RemotePromise(RemotePromise) - cdef cppclass Capability nogil: - cppclass Client nogil: - Client(Client&) - DynamicCapability.Client castAs"castAs< ::capnp::DynamicCapability>"(InterfaceSchema) - -cdef extern from "capnp/rpc-twoparty.h" namespace " ::capnp": - cdef cppclass RpcSystem" ::capnp::RpcSystem" nogil: - RpcSystem(RpcSystem&&) - - cdef cppclass Side" ::capnp::rpc::twoparty::Side" nogil: - pass - cdef Side CLIENT" ::capnp::rpc::twoparty::Side::CLIENT" - cdef Side SERVER" ::capnp::rpc::twoparty::Side::SERVER" - - cdef cppclass TwoPartyVatNetwork nogil: - TwoPartyVatNetwork(EventLoop &, AsyncIoStream& stream, Side, ReaderOptions) - VoidPromise onDisconnect() - VoidPromise onDrained() - RpcSystem makeRpcServer(TwoPartyVatNetwork&, Capability.Client) nogil - RpcSystem makeRpcClient(TwoPartyVatNetwork&) nogil - -cdef extern from "capnp/dynamic.h" namespace " ::capnp": - cdef cppclass Request" ::capnp::Request< ::capnp::DynamicStruct, ::capnp::DynamicStruct>" nogil: - Request() - Request(Request &) - DynamicValueForward.Builder get(char *) except +reraise_kj_exception - bint has(char *) except +reraise_kj_exception - void set(char *, DynamicValueForward.Reader) except +reraise_kj_exception - DynamicValueForward.Builder init(char *, uint size) except +reraise_kj_exception - DynamicValueForward.Builder init(char *) except +reraise_kj_exception - StructSchema getSchema() - Maybe[StructSchema.Field] which() - RemotePromise send() - cdef extern from "capnp/dynamic.h" namespace " ::capnp": cdef cppclass DynamicEnum nogil: uint16_t getRaw() @@ -346,35 +188,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": uint size() void set(uint index, DynamicValueForward.Reader value) except +reraise_kj_exception DynamicValueForward.Builder init(uint index, uint size) except +reraise_kj_exception - void adopt(uint, DynamicOrphan) except +reraise_kj_exception - DynamicOrphan disown(uint) StructSchema getStructElementType'getSchema().getStructElementType'() DynamicList.Reader asReader() except +reraise_kj_exception -cdef extern from "capnp/any.h" namespace " ::capnp": - cdef cppclass AnyPointer nogil: - cppclass Reader nogil: - DynamicStruct.Reader getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) except +reraise_kj_exception - DynamicCapability.Client getAsCapability"getAs< ::capnp::DynamicCapability>"( - InterfaceSchema) except +reraise_kj_exception - DynamicList.Reader getAsList"getAs< ::capnp::DynamicList>"(ListSchema) except +reraise_kj_exception - StringPtr getAsText"getAs< ::capnp::Text>"() except +reraise_kj_exception - cppclass Builder nogil: - Builder(Builder) - DynamicStruct_Builder getAs"getAs< ::capnp::DynamicStruct>"(StructSchema) except +reraise_kj_exception - DynamicCapability.Client getAsCapability"getAs< ::capnp::DynamicCapability>"( - InterfaceSchema) except +reraise_kj_exception - DynamicList.Builder getAsList"getAs< ::capnp::DynamicList>"(ListSchema) except +reraise_kj_exception - StringPtr getAsText"getAs< ::capnp::Text>"() except +reraise_kj_exception - void setAsStruct"setAs< ::capnp::DynamicStruct>"(DynamicStruct.Reader&) except +reraise_kj_exception - void setAsText"setAs< ::capnp::Text>"(char*) except +reraise_kj_exception - AnyPointer.Reader asReader() except +reraise_kj_exception - void set(AnyPointer.Reader) except +reraise_kj_exception - DynamicStruct_Builder initAsStruct"initAs< ::capnp::DynamicStruct>"( - StructSchema) except +reraise_kj_exception - DynamicList.Builder initAsList"initAs< ::capnp::DynamicList>"(ListSchema, uint) except +reraise_kj_exception - - cdef extern from "capnp/dynamic.h" namespace " ::capnp": cdef cppclass DynamicValue nogil: cppclass Reader nogil: @@ -398,9 +214,6 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": Reader(DynamicList.Reader& value) Reader(DynamicEnum value) Reader(DynamicStruct.Reader& value) - Reader(DynamicCapability.Client& value) - Reader(Own[PythonInterfaceDynamicImpl] value) - Reader(AnyPointer.Reader& value) Type getType() int64_t asInt"as"() uint64_t asUint"as"() @@ -409,8 +222,6 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": StringPtr asText"as< ::capnp::Text>"() DynamicList.Reader asList"as< ::capnp::DynamicList>"() DynamicStruct.Reader asStruct"as< ::capnp::DynamicStruct>"() - AnyPointer.Reader asObject"as< ::capnp::AnyPointer>"() - DynamicCapability.Client asCapability"as< ::capnp::DynamicCapability>"() DynamicEnum asEnum"as< ::capnp::DynamicEnum>"() Data.Reader asData"as< ::capnp::Data>"() @@ -423,22 +234,9 @@ cdef extern from "capnp/dynamic.h" namespace " ::capnp": StringPtr asText"as< ::capnp::Text>"() DynamicList.Builder asList"as< ::capnp::DynamicList>"() DynamicStruct_Builder asStruct"as< ::capnp::DynamicStruct>"() - AnyPointer.Builder asObject"as< ::capnp::AnyPointer>"() - DynamicCapability.Client asCapability"as< ::capnp::DynamicCapability>"() DynamicEnum asEnum"as< ::capnp::DynamicEnum>"() Data.Builder asData"as< ::capnp::Data>"() - cppclass Pipeline nogil: - Pipeline(Pipeline) - DynamicCapability.Client asCapability"releaseAs< ::capnp::DynamicCapability>"() - DynamicStruct.Pipeline asStruct"releaseAs< ::capnp::DynamicStruct>"() - Type getType() - -cdef extern from "capnp/schema-loader.h" namespace " ::capnp": - cdef cppclass SchemaLoader nogil: - SchemaLoader() - Schema load(Node.Reader reader) except +reraise_kj_exception - Schema get(uint64_t id_) except +reraise_kj_exception cdef extern from "capnp/schema-parser.h" namespace " ::capnp": cdef cppclass ParsedSchema(Schema) nogil: @@ -446,53 +244,3 @@ cdef extern from "capnp/schema-parser.h" namespace " ::capnp": cdef cppclass SchemaParser nogil: SchemaParser() ParsedSchema parseDiskFile(char * displayName, char * diskPath, ArrayPtr[StringPtr] importPath) - -cdef extern from "capnp/orphan.h" namespace " ::capnp": - cdef cppclass DynamicOrphan" ::capnp::Orphan< ::capnp::DynamicValue>" nogil: - DynamicValue.Builder get() - DynamicValue.Reader getReader() - -cdef extern from "capnp/capability.h" namespace " ::capnp": - cdef cppclass CallContext' ::capnp::CallContext< ::capnp::DynamicStruct, ::capnp::DynamicStruct>' nogil: - CallContext(CallContext&) - DynamicStruct.Reader getParams() except +reraise_kj_exception - void releaseParams() except +reraise_kj_exception - - DynamicStruct_Builder getResults() - DynamicStruct_Builder initResults() - void setResults(DynamicStruct.Reader value) - # void adoptResults(Orphan&& value); - # Orphanage getResultsOrphanage(uint firstSegmentWordSize = 0); - VoidPromise tailCall(Request & tailRequest) - -cdef extern from "kj/async.h" namespace " ::kj": - cdef cppclass EventPort: - bool wait() except* with gil - bool poll() except* with gil - void setRunnable(bool runnable) except* with gil - cdef cppclass EventLoop nogil: - EventLoop() - EventLoop(EventPort &) - void run() - cdef cppclass WaitScope nogil: - WaitScope(EventLoop &) - void poll() - cdef cppclass PromiseFulfiller[T] nogil: - void fulfill(T&& value) - void reject(Exception&& exception) - cdef cppclass VoidPromiseFulfiller"::kj::PromiseFulfiller" nogil: - void fulfill() - void reject(Exception&& exception) - -cdef extern from "capnp/helpers/capabilityHelper.h": - cdef cppclass PyAsyncIoStream(AsyncIoStream): - PyAsyncIoStream(Own[PyRefCounter] thisptr) - void rejectDisconnected[T](PromiseFulfiller[T]& fulfiller, StringPtr message) - void rejectVoidDisconnected(VoidPromiseFulfiller& fulfiller, StringPtr message) - Exception makeException(StringPtr message) - PyPromise tryReadMessage(AsyncIoStream& stream, ReaderOptions opts) - cppclass PythonInterfaceDynamicImpl: - PythonInterfaceDynamicImpl(InterfaceSchema&, Own[PyRefCounter] server, Own[PyRefCounter] kj_loop) - -cdef extern from "capnp/serialize-async.h" namespace " ::capnp": - VoidPromise writeMessage(AsyncIoStream& output, MessageBuilder& builder) diff --git a/capnp/includes/schema_cpp.pxd b/capnp/includes/schema_cpp.pxd index 4d869cb..7c2c185 100644 --- a/capnp/includes/schema_cpp.pxd +++ b/capnp/includes/schema_cpp.pxd @@ -1,648 +1,71 @@ -# schema.capnp.cpp.pyx # distutils: language = c++ -from libc.stdint cimport * from capnp.helpers.non_circular cimport c_reraise_kj_exception as reraise_kj_exception - from capnp.includes.types cimport * cdef extern from "capnp/dynamic.h" namespace " ::capnp": - cdef cppclass DynamicValue nogil: - cppclass Reader nogil: - pass - cppclass Builder nogil: - pass cdef cppclass DynamicStruct nogil: cppclass Reader nogil: pass - - cdef cppclass DynamicStruct_Builder" ::capnp::DynamicStruct::Builder" nogil: + cdef cppclass DynamicStruct_Builder " ::capnp::DynamicStruct::Builder" nogil: pass -cdef extern from "capnp/orphan.h" namespace " ::capnp": - cdef cppclass DynamicOrphan" ::capnp::Orphan< ::capnp::DynamicValue>" nogil: - DynamicValue.Builder get() - DynamicValue.Reader getReader() - cdef extern from "capnp/schema.h" namespace " ::capnp": - cdef cppclass Schema nogil: + cdef cppclass StructSchema nogil: pass - cdef cppclass StructSchema(Schema) nogil: - pass - -cdef extern from "capnp/any.h" namespace " ::capnp": - cdef cppclass AnyPointer nogil: - cppclass Reader nogil: - pass - cppclass Builder nogil: - pass cdef extern from "capnp/blob.h" namespace " ::capnp": cdef cppclass Data nogil: cppclass Reader nogil: - char * begin() + char* begin() size_t size() cppclass Builder nogil: - char * begin() + char* begin() size_t size() cdef cppclass Text nogil: cppclass Reader nogil: - char * cStr() - cppclass Builder nogil: - char * cStr() -cdef extern from "capnp/message.h" namespace " ::capnp": - cdef cppclass List[T] nogil: - cppclass Reader nogil: - T operator[](uint) - uint size() - cppclass Builder nogil: - T operator[](uint) - uint size() + char* cStr() cdef extern from "capnp/schema.capnp.h" namespace " ::capnp::schema": - enum: - _ElementSize_inlineComposite " ::capnp::schema::ElementSize::INLINE_COMPOSITE" - _ElementSize_eightBytes " ::capnp::schema::ElementSize::EIGHT_BYTES" - _ElementSize_pointer " ::capnp::schema::ElementSize::POINTER" - _ElementSize_bit " ::capnp::schema::ElementSize::BIT" - _ElementSize_twoBytes " ::capnp::schema::ElementSize::TWO_BYTES" - _ElementSize_fourBytes " ::capnp::schema::ElementSize::FOUR_BYTES" - _ElementSize_byte " ::capnp::schema::ElementSize::BYTE" - _ElementSize_empty " ::capnp::schema::ElementSize::EMPTY" - enum _Value_Body_Which: - _Value_Body_uint32Value " ::capnp::schema::Value::Body::Which::UINT32_VALUE" - _Value_Body_float64Value " ::capnp::schema::Value::Body::Which::FLOAT64_VALUE" - _Value_Body_voidValue " ::capnp::schema::Value::Body::Which::VOID_VALUE" - _Value_Body_dataValue " ::capnp::schema::Value::Body::Which::DATA_VALUE" - _Value_Body_listValue " ::capnp::schema::Value::Body::Which::LIST_VALUE" - _Value_Body_int32Value " ::capnp::schema::Value::Body::Which::INT32_VALUE" - _Value_Body_enumValue " ::capnp::schema::Value::Body::Which::ENUM_VALUE" - _Value_Body_int8Value " ::capnp::schema::Value::Body::Which::INT8_VALUE" - _Value_Body_boolValue " ::capnp::schema::Value::Body::Which::BOOL_VALUE" - _Value_Body_int16Value " ::capnp::schema::Value::Body::Which::INT16_VALUE" - _Value_Body_float32Value " ::capnp::schema::Value::Body::Which::FLOAT32_VALUE" - _Value_Body_interfaceValue " ::capnp::schema::Value::Body::Which::INTERFACE_VALUE" - _Value_Body_uint16Value " ::capnp::schema::Value::Body::Which::UINT16_VALUE" - _Value_Body_uint8Value " ::capnp::schema::Value::Body::Which::UINT8_VALUE" - _Value_Body_int64Value " ::capnp::schema::Value::Body::Which::INT64_VALUE" - _Value_Body_structValue " ::capnp::schema::Value::Body::Which::STRUCT_VALUE" - _Value_Body_textValue " ::capnp::schema::Value::Body::Which::TEXT_VALUE" - _Value_Body_uint64Value " ::capnp::schema::Value::Body::Which::UINT64_VALUE" - _Value_Body_objectValue " ::capnp::schema::Value::Body::Which::OBJECT_VALUE" - enum _Type_Body_Which: - _Type_Body_boolType " ::capnp::schema::Type::Body::Which::BOOL_TYPE" - _Type_Body_structType " ::capnp::schema::Type::Body::Which::STRUCT_TYPE" - _Type_Body_int32Type " ::capnp::schema::Type::Body::Which::INT32_TYPE" - _Type_Body_voidType " ::capnp::schema::Type::Body::Which::VOID_TYPE" - _Type_Body_uint16Type " ::capnp::schema::Type::Body::Which::UINT16_TYPE" - _Type_Body_dataType " ::capnp::schema::Type::Body::Which::DATA_TYPE" - _Type_Body_objectType " ::capnp::schema::Type::Body::Which::OBJECT_TYPE" - _Type_Body_int64Type " ::capnp::schema::Type::Body::Which::INT64_TYPE" - _Type_Body_float64Type " ::capnp::schema::Type::Body::Which::FLOAT64_TYPE" - _Type_Body_interfaceType " ::capnp::schema::Type::Body::Which::INTERFACE_TYPE" - _Type_Body_uint32Type " ::capnp::schema::Type::Body::Which::UINT32_TYPE" - _Type_Body_uint8Type " ::capnp::schema::Type::Body::Which::UINT8_TYPE" - _Type_Body_listType " ::capnp::schema::Type::Body::Which::LIST_TYPE" - _Type_Body_int8Type " ::capnp::schema::Type::Body::Which::INT8_TYPE" - _Type_Body_float32Type " ::capnp::schema::Type::Body::Which::FLOAT32_TYPE" - _Type_Body_enumType " ::capnp::schema::Type::Body::Which::ENUM_TYPE" - _Type_Body_uint64Type " ::capnp::schema::Type::Body::Which::UINT64_TYPE" - _Type_Body_textType " ::capnp::schema::Type::Body::Which::TEXT_TYPE" - _Type_Body_int16Type " ::capnp::schema::Type::Body::Which::INT16_TYPE" - enum _Node_Body_Which: - _Node_Body_annotationNode " ::capnp::schema::Node::Body::Which::ANNOTATION_NODE" - _Node_Body_interfaceNode " ::capnp::schema::Node::Body::Which::INTERFACE_NODE" - _Node_Body_enumNode " ::capnp::schema::Node::Body::Which::ENUM_NODE" - _Node_Body_structNode " ::capnp::schema::Node::Body::Which::STRUCT_NODE" - _Node_Body_constNode " ::capnp::schema::Node::Body::Which::CONST_NODE" - _Node_Body_fileNode " ::capnp::schema::Node::Body::Which::FILE_NODE" - enum _StructNode_Member_Body_Which: - _StructNode_Member_Body_fieldMember " ::capnp::schema::StructNode::Member::Body::Which::FIELD_MEMBER" - _StructNode_Member_Body_unionMember " ::capnp::schema::StructNode::Member::Body::Which::UNION_MEMBER" - cdef cppclass CodeGeneratorRequest - - cdef cppclass InterfaceNode - cdef cppclass Value - cdef cppclass ConstNode - cdef cppclass Type - cdef cppclass FileNode - cdef cppclass Node - cdef cppclass AnnotationNode - cdef cppclass EnumNode - cdef cppclass StructNode - cdef cppclass Annotation - cdef cppclass CodeGeneratorRequest nogil: - - cppclass Reader nogil: - - List[CodeGeneratorRequest.Node].Reader getNodes() - List[UInt64].Reader getRequestedFiles() - cppclass Builder nogil: - - List[CodeGeneratorRequest.Node].Builder getNodes() - List[CodeGeneratorRequest.Node].Builder initNodes(int) - List[UInt64].Builder getRequestedFiles() - List[UInt64].Builder initRequestedFiles(int) - - cdef cppclass InterfaceNode nogil: - cppclass Method - - cppclass Method: - cppclass Param - - cppclass Param nogil: - - cppclass Reader nogil: - - Value getDefaultValue() - Type getType() - Text.Reader getName() - List[InterfaceNode.Method.Param.Annotation].Reader getAnnotations() - cppclass Builder nogil: - - Value getDefaultValue() - void setDefaultValue(Value) - Type getType() - void setType(Type) - Text.Builder getName() - void setName(Text) - List[InterfaceNode.Method.Param.Annotation].Builder getAnnotations() - List[InterfaceNode.Method.Param.Annotation].Builder initAnnotations(int) - - cppclass Reader nogil: - - UInt16 getCodeOrder() - Text.Reader getName() - List[InterfaceNode.Method.InterfaceNode.Method.Param].Reader getParams() - UInt16 getRequiredParamCount() - Type getReturnType() - List[InterfaceNode.Method.Annotation].Reader getAnnotations() - - cppclass Builder nogil: - - UInt16 getCodeOrder() - void setCodeOrder(UInt16) - Text.Builder getName() - void setName(Text) - List[InterfaceNode.Method.InterfaceNode.Method.Param].Builder getParams() - List[InterfaceNode.Method.InterfaceNode.Method.Param].Builder initParams(int) - UInt16 getRequiredParamCount() - void setRequiredParamCount(UInt16) - Type getReturnType() - void setReturnType(Type) - List[InterfaceNode.Method.Annotation].Builder getAnnotations() - List[InterfaceNode.Method.Annotation].Builder initAnnotations(int) - - cppclass Reader nogil: - - List[InterfaceNode.InterfaceNode.Method].Reader getMethods() - - cppclass Builder nogil: - - List[InterfaceNode.InterfaceNode.Method].Builder getMethods() - List[InterfaceNode.InterfaceNode.Method].Builder initMethods(int) - - cdef cppclass Value nogil: - cppclass Body - - cppclass Body nogil: - - cppclass Reader nogil: - int which() - UInt32 getUint32Value() - Float64 getFloat64Value() - Void getVoidValue() - Data.Reader getDataValue() - Object getListValue() - Int32 getInt32Value() - UInt16 getEnumValue() - Int8 getInt8Value() - Bool getBoolValue() - Int16 getInt16Value() - Float32 getFloat32Value() - Void getInterfaceValue() - UInt16 getUint16Value() - UInt8 getUint8Value() - Int64 getInt64Value() - Object getStructValue() - Text.Reader getTextValue() - UInt64 getUint64Value() - Object getObjectValue() - - cppclass Builder nogil: - int which() - UInt32 getUint32Value() - void setUint32Value(UInt32) - Float64 getFloat64Value() - void setFloat64Value(Float64) - Void getVoidValue() - void setVoidValue(Void) - Data.Builder getDataValue() - void setDataValue(Data) - Object getListValue() - void setListValue(Object) - Int32 getInt32Value() - void setInt32Value(Int32) - UInt16 getEnumValue() - void setEnumValue(UInt16) - Int8 getInt8Value() - void setInt8Value(Int8) - Bool getBoolValue() - void setBoolValue(Bool) - Int16 getInt16Value() - void setInt16Value(Int16) - Float32 getFloat32Value() - void setFloat32Value(Float32) - Void getInterfaceValue() - void setInterfaceValue(Void) - UInt16 getUint16Value() - void setUint16Value(UInt16) - UInt8 getUint8Value() - void setUint8Value(UInt8) - Int64 getInt64Value() - void setInt64Value(Int64) - Object getStructValue() - void setStructValue(Object) - Text.Builder getTextValue() - void setTextValue(Text) - UInt64 getUint64Value() - void setUint64Value(UInt64) - Object getObjectValue() - void setObjectValue(Object) - - cppclass Reader nogil: - - Value.Body getBody() - - cppclass Builder nogil: - - Value.Body getBody() - void setBody(Value.Body) - - cdef cppclass ConstNode nogil: - - cppclass Reader nogil: - - Type getType() - Value getValue() - - cppclass Builder nogil: - - Type getType() - void setType(Type) - Value getValue() - void setValue(Value) - - cdef cppclass Type nogil: - cppclass Body - - cppclass Body nogil: - - cppclass Reader nogil: - int which() - Void getBoolType() - UInt64 getStructType() - Void getInt32Type() - Void getVoidType() - Void getUint16Type() - Void getDataType() - Void getObjectType() - Void getInt64Type() - Void getFloat64Type() - UInt64 getInterfaceType() - Void getUint32Type() - Void getUint8Type() - Type getListType() - Void getInt8Type() - Void getFloat32Type() - UInt64 getEnumType() - Void getUint64Type() - Void getTextType() - Void getInt16Type() - - cppclass Builder nogil: - int which() - Void getBoolType() - void setBoolType(Void) - UInt64 getStructType() - void setStructType(UInt64) - Void getInt32Type() - void setInt32Type(Void) - Void getVoidType() - void setVoidType(Void) - Void getUint16Type() - void setUint16Type(Void) - Void getDataType() - void setDataType(Void) - Void getObjectType() - void setObjectType(Void) - Void getInt64Type() - void setInt64Type(Void) - Void getFloat64Type() - void setFloat64Type(Void) - UInt64 getInterfaceType() - void setInterfaceType(UInt64) - Void getUint32Type() - void setUint32Type(Void) - Void getUint8Type() - void setUint8Type(Void) - Type getListType() - void setListType(Type) - Void getInt8Type() - void setInt8Type(Void) - Void getFloat32Type() - void setFloat32Type(Void) - UInt64 getEnumType() - void setEnumType(UInt64) - Void getUint64Type() - void setUint64Type(Void) - Void getTextType() - void setTextType(Void) - Void getInt16Type() - void setInt16Type(Void) - - cppclass Reader nogil: - - Type.Body getBody() - - cppclass Builder nogil: - - Type.Body getBody() - void setBody(Type.Body) - - cdef cppclass FileNode nogil: - cppclass Import - - cppclass Import nogil: - - cppclass Reader nogil: - - UInt64 getId() - Text.Reader getName() - - cppclass Builder nogil: - - UInt64 getId() - void setId(UInt64) - Text.Builder getName() - void setName(Text) - - cppclass Reader nogil: - - List[FileNode.FileNode.Import].Reader getImports() - - cppclass Builder nogil: - - List[FileNode.FileNode.Import].Builder getImports() - List[FileNode.FileNode.Import].Builder initImports(int) - cdef cppclass Node nogil: - cppclass Body - cppclass NestedNode - - cppclass Body nogil: - - cppclass Reader nogil: - int which() - AnnotationNode getAnnotationNode() - InterfaceNode getInterfaceNode() - EnumNode getEnumNode() - StructNode getStructNode() - ConstNode getConstNode() - FileNode getFileNode() - cppclass Builder nogil: - int which() - AnnotationNode getAnnotationNode() - void setAnnotationNode(AnnotationNode) - InterfaceNode getInterfaceNode() - void setInterfaceNode(InterfaceNode) - EnumNode getEnumNode() - void setEnumNode(EnumNode) - StructNode getStructNode() - void setStructNode(StructNode) - ConstNode getConstNode() - void setConstNode(ConstNode) - FileNode getFileNode() - void setFileNode(FileNode) - cppclass NestedNode nogil: - cppclass Reader nogil: - Text.Reader getName() - UInt64 getId() - cppclass Builder nogil: - - Text.Builder getName() - void setName(Text) - UInt64 getId() - void setId(UInt64) + uint64_t getId() cppclass Reader nogil: - - Node.Body getBody() Text.Reader getDisplayName() - List[Node.Annotation].Reader getAnnotations() - UInt64 getScopeId() - List[Node.Node.NestedNode].Reader getNestedNodes() - UInt64 getId() - bint isFile() + uint64_t getScopeId() + uint64_t getId() + ListNestedNodeReader getNestedNodes() bint isStruct() bint isEnum() bint isInterface() bint isConst() - bint isAnnotation() - - cppclass Builder nogil: - - Node.Body getBody() - void setBody(Node.Body) - Text.Builder getDisplayName() - void setDisplayName(Text) - List[Node.Annotation].Builder getAnnotations() - List[Node.Annotation].Builder initAnnotations(int) - UInt64 getScopeId() - void setScopeId(UInt64) - List[Node.Node.NestedNode].Builder getNestedNodes() - List[Node.Node.NestedNode].Builder initNestedNodes(int) - UInt64 getId() - void setId(UInt64) - bint isFile() - bint isStruct() - bint isEnum() - bint isInterface() - bint isConst() - bint isAnnotation() - - cdef cppclass AnnotationNode nogil: + cdef cppclass Field nogil: cppclass Reader nogil: + Text.Reader getName() - Bool getTargetsField() - Bool getTargetsConst() - Bool getTargetsFile() - Bool getTargetsStruct() - Bool getTargetsParam() - Bool getTargetsUnion() - Bool getTargetsAnnotation() - Bool getTargetsEnumerant() - Type getType() - Bool getTargetsEnum() - Bool getTargetsInterface() - Bool getTargetsMethod() - - cppclass Builder nogil: - - Bool getTargetsField() - void setTargetsField(Bool) - Bool getTargetsConst() - void setTargetsConst(Bool) - Bool getTargetsFile() - void setTargetsFile(Bool) - Bool getTargetsStruct() - void setTargetsStruct(Bool) - Bool getTargetsParam() - void setTargetsParam(Bool) - Bool getTargetsUnion() - void setTargetsUnion(Bool) - Bool getTargetsAnnotation() - void setTargetsAnnotation(Bool) - Bool getTargetsEnumerant() - void setTargetsEnumerant(Bool) - Type getType() - void setType(Type) - Bool getTargetsEnum() - void setTargetsEnum(Bool) - Bool getTargetsInterface() - void setTargetsInterface(Bool) - Bool getTargetsMethod() - void setTargetsMethod(Bool) - - cdef cppclass EnumNode nogil: - cppclass Enumerant - - cppclass Enumerant nogil: - - cppclass Reader nogil: - - UInt16 getCodeOrder() - Text.Reader getName() - List[EnumNode.Enumerant.Annotation].Reader getAnnotations() - - cppclass Builder nogil: - - UInt16 getCodeOrder() - void setCodeOrder(UInt16) - Text.Builder getName() - void setName(Text) - List[EnumNode.Enumerant.Annotation].Builder getAnnotations() - List[EnumNode.Enumerant.Annotation].Builder initAnnotations(int) - + cdef cppclass Enumerant nogil: cppclass Reader nogil: + Text.Reader getName() - List[EnumNode.EnumNode.Enumerant].Reader getEnumerants() - - cppclass Builder nogil: - - List[EnumNode.EnumNode.Enumerant].Builder getEnumerants() - List[EnumNode.EnumNode.Enumerant].Builder initEnumerants(int) - - cdef cppclass StructNode nogil: - cppclass Union - cppclass Member - cppclass Field - - cppclass Union nogil: - - cppclass Reader nogil: - - UInt32 getDiscriminantOffset() - List[StructNode.Union.StructNode.Member].Reader getMembers() - cppclass Builder nogil: - - UInt32 getDiscriminantOffset() - void setDiscriminantOffset(UInt32) - List[StructNode.Union.StructNode.Member].Builder getMembers() - List[StructNode.Union.StructNode.Member].Builder initMembers(int) - cppclass Member nogil: - cppclass Body - - cppclass Body nogil: - - cppclass Reader nogil: - int which() - Field getFieldMember() - Union getUnionMember() - cppclass Builder nogil: - int which() - Field getFieldMember() - void setFieldMember(Field) - Union getUnionMember() - void setUnionMember(Union) - cppclass Reader nogil: - - UInt16 getOrdinal() - StructNode.Member.Body getBody() - UInt16 getCodeOrder() - Text.Reader getName() - List[StructNode.Member.Annotation].Reader getAnnotations() - cppclass Builder nogil: - - UInt16 getOrdinal() - void setOrdinal(UInt16) - StructNode.Member.Body getBody() - void setBody(StructNode.Member.Body) - UInt16 getCodeOrder() - void setCodeOrder(UInt16) - Text.Builder getName() - void setName(Text) - List[StructNode.Member.Annotation].Builder getAnnotations() - List[StructNode.Member.Annotation].Builder initAnnotations(int) - - cppclass Field nogil: - - cppclass Reader nogil: - - Value getDefaultValue() - Type getType() - UInt32 getOffset() - cppclass Builder nogil: - - Value getDefaultValue() - void setDefaultValue(Value) - Type getType() - void setType(Type) - UInt32 getOffset() - void setOffset(UInt32) - cppclass Reader nogil: - - UInt16 getDataSectionWordSize() - List[StructNode.StructNode.Member].Reader getMembers() - UInt16 getPointerSectionSize() - cppclass Builder nogil: - - UInt16 getDataSectionWordSize() - void setDataSectionWordSize(UInt16) - List[StructNode.StructNode.Member].Builder getMembers() - List[StructNode.StructNode.Member].Builder initMembers(int) - UInt16 getPointerSectionSize() - void setPointerSectionSize(UInt16) - cdef cppclass Annotation nogil: - - cppclass Reader nogil: - - UInt64 getId() - Value getValue() - cppclass Builder nogil: - - UInt64 getId() - void setId(UInt64) - Value getValue() - void setValue(Value) - cdef cppclass ListNestedNodeReader"capnp::List::Reader" nogil: - ListNestedNodeReader() - ListNestedNodeReader(ListNestedNodeReader) - Node.NestedNode.Reader operator[](uint) + cdef cppclass ListNestedNodeReader "capnp::List::Reader" nogil: uint size() + Node.NestedNode.Reader operator[](uint) + +cdef extern from "capnp/common.h" namespace " ::capnp": + cdef cppclass word nogil: + pass + +cdef extern from "kj/common.h" namespace " ::kj": + cdef cppclass WordArrayPtr " ::kj::ArrayPtr< ::capnp::word>" nogil: + WordArrayPtr(word*, size_t) + +cdef extern from "kj/array.h" namespace " ::kj": + cdef cppclass WordArray " ::kj::Array< ::capnp::word>" nogil: + word* begin() + size_t size() cdef extern from "capnp/message.h" namespace " ::capnp": cdef cppclass ReaderOptions nogil: @@ -650,177 +73,20 @@ cdef extern from "capnp/message.h" namespace " ::capnp": uint nestingLimit cdef cppclass MessageBuilder nogil: - CodeGeneratorRequest.Builder getRootCodeGeneratorRequest'getRoot< ::capnp::schema::CodeGeneratorRequest>'() - CodeGeneratorRequest.Builder initRootCodeGeneratorRequest'initRoot< ::capnp::schema::CodeGeneratorRequest>'() - InterfaceNode.Builder getRootInterfaceNode'getRoot< ::capnp::schema::InterfaceNode>'() - InterfaceNode.Builder initRootInterfaceNode'initRoot< ::capnp::schema::InterfaceNode>'() - Value.Builder getRootValue'getRoot< ::capnp::schema::Value>'() - Value.Builder initRootValue'initRoot< ::capnp::schema::Value>'() - ConstNode.Builder getRootConstNode'getRoot< ::capnp::schema::ConstNode>'() - ConstNode.Builder initRootConstNode'initRoot< ::capnp::schema::ConstNode>'() - Type.Builder getRootType'getRoot< ::capnp::schema::Type>'() - Type.Builder initRootType'initRoot< ::capnp::schema::Type>'() - FileNode.Builder getRootFileNode'getRoot< ::capnp::schema::FileNode>'() - FileNode.Builder initRootFileNode'initRoot< ::capnp::schema::FileNode>'() - Node.Builder getRootNode'getRoot< ::capnp::schema::Node>'() - Node.Builder initRootNode'initRoot< ::capnp::schema::Node>'() - AnnotationNode.Builder getRootAnnotationNode'getRoot< ::capnp::schema::AnnotationNode>'() - AnnotationNode.Builder initRootAnnotationNode'initRoot< ::capnp::schema::AnnotationNode>'() - EnumNode.Builder getRootEnumNode'getRoot< ::capnp::schema::EnumNode>'() - EnumNode.Builder initRootEnumNode'initRoot< ::capnp::schema::EnumNode>'() - StructNode.Builder getRootStructNode'getRoot< ::capnp::schema::StructNode>'() - StructNode.Builder initRootStructNode'initRoot< ::capnp::schema::StructNode>'() - Annotation.Builder getRootAnnotation'getRoot< ::capnp::schema::Annotation>'() - Annotation.Builder initRootAnnotation'initRoot< ::capnp::schema::Annotation>'() - - DynamicStruct_Builder getRootDynamicStruct'getRoot< ::capnp::DynamicStruct>'(StructSchema) except +reraise_kj_exception - DynamicStruct_Builder initRootDynamicStruct'initRoot< ::capnp::DynamicStruct>'(StructSchema) - void setRootDynamicStruct'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader) - - ConstWordArrayArrayPtr getSegmentsForOutput'getSegmentsForOutput'() - - AnyPointer.Builder getRootAnyPointer'getRoot< ::capnp::AnyPointer>'() - - DynamicOrphan newOrphan'getOrphanage().newOrphan'(StructSchema) + DynamicStruct_Builder getRootDynamicStruct 'getRoot< ::capnp::DynamicStruct>'(StructSchema) except +reraise_kj_exception + DynamicStruct_Builder initRootDynamicStruct 'initRoot< ::capnp::DynamicStruct>'(StructSchema) + void setRootDynamicStruct 'setRoot< ::capnp::DynamicStruct::Reader>'(DynamicStruct.Reader) cdef cppclass MessageReader nogil: - CodeGeneratorRequest.Reader getRootCodeGeneratorRequest'getRoot< ::capnp::schema::CodeGeneratorRequest>'() - InterfaceNode.Reader getRootInterfaceNode'getRoot< ::capnp::schema::InterfaceNode>'() - Value.Reader getRootValue'getRoot< ::capnp::schema::Value>'() - ConstNode.Reader getRootConstNode'getRoot< ::capnp::schema::ConstNode>'() - Type.Reader getRootType'getRoot< ::capnp::schema::Type>'() - FileNode.Reader getRootFileNode'getRoot< ::capnp::schema::FileNode>'() - Node.Reader getRootNode'getRoot< ::capnp::schema::Node>'() - AnnotationNode.Reader getRootAnnotationNode'getRoot< ::capnp::schema::AnnotationNode>'() - EnumNode.Reader getRootEnumNode'getRoot< ::capnp::schema::EnumNode>'() - StructNode.Reader getRootStructNode'getRoot< ::capnp::schema::StructNode>'() - Annotation.Reader getRootAnnotation'getRoot< ::capnp::schema::Annotation>'() - - DynamicStruct.Reader getRootDynamicStruct'getRoot< ::capnp::DynamicStruct>'(StructSchema) except +reraise_kj_exception - AnyPointer.Reader getRootAnyPointer'getRoot< ::capnp::AnyPointer>'() + DynamicStruct.Reader getRootDynamicStruct 'getRoot< ::capnp::DynamicStruct>'(StructSchema) except +reraise_kj_exception cdef cppclass MallocMessageBuilder(MessageBuilder) nogil: MallocMessageBuilder() MallocMessageBuilder(int) - cdef cppclass SegmentArrayMessageReader(MessageReader) nogil: - SegmentArrayMessageReader(ConstWordArrayArrayPtr array) except +reraise_kj_exception - SegmentArrayMessageReader(ConstWordArrayArrayPtr array, ReaderOptions) except +reraise_kj_exception - - cdef cppclass FlatMessageBuilder(MessageBuilder) nogil: - FlatMessageBuilder(WordArrayPtr array) - FlatMessageBuilder(WordArrayPtr array, ReaderOptions) - - enum Void: - VOID - -cdef extern from "PyCustomMessageBuilder.h" namespace " ::capnp": - cdef cppclass PyCustomMessageBuilder(MessageBuilder): - PyCustomMessageBuilder(PyObject* allocateSegmentCallable) - PyCustomMessageBuilder(PyObject* allocateSegmentCallable, int firstSegmentSize) - -cdef extern from "capnp/common.h" namespace " ::capnp": - cdef cppclass word nogil: - pass - -cdef extern from "kj/common.h" namespace " ::kj": - # Cython can't handle ArrayPtr[word] as a function argument - cdef cppclass WordArrayPtr " ::kj::ArrayPtr< ::capnp::word>" nogil: - WordArrayPtr() - WordArrayPtr(word *, size_t size) - size_t size() - word& operator[](size_t index) - cdef cppclass ByteArrayPtr " ::kj::ArrayPtr< ::capnp::byte>" nogil: - ByteArrayPtr() - ByteArrayPtr(byte *, size_t size) - size_t size() - byte& operator[](size_t index) - cdef cppclass ConstWordArrayPtr " ::kj::ArrayPtr< const ::capnp::word>" nogil: - ConstWordArrayPtr() - ConstWordArrayPtr(word *, size_t size) - size_t size() - const word* begin() - cdef cppclass ConstWordArrayArrayPtr " ::kj::ArrayPtr< const ::kj::ArrayPtr< const ::capnp::word>>" nogil: - ConstWordArrayArrayPtr() - ConstWordArrayArrayPtr(ConstWordArrayPtr*, size_t size) - size_t size() - ConstWordArrayPtr& operator[](size_t index) - -cdef extern from "kj/array.h" namespace " ::kj": - # Cython can't handle Array[word] as a function argument - cdef cppclass WordArray " ::kj::Array< ::capnp::word>" nogil: - word* begin() - size_t size() - cdef cppclass ByteArray " ::kj::Array< ::capnp::byte>" nogil: - char* begin() - size_t size() - -cdef extern from "kj/array.h" namespace " ::kj": - cdef cppclass InputStream nogil: - void read(void* buffer, size_t bytes) except +reraise_kj_exception - size_t read(void* buffer, size_t minBytes, size_t maxBytes) except +reraise_kj_exception - size_t tryRead(void* buffer, size_t minBytes, size_t maxBytes) except +reraise_kj_exception - void skip(size_t bytes) except +reraise_kj_exception - - cdef cppclass OutputStream nogil: - void write(const void* buffer, size_t size) except +reraise_kj_exception - # void write(ArrayPtr> pieces); - - cdef cppclass BufferedInputStream(InputStream) nogil: - pass - cdef cppclass BufferedOutputStream(OutputStream) nogil: - pass - - cdef cppclass BufferedInputStreamWrapper(BufferedInputStream) nogil: - BufferedInputStreamWrapper(InputStream&) - cdef cppclass BufferedOutputStreamWrapper(BufferedOutputStream) nogil: - BufferedOutputStreamWrapper(OutputStream&) - - cdef cppclass ArrayInputStream(BufferedInputStream) nogil: - ArrayInputStream(ByteArrayPtr) - ByteArrayPtr getArray() - # ByteArrayPtr tryGetReadBuffer() except +reraise_kj_exception - cdef cppclass ArrayOutputStream(BufferedOutputStream) nogil: - ArrayOutputStream(ByteArrayPtr) - ByteArrayPtr getArray() - ByteArrayPtr getWriteBuffer() - - cdef cppclass FdInputStream(InputStream) nogil: - FdInputStream(int) - cdef cppclass FdOutputStream(OutputStream) nogil: - FdOutputStream(int) - cdef extern from "capnp/serialize.h" namespace " ::capnp": - cdef cppclass InputStreamMessageReader(MessageReader) nogil: - InputStreamMessageReader(InputStream&) except +reraise_kj_exception - InputStreamMessageReader(InputStream&, ReaderOptions) except +reraise_kj_exception - cdef cppclass StreamFdMessageReader(MessageReader) nogil: - StreamFdMessageReader(int) except +reraise_kj_exception - StreamFdMessageReader(int, ReaderOptions) except +reraise_kj_exception - cdef cppclass FlatArrayMessageReader(MessageReader) nogil: - FlatArrayMessageReader(WordArrayPtr array) except +reraise_kj_exception - FlatArrayMessageReader(WordArrayPtr array, ReaderOptions) except +reraise_kj_exception + FlatArrayMessageReader(WordArrayPtr, ReaderOptions) except +reraise_kj_exception const word* getEnd() const - void writeMessageToFd(int, MessageBuilder&) except +reraise_kj_exception nogil - - WordArray messageToFlatArray(MessageBuilder &) nogil - -cdef extern from "capnp/serialize-packed.h" namespace " ::capnp": - cdef cppclass PackedInputStream(InputStream) nogil: - PackedInputStream(BufferedInputStream&) except +reraise_kj_exception - cdef cppclass PackedOutputStream(OutputStream) nogil: - PackedOutputStream(BufferedOutputStream&) except +reraise_kj_exception - - cdef cppclass PackedMessageReader(MessageReader) nogil: - PackedMessageReader(BufferedInputStream&) except +reraise_kj_exception - PackedMessageReader(BufferedInputStream&, ReaderOptions) except +reraise_kj_exception - - cdef cppclass PackedFdMessageReader(MessageReader) nogil: - PackedFdMessageReader(int) except +reraise_kj_exception - PackedFdMessageReader(int, ReaderOptions) except +reraise_kj_exception - - void writePackedMessage(BufferedOutputStream&, MessageBuilder&) except +reraise_kj_exception nogil - void writePackedMessage(OutputStream&, MessageBuilder&) except +reraise_kj_exception nogil - void writePackedMessageToFd(int, MessageBuilder&) except +reraise_kj_exception nogil + WordArray messageToFlatArray(MessageBuilder&) nogil diff --git a/capnp/lib/capnp.pxd b/capnp/lib/capnp.pxd index a4b1da7..9d161b5 100644 --- a/capnp/lib/capnp.pxd +++ b/capnp/lib/capnp.pxd @@ -3,17 +3,13 @@ from capnp.includes cimport capnp_cpp as capnp from capnp.includes cimport schema_cpp from capnp.includes.capnp_cpp cimport ( - Schema as C_Schema, StructSchema as C_StructSchema, InterfaceSchema as C_InterfaceSchema, + Schema as C_Schema, StructSchema as C_StructSchema, EnumSchema as C_EnumSchema, ListSchema as C_ListSchema, DynamicStruct as C_DynamicStruct, - DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaLoader as C_SchemaLoader, + DynamicValue as C_DynamicValue, Type as C_Type, DynamicList as C_DynamicList, SchemaParser as C_SchemaParser, ParsedSchema as C_ParsedSchema, VOID, ArrayPtr, StringPtr, - String, StringTree, DynamicOrphan as C_DynamicOrphan, AnyPointer as C_DynamicObject, - DynamicCapability as C_DynamicCapability, Request, Response, RemotePromise, Promise, - CallContext, RpcSystem, makeRpcServer, makeRpcClient, Capability as C_Capability, - TwoPartyVatNetwork as C_TwoPartyVatNetwork, Side, AsyncIoStream, Own, - DynamicStruct_Builder, PyRefCounter, PyAsyncIoStream + String, StringTree, DynamicStruct_Builder ) -from capnp.includes.schema_cpp cimport Node as C_Node, EnumNode as C_EnumNode +from capnp.includes.schema_cpp cimport Node as C_Node from capnp.includes.types cimport * from capnp.helpers cimport helpers @@ -30,9 +26,6 @@ cdef class _StringArrayPtr: cdef size_t size cdef ArrayPtr[StringPtr] asArrayPtr(self) -cdef class SchemaLoader: - cdef C_SchemaLoader * thisptr - cdef class SchemaParser: cdef C_SchemaParser * thisptr cdef public dict modules_by_id @@ -40,16 +33,6 @@ cdef class SchemaParser: cdef _StringArrayPtr _last_import_array cpdef _parse_disk_file(self, displayName, diskPath, imports) -cdef class _DynamicOrphan: - cdef C_DynamicOrphan thisptr - cdef public object _parent - - cdef _init(self, C_DynamicOrphan other, object parent) - - cdef C_DynamicOrphan move(self) - cpdef get(self) - - cdef class _DynamicStructReader: cdef C_DynamicStruct.Reader thisptr cdef public object _parent @@ -57,7 +40,7 @@ cdef class _DynamicStructReader: cdef object _obj_to_pin cdef object _schema - cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=?, bint tryRegistry=?) + cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=?) cpdef _get(self, field) cpdef _has(self, field) @@ -65,9 +48,7 @@ cdef class _DynamicStructReader: cpdef _which_str(self) cpdef _get_by_field(self, _StructSchemaField field) cpdef _has_by_field(self, _StructSchemaField field) - cpdef get_data_as_view(self, field) - - cpdef as_builder(self, num_first_segment_words=?, allocate_seg_callable=?) + cpdef as_builder(self, num_first_segment_words=?) cdef class _DynamicStructBuilder: @@ -77,15 +58,10 @@ cdef class _DynamicStructBuilder: cdef public bint _is_written cdef object _schema - cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?, bint tryRegistry=?) + cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=?) cdef _check_write(self) cpdef to_bytes(_DynamicStructBuilder self) - cpdef to_segments(_DynamicStructBuilder self) - cpdef to_segment_views(_DynamicStructBuilder self) - cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count) - cpdef to_bytes_packed(_DynamicStructBuilder self) - cpdef _get(self, field) cpdef _set(self, field, value) cpdef _has(self, field) @@ -94,15 +70,10 @@ cdef class _DynamicStructBuilder: cpdef _set_by_field(self, _StructSchemaField field, value) cpdef _has_by_field(self, _StructSchemaField field) cpdef _init_by_field(self, _StructSchemaField field, size=?) - cpdef init_resizable_list(self, field) cpdef _DynamicEnumField _which(self) cpdef _which_str(self) - cpdef adopt(self, field, _DynamicOrphan orphan) - cpdef disown(self, field) - cpdef get_data_as_view(self, field) - cpdef as_reader(self) - cpdef copy(self, num_first_segment_words=?, allocate_seg_callable=?) + cpdef copy(self, num_first_segment_words=?) cdef class _DynamicEnumField: cdef object thisptr @@ -117,15 +88,9 @@ cdef class _Schema: cpdef as_const_value(self) cpdef as_struct(self) - cpdef as_interface(self) cpdef as_enum(self) cpdef get_proto(self) -cdef class _InterfaceSchema: - cdef C_InterfaceSchema thisptr - cdef object __method_names, __method_names_inherited, __methods, __methods_inherited - cdef _init(self, C_InterfaceSchema other) - cdef class _DynamicEnum: cdef capnp.DynamicEnum thisptr cdef public object _parent @@ -141,31 +106,19 @@ cdef class _DynamicListBuilder: cpdef _get(self, int64_t index) cpdef _set(self, index, value) - cpdef adopt(self, index, _DynamicOrphan orphan) - cpdef disown(self, index) - cpdef init(self, index, size) cdef class _MessageBuilder: cdef schema_cpp.MessageBuilder * thisptr cpdef init_root(self, schema) cpdef get_root(self, schema) - cpdef get_root_as_any(self) cpdef set_root(self, value) - cpdef get_segments_for_output(self) - cpdef new_orphan(self, schema) - cdef to_python_reader(C_DynamicValue.Reader self, object parent) cdef to_python_builder(C_DynamicValue.Builder self, object parent) -cdef _to_dict(msg, bint verbose, bint ordered) +cdef _to_dict(msg, bint verbose, bint ordered, bint encode_bytes_as_base64=?) cdef _from_list(_DynamicListBuilder msg, list d) cdef _from_tuple(_DynamicListBuilder msg, tuple d) cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField field, value, parent) cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) -cdef api object wrap_dynamic_struct_reader(Response & r) with gil -cdef api Promise[void] * call_server_method( - object server, char * _method_name, CallContext & _context, object kj_loop) except * with gil -cdef api object wrap_kj_exception(capnp.Exception & exception) with gil cdef api object wrap_kj_exception_for_reraise(capnp.Exception & exception) with gil -cdef api object get_exception_info(object exc_type, object exc_obj, object exc_tb) with gil diff --git a/capnp/lib/capnp.pyx b/capnp/lib/capnp.pyx index d4e6f5c..5519392 100644 --- a/capnp/lib/capnp.pyx +++ b/capnp/lib/capnp.pyx @@ -1,6 +1,6 @@ # capnp.pyx # distutils: language = c++ -# distutils: libraries = capnpc capnp-rpc capnp kj-async kj +# distutils: libraries = capnpc capnp kj # distutils: include_dirs = . # cython: c_string_type = str # cython: c_string_encoding = default @@ -10,44 +10,25 @@ cimport cython # noqa: E402 from capnp.helpers.helpers cimport init_capnp_api -from capnp.includes.capnp_cpp cimport AsyncIoStream, WaitScope, PyPromise, VoidPromise, EventPort, EventLoop, PyAsyncIoStream, PromiseFulfiller, VoidPromiseFulfiller, tryReadMessage, writeMessage, makeException, PythonInterfaceDynamicImpl -from capnp.includes.schema_cpp cimport (MessageReader,) from builtins import memoryview as BuiltinsMemoryview -from cpython cimport array, Py_buffer, PyObject_CheckBuffer -from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_WRITABLE, PyBUF_WRITE, PyBUF_READ, PyBUF_CONTIG_RO, PyBuffer_FillInfo -from cpython.memoryview cimport PyMemoryView_FromMemory, PyMemoryView_FromObject -from cpython.bytes cimport PyBytes_FromStringAndSize +from cpython cimport Py_buffer, PyObject_CheckBuffer +from cpython.buffer cimport PyBUF_SIMPLE, PyBUF_CONTIG_RO from cpython.exc cimport PyErr_Clear -from cpython.pyport cimport PY_SSIZE_T_MAX from cython.operator cimport dereference as deref from libc.stdlib cimport malloc, free from libc.string cimport memcpy from libcpp.utility cimport move -import array -import asyncio import collections as _collections import contextlib import base64 import enum as _enum -import inspect as _inspect import os as _os -import random as _random -import socket as _socket -import sys as _sys -import threading as _threading -import traceback as _traceback import warnings as _warnings -import weakref as _weakref -import traceback as _traceback from types import ModuleType as _ModuleType -from operator import attrgetter as _attrgetter -from functools import partial as _partial -from contextlib import asynccontextmanager as _asynccontextmanager -from importlib.machinery import ModuleSpec _CAPNP_VERSION_MAJOR = capnp.CAPNP_VERSION_MAJOR _CAPNP_VERSION_MINOR = capnp.CAPNP_VERSION_MINOR @@ -56,128 +37,9 @@ _CAPNP_VERSION = capnp.CAPNP_VERSION cdef char _EMPTY_DATA_VIEW_SENTINEL = 0 -cdef dict _type_registry = {} - - -def register_type(id, klass): - _type_registry[id] = klass - - -def deregister_all_types(): - _type_registry = {} - - -# By making it public, we'll be able to call it from capabilityHelper.h -cdef api object wrap_dynamic_struct_reader(Response & r) with gil: - return _Response()._init_childptr(new Response(move(r)), None) - -cdef _find_field_order(struct_node): - return [f.name for f in sorted(struct_node.fields, key=_attrgetter('codeOrder'))] - -cdef class _VoidPromiseFulfiller: - cdef VoidPromiseFulfiller* fulfiller - - cdef _init(self, VoidPromiseFulfiller* fulfiller): - self.fulfiller = fulfiller - return self - -def void_task_done_callback(method_name, _VoidPromiseFulfiller fulfiller, task): - if fulfiller.fulfiller == NULL: - if not task.cancelled(): - exc = task.exception() - if exc is not None: - context = { - 'message': f"Cancelled server method {method_name} raised an exception", - 'exception': exc, - 'task': task, - } - asyncio.get_running_loop().call_exception_handler(context) - return - - if task.cancelled(): - fulfiller.fulfiller.reject(makeException(capnp.StringPtr( - f"Server task for method {method_name} was cancelled"))) - return - - exc = task.exception() - if exc is not None: - fulfiller.fulfiller.reject(makeException(capnp.StringPtr(''.join( - _traceback.format_exception(type(exc), exc, exc.__traceback__))))) - return - - res = task.result() - if res is not None: - fulfiller.fulfiller.reject(makeException(capnp.StringPtr( - f"Async server function ({method_name}) returned a non-none value: return = {res}"))) - else: - fulfiller.fulfiller.fulfill() - -cdef api void promise_task_add_done_callback(object task, object callback, VoidPromiseFulfiller& fulfiller): - wrapper = _VoidPromiseFulfiller()._init(&fulfiller) - task.add_done_callback(_partial(callback, wrapper)) - task._fulfiller = wrapper - -cdef api void promise_task_cancel(object task): - (<_VoidPromiseFulfiller>task._fulfiller).fulfiller = NULL - task.cancel() - -def fill_context(method_name, context, returned_data): - if returned_data is None: - return - if not isinstance(returned_data, tuple): - returned_data = (returned_data,) - names = _find_field_order(context.results.schema.node.struct) - if len(returned_data) > len(names): - raise KjException( - "Too many values returned from `{}`. Expected {} and got {}" - .format(method_name, len(names), len(returned_data))) - - results = context.results - for arg_name, arg_val in zip(names, returned_data): - setattr(results, arg_name, arg_val) - -cdef api Promise[void]* call_server_method(object server, - char * _method_name, - CallContext & _context, - object _kj_loop) except* with gil: - method_name = _method_name - kj_loop = <_EventLoop>_kj_loop - kj_loop.check() - - context = _CallContext()._init(_context) # TODO:MEMORY: invalidate this with promise chain - func = getattr(server, method_name+'_context', None) - if func is not None: - ret = func(context) - if not asyncio.iscoroutine(ret): - raise ValueError( - "Server function ({}) is not a coroutine" - .format(method_name, str(ret))) - task = asyncio.create_task(ret) - else: - async def finalize(): - params = context.params - params_dict = {name: getattr(params, name) for name in params.schema.fieldnames} - params_dict['_context'] = context - func = getattr(server, method_name) # will raise if no function found - ret = func(**params_dict) - if not asyncio.iscoroutine(ret): - raise ValueError( - "Server function ({}) is not a coroutine" - .format(method_name, str(ret))) - fill_context(method_name, context, await ret) - task = asyncio.create_task(finalize()) - - kj_loop.active_tasks.add(task) - callback = _partial(void_task_done_callback, method_name) - return new VoidPromise(helpers.taskToPromise( - capnp.heap[PyRefCounter](task), - callback)) - - cdef extern from "" namespace " ::kj": String strStructReader" ::kj::str"(C_DynamicStruct.Reader) String strStructBuilder" ::kj::str"(DynamicStruct_Builder) - String strRequest" ::kj::str"(Request &) String strListReader" ::kj::str"(C_DynamicList.Reader) String strListBuilder" ::kj::str"(C_DynamicList.Builder) String strException" ::kj::str"(capnp.Exception) @@ -282,14 +144,6 @@ class KjException(Exception): return self -cdef api object wrap_kj_exception(capnp.Exception & exception) with gil: - PyErr_Clear() - wrapper = _KjExceptionWrapper()._init(exception) - ret = KjException(wrapper=wrapper) - - return ret - - cdef api object wrap_kj_exception_for_reraise(capnp.Exception & exception) with gil: PyErr_Clear() wrapper = _KjExceptionWrapper()._init(exception) @@ -301,15 +155,6 @@ cdef void reraise_kj_exception(): helpers.reraise_kj_exception() -cdef api object get_exception_info(object exc_type, object exc_obj, object exc_tb) with gil: - try: - return (exc_tb.tb_frame.f_code.co_filename.encode(), - exc_tb.tb_lineno, - (repr(exc_type) + ":" + str(exc_obj)).encode()) - except Exception: - return (b'', 0, b"Couldn't determine python exception") - - cdef schema_cpp.ReaderOptions make_reader_opts(traversal_limit_in_words, nesting_limit): cdef schema_cpp.ReaderOptions opts if traversal_limit_in_words is not None: @@ -347,7 +192,6 @@ cdef extern from "capnp/list.h" namespace " ::capnp": cdef extern from "" namespace " ::capnp": StringTree printStructReader" ::capnp::prettyPrint"(C_DynamicStruct.Reader) except +reraise_kj_exception StringTree printStructBuilder" ::capnp::prettyPrint"(DynamicStruct_Builder) except +reraise_kj_exception - StringTree printRequest" ::capnp::prettyPrint"(Request &) except +reraise_kj_exception StringTree printListReader" ::capnp::prettyPrint"(C_DynamicList.Reader) except +reraise_kj_exception StringTree printListBuilder" ::capnp::prettyPrint"(C_DynamicList.Builder) except +reraise_kj_exception @@ -389,7 +233,7 @@ cdef class _NodeReader: property isEnum: def __get__(self): return self.thisptr.isEnum() - + property node: """A property that returns the NodeReader as a DynamicStructReader.""" def __get__(self): @@ -417,7 +261,7 @@ cdef class _DynamicListReader: have been defined properly, so you can treat this class mostly like any other iterable class:: ... - person = addressbook.Person.read(file) + person = addressbook.Person.new_message() phones = person.phones # This returns a _DynamicListReader @@ -456,83 +300,6 @@ cdef class _DynamicListReader: return '' % strListReader(self.thisptr).cStr() -cdef class _DynamicResizableListBuilder: - """Class for building growable Cap'n Proto Lists - - .. warning:: - You need to call :meth:`finish` on this object before serializing the - Cap'n Proto message. Failure to do so will cause your objects not to be - written out as well as leaking orphan structs into your message. - - This class works much like :class:`_DynamicListBuilder`, but it allows growing the list dynamically. - It is meant for lists of structs, since for primitive types like int or float, you're much better off - using a normal python list and then serializing straight to a Cap'n Proto list. - It has __getitem__ and __len__ defined, but not __setitem__:: - - ... - person = addressbook.Person.new_message() - - phones = person.init_resizable_list('phones') # This returns a _DynamicResizableListBuilder - - phone = phones.add() - phone.number = 'foo' - phone = phones.add() - phone.number = 'bar' - - phones.finish() - - f = open('example', 'w') - person.write(f) - """ - cdef public object _parent, _message, _field, _schema - cdef public list _list - - def __init__(self, parent, field, schema): - self._parent = parent - self._message = parent._parent - self._field = field - self._schema = schema - - self._list = list() - - cpdef add(self): - """A method for adding a new struct to the list - - This will return a struct, in which you can set fields that will be reflected in the serialized - Cap'n Proto message. - - :rtype: :class:`_DynamicStructBuilder` - """ - orphan = self._message.new_orphan(self._schema) - orphan_val = orphan.get() - self._list.append((orphan, orphan_val)) - return orphan_val - - cpdef _get(self, index): - return self._list[index][1] - - def __getitem__(self, index): - return self._list[index][1] - - # def __setitem__(self, index, val): - # self._list[index] = val - - def __len__(self): - return len(self._list) - - def finish(self): - """A method for closing this list and serializing all its members to the message - - If you don't call this method, the items you previously added from this object will leak into the message, - ie. inaccessible but still taking up space. - """ - cdef int i = 0 - new_list = self._parent.init(self._field, len(self)) - for orphan, _ in self._list: - new_list.adopt(i, orphan) - i += 1 - - cdef class _DynamicListBuilder: """Class for building Cap'n Proto Lists @@ -581,34 +348,6 @@ cdef class _DynamicListBuilder: def __len__(self): return self.thisptr.size() - cpdef adopt(self, index, _DynamicOrphan orphan): - """A method for adopting Cap'n Proto orphans - - Don't use this method unless you know what you're doing. - Orphans are useful for dynamically allocating objects for an unknown sized list. - - :type index: int - :param index: The index of the element in the list to replace with the newly adopted object - - :type orphan: :class:`_DynamicOrphan` - :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation. - - :rtype: void - """ - self.thisptr.adopt(index, orphan.move()) - - cpdef disown(self, index): - """A method for disowning Cap'n Proto orphans - - Don't use this method unless you know what you're doing. - - :type index: int - :param index: The index of the element in the list to disown - - :rtype: :class:`_DynamicOrphan` - """ - return _DynamicOrphan()._init(self.thisptr.disown(index), self._parent) - cpdef init(self, index, size): """A method for initializing an element in a list @@ -630,9 +369,9 @@ cdef class _DynamicListBuilder: cdef class _List_NestedNode_Reader: - cdef C_Node.NestedNode.Reader.ListNestedNodeReader thisptr - cdef _init(self, List[C_Node.NestedNode].Reader other): - self.thisptr = other + cdef schema_cpp.ListNestedNodeReader thisptr + cdef _init(self, schema_cpp.ListNestedNodeReader other): + self.thisptr = other return self def __getitem__(self, index): @@ -645,18 +384,6 @@ cdef class _List_NestedNode_Reader: def __len__(self): return self.thisptr.size() -# cdef to_python_pipeline(C_DynamicValue.Pipeline self, object parent): -# cdef int type = self.getType() -# if type == capnp.TYPE_CAPABILITY: -# return _DynamicCapabilityClient()._init(self.asCapability(), parent) -# # elif type == capnp.TYPE_STRUCT: -# # return _DynamicStructReader()._init(self.asStruct(), parent) -# elif type == capnp.TYPE_UNKNOWN: -# raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") -# else: -# raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") - - cdef to_python_reader(C_DynamicValue.Reader self, object parent): cdef int type = self.getType() if type == capnp.TYPE_BOOL: @@ -681,10 +408,6 @@ cdef to_python_reader(C_DynamicValue.Reader self, object parent): return _DynamicEnum()._init(self.asEnum(), parent) elif type == capnp.TYPE_VOID: return None - elif type == capnp.TYPE_ANY_POINTER: - return _DynamicObjectReader()._init(self.asObject(), parent) - elif type == capnp.TYPE_CAPABILITY: - return _DynamicCapabilityClient()._init(self.asCapability(), parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -715,10 +438,6 @@ cdef to_python_builder(C_DynamicValue.Builder self, object parent): return _DynamicEnum()._init(self.asEnum(), parent) elif type == capnp.TYPE_VOID: return None - elif type == capnp.TYPE_ANY_POINTER: - return _DynamicObjectBuilder()._init(self.asObject(), parent) - elif type == capnp.TYPE_CAPABILITY: - return _DynamicCapabilityClient()._init(self.asCapability(), parent) elif type == capnp.TYPE_UNKNOWN: raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") else: @@ -741,31 +460,10 @@ cdef C_DynamicValue.Reader _extract_dynamic_list_reader(_DynamicListReader value return C_DynamicValue.Reader(value.thisptr) -cdef C_DynamicValue.Reader _extract_dynamic_client(_DynamicCapabilityClient value): - return C_DynamicValue.Reader(value.thisptr) - - -cdef C_DynamicValue.Reader _extract_dynamic_server(object value): - cdef _InterfaceSchema schema = value.schema - kj_loop = C_DEFAULT_EVENT_LOOP_GETTER() - return C_DynamicValue.Reader(capnp.heap[PythonInterfaceDynamicImpl]( - schema.thisptr, - capnp.heap[PyRefCounter](value), - capnp.heap[PyRefCounter](kj_loop))) - - cdef C_DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): return C_DynamicValue.Reader(value.thisptr) -cdef C_DynamicValue.Reader _extract_any_pointer(_DynamicObjectReader value): - return C_DynamicValue.Reader(value.thisptr) - - -cdef C_DynamicValue.Reader _extract_any_pointer_builder(_DynamicObjectBuilder value): - return C_DynamicValue.Reader(value.thisptr.asReader()) - - cdef _setBytes(_DynamicSetterClasses thisptr, field, value): cdef capnp.StringPtr temp_string = capnp.StringPtr(value, len(value)) cdef C_DynamicValue.Reader temp = C_DynamicValue.Reader(temp_string) @@ -870,16 +568,8 @@ cdef _setDynamicField(_DynamicSetterClasses thisptr, field, value, parent): thisptr.set(field, _extract_dynamic_list_builder(value)) elif value_type is _DynamicListReader: thisptr.set(field, _extract_dynamic_list_reader(value)) - elif value_type is _DynamicCapabilityClient: - thisptr.set(field, _extract_dynamic_client(value)) - elif isinstance(value, _DynamicCapabilityServer): - thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) - elif value_type is _DynamicObjectReader: - thisptr.set(field, _extract_any_pointer(value)) - elif value_type is _DynamicObjectBuilder: - thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException( "Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'" @@ -928,16 +618,8 @@ cdef _setDynamicFieldWithField(DynamicStruct_Builder thisptr, _StructSchemaField thisptr.setByField(field.thisptr, _extract_dynamic_list_builder(value)) elif value_type is _DynamicListReader: thisptr.setByField(field.thisptr, _extract_dynamic_list_reader(value)) - elif value_type is _DynamicCapabilityClient: - thisptr.setByField(field.thisptr, _extract_dynamic_client(value)) - elif isinstance(value, _DynamicCapabilityServer): - thisptr.setByField(field.thisptr, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.setByField(field.thisptr, _extract_dynamic_enum(value)) - elif value_type is _DynamicObjectReader: - thisptr.set(field, _extract_any_pointer(value)) - elif value_type is _DynamicObjectBuilder: - thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException( "Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'" @@ -984,16 +666,8 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) thisptr.set(field, _extract_dynamic_list_builder(value)) elif value_type is _DynamicListReader: thisptr.set(field, _extract_dynamic_list_reader(value)) - elif value_type is _DynamicCapabilityClient: - thisptr.set(field, _extract_dynamic_client(value)) - elif isinstance(value, _DynamicCapabilityServer): - thisptr.set(field, _extract_dynamic_server(value)) elif value_type is _DynamicEnum: thisptr.set(field, _extract_dynamic_enum(value)) - elif value_type is _DynamicObjectReader: - thisptr.set(field, _extract_any_pointer(value)) - elif value_type is _DynamicObjectBuilder: - thisptr.set(field, _extract_any_pointer_builder(value)) else: raise KjException( "Tried to set field: '{}' with a value of: '{}' which is an unsupported type: '{}'" @@ -1002,7 +676,6 @@ cdef _setDynamicFieldStatic(DynamicStruct_Builder thisptr, field, value, parent) cdef _DynamicListBuilder temp_list_b cdef _DynamicListReader temp_list_r -cdef _DynamicResizableListBuilder temp_list_rb cdef _DynamicStructBuilder temp_msg_b cdef _DynamicStructReader temp_msg_r @@ -1015,11 +688,8 @@ cdef _to_dict(msg, bint verbose, bint ordered, bint encode_bytes_as_base64=False elif msg_type is _DynamicListReader: temp_list_r = msg return [_to_dict(temp_list_r._get(i), verbose, ordered, encode_bytes_as_base64) for i in range(len(msg))] - elif msg_type is _DynamicResizableListBuilder: - temp_list_rb = msg - return [_to_dict(temp_list_rb._get(i), verbose, ordered, encode_bytes_as_base64) for i in range(len(msg))] - if msg_type is _DynamicStructBuilder or isinstance(msg, _Request): + if msg_type is _DynamicStructBuilder: temp_msg_b = msg if ordered: ret = _collections.OrderedDict() @@ -1036,7 +706,7 @@ cdef _to_dict(msg, bint verbose, bint ordered, bint encode_bytes_as_base64=False ret[field] = _to_dict(temp_msg_b._get(field), verbose, ordered, encode_bytes_as_base64) return ret - elif msg_type is _DynamicStructReader or isinstance(msg, _Response): + elif msg_type is _DynamicStructReader: temp_msg_r = msg if ordered: ret = _collections.OrderedDict() @@ -1177,129 +847,9 @@ cdef class _MessageSize: self.cap_count = cap_count -@cython.internal -cdef class _BorrowedBufferView: - """Buffer-protocol exporter that pins an owner while a view borrows its memory.""" - cdef object _owner - cdef const char* _ptr - cdef Py_ssize_t _size - cdef bint _readonly - - cdef _init(self, object owner, const void* ptr, Py_ssize_t size, bint readonly): - self._owner = owner - self._ptr = ptr - self._size = size - self._readonly = readonly - return self - - def __getbuffer__(self, Py_buffer *buffer, int flags): - if PyBuffer_FillInfo(buffer, self, self._ptr, self._size, - self._readonly, flags) < 0: - raise BufferError("Failed to create borrowed buffer view") - - def __releasebuffer__(self, Py_buffer *buffer): - pass - - def __len__(self): - return self._size - - def __repr__(self): - if self._readonly: - return '' % self._size - return '' % self._size - - -cdef inline object _memoryview_borrowing(object owner, void* ptr, Py_ssize_t size, - bint readonly): - cdef _BorrowedBufferView exporter - exporter = _BorrowedBufferView()._init(owner, ptr, size, readonly) - return PyMemoryView_FromObject(exporter) - - -cdef void _data_field_ptr_reader(_DynamicStructReader self, field, - void** data_ptr, size_t* data_size) except *: - cdef C_DynamicValue.Reader val - cdef capnp.Data.Reader temp_data - - try: - val = self.thisptr.get(field) - except KjException as e: - raise e._to_python() from None - - if val.getType() != capnp.TYPE_DATA: - raise TypeError("Field '{}' is not a DATA field".format(field)) - - temp_data = val.asData() - data_ptr[0] = temp_data.begin() - data_size[0] = temp_data.size() - if data_size[0] == 0 and data_ptr[0] == NULL: - data_ptr[0] = &_EMPTY_DATA_VIEW_SENTINEL - - -cdef void _data_field_ptr_builder(_DynamicStructBuilder self, field, - void** data_ptr, size_t* data_size) except *: - cdef C_DynamicValue.Builder val - cdef capnp.Data.Builder temp_data - - try: - val = self.thisptr.get(field) - except KjException as e: - raise e._to_python() from None - - if val.getType() != capnp.TYPE_DATA: - raise TypeError("Field '{}' is not a DATA field".format(field)) - - temp_data = val.asData() - data_ptr[0] = temp_data.begin() - data_size[0] = temp_data.size() - if data_size[0] == 0 and data_ptr[0] == NULL: - data_ptr[0] = &_EMPTY_DATA_VIEW_SENTINEL - - -@cython.internal -cdef class _SegmentViews: - cdef object _builder - cdef list _views - - cdef _init(self, _MessageBuilder builder): - cdef schema_cpp.ConstWordArrayArrayPtr segments = builder.thisptr.getSegmentsForOutput() - cdef size_t i - cdef size_t word_count - cdef Py_ssize_t byte_count - - self._builder = builder - self._views = [] - for i in range(0, segments.size()): - word_count = segments[i].size() - if word_count > (PY_SSIZE_T_MAX // 8): - raise OverflowError("segment is too large to expose as a Python buffer") - byte_count = (8 * word_count) - self._views.append(_BorrowedBufferView()._init( - builder, - segments[i].begin(), - byte_count, - True)) - return self - - def __getitem__(self, index): - return self._views[index] - - def __iter__(self): - return iter(self._views) - - def __len__(self): - return len(self._views) - - def __repr__(self): - return '' % len(self) - - -if getattr(_sys, 'subversion', [''])[0] == 'PyPy': - from pickle_helper import _struct_reducer -else: - def _struct_reducer(schema_id, data): - with _global_schema_parser.modules_by_id[schema_id].from_bytes(data) as msg: - return msg +def _struct_reducer(schema_id, data): + with _global_schema_parser.modules_by_id[schema_id].from_bytes(data) as msg: + return msg cdef class _DynamicStructReader: @@ -1311,20 +861,16 @@ cdef class _DynamicStructReader: For field names that don't follow valid python naming convention for fields, use the global function :py:func:`getattr`:: - person = addressbook.Person.read(file) # This returns a _DynamicStructReader + person = addressbook.Person.new_message() # This returns a _DynamicStructReader print person.name # using . syntax print getattr(person, 'field-with-hyphens') # for names that are invalid for python, use getattr """ - cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=False, bint tryRegistry=True): + cdef _init(self, C_DynamicStruct.Reader other, object parent, bint isRoot=False): self.thisptr = other self._parent = parent self.is_root = isRoot self._schema = None - if tryRegistry and len(_type_registry) > 0: - registered_type = _type_registry.get(self.thisptr.getId(), None) - if registered_type: - return registered_type[0](self) return self cpdef _get(self, field): @@ -1347,23 +893,6 @@ cdef class _DynamicStructReader: cpdef _has_by_field(self, _StructSchemaField field): return self.thisptr.hasByField(field.thisptr) - cpdef get_data_as_view(self, field): - """Efficiently get a read-only memoryview for a DATA field without copying. - - .. warning:: - The returned memoryview *borrows* memory owned by this message. It stays valid while - the memoryview (and any object derived from it) is alive; an internal exporter pins - this reader for that duration. Do not let the message be mutated underneath an - outstanding view. - - An unset/empty DATA field yields a valid, zero-length view (it does not raise). - """ - cdef void* data_ptr - cdef size_t data_size - - _data_field_ptr_reader(self, field, &data_ptr, &data_size) - return _memoryview_borrowing(self, data_ptr, data_size, True) - cpdef _which_str(self): try: return helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr() @@ -1420,7 +949,7 @@ cdef class _DynamicStructReader: def to_dict(self, verbose=False, ordered=False, encode_bytes_as_base64=False): return _to_dict(self, verbose, ordered, encode_bytes_as_base64) - cpdef as_builder(self, num_first_segment_words=None, allocate_seg_callable=None): + cpdef as_builder(self, num_first_segment_words=None): """A method for casting this Reader to a Builder This is a copying operation with respect to the message's buffer. @@ -1428,21 +957,11 @@ cdef class _DynamicStructReader: :type num_first_segment_words: int :param num_first_segment_words: Size of the first segment to allocate (in words ie. 8 byte increments) - - :type allocate_seg_callable: Callable[[int], Buffer] - :param allocate_seg_callable: A python callable object that takes the minimum number of 8-byte - words to allocate (as an `int`) and returns any object supporting the writable buffer protocol - (e.g., `bytearray`, `memoryview`, `numpy.ndarray`). This enables custom memory allocation - strategies including shared memory. :rtype: :class:`_DynamicStructBuilder` """ - if allocate_seg_callable is None: - builder = _MallocMessageBuilder(num_first_segment_words) - return builder.set_root(self) - else: - builder = _PyCustomMessageBuilder(allocate_seg_callable, num_first_segment_words) - return builder.set_root(self) + builder = _MallocMessageBuilder(num_first_segment_words) + return builder.set_root(self) property total_size: def __get__(self): @@ -1471,22 +990,18 @@ cdef class _DynamicStructBuilder: setattr(person, 'field-with-hyphens', 'foo') # for names that are invalid for python, use setattr print getattr(person, 'field-with-hyphens') # for names that are invalid for python, use getattr """ - cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=False, bint tryRegistry=True): + cdef _init(self, DynamicStruct_Builder other, object parent, bint isRoot=False): self.thisptr = other self._parent = parent self.is_root = isRoot self._is_written = False self._schema = None - if tryRegistry and len(_type_registry) > 0: - registered_type = _type_registry.get(self.thisptr.getId(), None) - if registered_type: - return registered_type[1](self) return self cdef _check_write(self): if not self.is_root: - raise KjException("You can only call write() on the message's root struct.") + raise KjException("You can only serialize the message's root struct.") if self._is_written: _warnings.warn( "This message has already been written once. Be very careful that you're not setting " @@ -1494,58 +1009,6 @@ cdef class _DynamicStructBuilder: "(both in memory and in the serialized data). You can disable this warning by " "calling the `clear_write_flag` method of this object after every write.") - def write(self, file): - """Writes the struct's containing message to the given file object in unpacked binary format. - - This is a shortcut for calling capnp._write_message_to_fd(). This can only be called on the - message's root struct. - - :type file: file - :param file: A file or socket object (or anything with a fileno() method), open for write. - - :rtype: void - - :Raises: :exc:`KjException` if this isn't the message's root struct. - """ - self._check_write() - _write_message_to_fd(file.fileno(), self._parent) - self._is_written = True - - async def write_async(self, _AsyncIoStream stream): - """Async version of of write(). - - This is a shortcut for calling capnp._write_message_to_fd(). This can only be called on the - message's root struct. - - :type file: AsyncIoStream - :param file: The AsyncIoStream to write the message to - - :rtype: void - - :Raises: :exc:`KjException` if this isn't the message's root struct. - """ - self._check_write() - await _voidpromise_to_asyncio( - writeMessage(deref(stream.thisptr), deref((<_MessageBuilder>self._parent).thisptr))) - self._is_written = True - - def write_packed(self, file): - """Writes the struct's containing message to the given file object in packed binary format. - - This is a shortcut for calling capnp._write_packed_message_to_fd(). This can only be called on - the message's root struct. - - :type file: file - :param file: A file or socket object (or anything with a fileno() method), open for write. - - :rtype: void - - :Raises: :exc:`KjException` if this isn't the message's root struct. - """ - self._check_write() - _write_packed_message_to_fd(file.fileno(), self._parent) - self._is_written = True - cpdef to_bytes(_DynamicStructBuilder self): """Returns the struct's containing message as a Python bytes object in the unpacked binary format. @@ -1563,69 +1026,6 @@ cdef class _DynamicStructBuilder: self._is_written = True return ret - cpdef to_segments(_DynamicStructBuilder self): - """Returns the struct's containing message as a Python list of Python bytes objects. - - This copies each output segment into a Python-owned bytes object. Use - to_segment_views() for zero-copy, read-only borrowed segment views. - - NB: This is not currently supported on PyPy. - - :rtype: list - """ - self._check_write() - cdef _MessageBuilder builder = self._parent - segments = builder.get_segments_for_output() - return segments - - cpdef to_segment_views(_DynamicStructBuilder self): - """Returns the struct's containing message as zero-copy, read-only segment views. - - The returned object is a sequence of read-only buffer-protocol views, one per output - segment. Each view borrows memory owned by the message builder; the segment pointers and - sizes are captured eagerly at call time (a snapshot). - - .. warning:: - The views (and any buffer exported from them, e.g. by ``memoryview()`` or by a - consumer that holds them) keep the builder pinned and remain valid only while no - mutation happens. Do NOT mutate, re-set, reset, or reuse the builder while any view or - exported buffer is still alive -- this includes calls that may allocate (e.g. getting - an unset pointer/struct/list field). Mutating after the snapshot can grow/relocate - segments, leaving the views pointing at stale or truncated data. Sharing the views - across threads or ``await`` points while the builder may change is a data race. - - Lifetime is enforced only by buffer-protocol reference counting (memory is not freed - while a view is held); correctness of the *contents* is the caller's responsibility. - - :rtype: sequence - """ - self._check_write() - cdef _MessageBuilder builder = self._parent - return _SegmentViews()._init(builder) - - cpdef _to_bytes_packed_helper(_DynamicStructBuilder self, word_count): - cdef _MessageBuilder builder = self._parent - array = helpers.messageToPackedBytes(deref(builder.thisptr), word_count) - cdef const char* ptr = array.begin() - cdef bytes ret = ptr[:array.size()] - return ret - - cpdef to_bytes_packed(_DynamicStructBuilder self): - self._check_write() - word_count = self.total_size.word_count + 2 - - try: - ret = self._to_bytes_packed_helper(word_count) - except Exception as e: - if 'backing array was not large enough' in str(e): - word_count *= 2 - ret = self._to_bytes_packed_helper(word_count) - else: - raise - - self._is_written = True - return ret - cpdef _get(self, field): ptr = self.thisptr.get(field) return to_python_builder(ptr, self._parent) @@ -1704,30 +1104,6 @@ cdef class _DynamicStructBuilder: ptr = self.thisptr.initByField(field.thisptr, size) return to_python_builder(ptr, self._parent) - cpdef init_resizable_list(self, field): - """Method for initializing fields that are of type list (of structs) - - This version of init returns a :class:`_DynamicResizableListBuilder` that allows - you to add members one at a time (ie. if you don't know the size for sure). - This is only meant for lists of Cap'n Proto objects, since for primitive types - you can just define a normal python list and fill it yourself. - - .. warning:: - You need to call :meth:`_DynamicResizableListBuilder.finish` on the - list object before serializing the Cap'n Proto message. Failure to do - so will cause your objects not to be written out as well as leaking - orphan structs into your message. - - :type field: str - :param field: The field name to initialize - - :rtype: :class:`_DynamicResizableListBuilder` - - :Raises: :exc:`KjException` if the field isn't in this struct - """ - return _DynamicResizableListBuilder(self, field, _StructSchema()._init_child( - (self.thisptr.get(field)).asList().getStructElementType())) - cpdef _which_str(self): try: return helpers.fixMaybe(self.thisptr.which()).getProto().getName().cStr() @@ -1765,59 +1141,6 @@ cdef class _DynamicStructBuilder: def __get__(_DynamicStructBuilder self): return self._which() - cpdef adopt(self, field, _DynamicOrphan orphan): - """A method for adopting Cap'n Proto orphans - - Don't use this method unless you know what you're doing. - Orphans are useful for dynamically allocating objects for an unknown sized list. - - :type field: str - :param field: The field name in the struct - - :type orphan: :class:`_DynamicOrphan` - :param orphan: A Cap'n proto orphan to adopt. It will be unusable after this operation. - - :rtype: void - """ - self.thisptr.adopt(field, orphan.move()) - - cpdef disown(self, field): - """A method for disowning Cap'n Proto orphans - - Don't use this method unless you know what you're doing. - - :type field: str - :param field: The field name in the struct - - :rtype: :class:`_DynamicOrphan` - """ - return _DynamicOrphan()._init(self.thisptr.disown(field), self._parent) - - cpdef get_data_as_view(self, field): - """Efficiently get a writable memoryview for a DATA field without copying. - - This allows in-place modification of the underlying buffer:: - - msg.get_data_as_view('myField')[0] = 0xFF - - .. warning:: - The returned memoryview *borrows* mutable memory owned by this message builder. It is - valid while the memoryview (and any object derived from it) is alive; an internal - exporter pins this builder for that duration. Do NOT mutate, re-set, reset, or reuse the - builder while a view is outstanding -- including calls that may allocate (e.g. getting - an unset pointer/struct/list field), since those can relocate or stale the borrowed - memory. Sharing a view across threads or ``await`` points while the builder may change - is a data race. - - An unset/empty DATA field yields a valid but zero-length view (writes are no-ops); to write - into the field, initialize it to the desired size first. - """ - cdef void* data_ptr - cdef size_t data_size - - _data_field_ptr_builder(self, field, &data_ptr, &data_size) - return _memoryview_borrowing(self, data_ptr, data_size, False) - cpdef as_reader(self): """A method for casting this Builder to a Reader @@ -1832,7 +1155,7 @@ cdef class _DynamicStructBuilder: reader._obj_to_pin = self return reader - cpdef copy(self, num_first_segment_words=None, allocate_seg_callable=None): + cpdef copy(self, num_first_segment_words=None): """A method for copying this Builder This is a copying operation with respect to the message's buffer. @@ -1840,21 +1163,11 @@ cdef class _DynamicStructBuilder: :type num_first_segment_words: int :param num_first_segment_words: Size of the first segment to allocate (in words ie. 8 byte increments) - - :type allocate_seg_callable: Callable[[int], Buffer] - :param allocate_seg_callable: A python callable object that takes the minimum number of 8-byte - words to allocate (as an `int`) and returns any object supporting the writable buffer protocol - (e.g., `bytearray`, `memoryview`, `numpy.ndarray`). This enables custom memory allocation - strategies including shared memory. :rtype: :class:`_DynamicStructBuilder` """ - if allocate_seg_callable is None: - builder = _MallocMessageBuilder(num_first_segment_words) - return builder.set_root(self) - else: - builder = _PyCustomMessageBuilder(allocate_seg_callable, num_first_segment_words) - return builder.set_root(self) + builder = _MallocMessageBuilder(num_first_segment_words) + return builder.set_root(self) property schema: """A property that returns the _StructSchema object matching this writer""" @@ -1908,1080 +1221,6 @@ cdef class _DynamicStructBuilder: return _struct_reducer, (self.schema.node.id, self.to_bytes()) -cdef class _DynamicStructPipeline: - """Reads Cap'n Proto structs - - This class is almost a 1 for 1 wrapping of the Cap'n Proto C++ DynamicStruct::Pipeline. - The only difference is that instead of a `get` method, __getattr__ is overloaded and the - field name is passed onto the C++ equivalent `get`. This means you just use . syntax to - access any field. For field names that don't follow valid python naming convention for fields, - use the global function :py:func:`getattr`:: - """ - cdef C_DynamicStruct.Pipeline * thisptr - cdef public object _parent - - cdef _init(self, C_DynamicStruct.Pipeline * other, object parent): - self.thisptr = other - self._parent = parent - return self - - def __dealloc__(self): - del self.thisptr - - cpdef _get(self, field): - cdef int type = (self.thisptr.get(field)).getType() - if type == capnp.TYPE_CAPABILITY: - return _DynamicCapabilityClient()._init( - (self.thisptr.get(field)).asCapability(), self._parent) - elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init( - new C_DynamicStruct.Pipeline( - (self.thisptr.get(field)).asStruct()), self._parent) - elif type == capnp.TYPE_UNKNOWN: - raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") - else: - raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") - - def __getattr__(self, field): - try: - return self._get(field) - except KjException as e: - raise e._to_python() from None - - property schema: - """A property that returns the _StructSchema object matching this reader""" - def __get__(self): - return _StructSchema()._init_child(self.thisptr.getSchema()) - - def __dir__(self): - return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) - - # def __str__(self): - # return printStructReader(self.thisptr).flatten().cStr() - - # def __repr__(self): - # return '<%s reader %s>' % (self.schema.node.displayName, strStructReader(self.thisptr).cStr()) - - def to_dict(self, verbose=False, ordered=False, encode_bytes_as_base64=False): - return _to_dict(self, verbose, ordered, encode_bytes_as_base64) - - -cdef class _DynamicOrphan: - cdef _init(self, C_DynamicOrphan other, object parent): - self.thisptr = move(other) - self._parent = parent - return self - - cdef C_DynamicOrphan move(self): - return move(self.thisptr) - - cpdef get(self): - """Returns a python object corresponding to the DynamicValue owned by this orphan - - Use this DynamicValue to set fields inside the orphan - """ - return to_python_builder(self.thisptr.get(), self._parent) - - def __str__(self): - return str(self.get()) - - def __repr__(self): - return repr(self.get()) - - -cdef class _DynamicObjectReader: - cdef C_DynamicObject.Reader thisptr - cdef public object _parent - - cdef _init(self, C_DynamicObject.Reader other, object parent): - self.thisptr = other - self._parent = parent - return self - - cpdef as_struct(self, schema): - cdef _StructSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicStructReader()._init(self.thisptr.getAs(s._thisptr()), self._parent) - - cpdef as_interface(self, schema): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent) - - cpdef as_list(self, schema): - cdef _ListSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicListReader()._init(self.thisptr.getAsList(s.thisptr), self._parent) - - cpdef as_text(self): - return (self.thisptr.getAsText().cStr())[:] - - -cdef class _DynamicObjectBuilder: - cdef C_DynamicObject.Builder * thisptr - cdef public object _parent - - cdef _init(self, C_DynamicObject.Builder other, object parent): - self.thisptr = new C_DynamicObject.Builder(other) - self._parent = parent - return self - - def __dealloc__(self): - del self.thisptr - - cpdef as_struct(self, schema): - cdef _StructSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - ptr = s._thisptr() - return _DynamicStructBuilder()._init(self.thisptr.getAs(ptr), self._parent) - - cpdef as_interface(self, schema): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicCapabilityClient()._init(self.thisptr.getAsCapability(s.thisptr), self._parent) - - cpdef as_list(self, schema): - cdef _ListSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicListBuilder()._init(self.thisptr.getAsList(s.thisptr), self._parent) - - cpdef set(self, other): - "Set value of this object with the value of another AnyPointer::Reader. Don't use this for structs" - cdef _DynamicObjectReader reader = other - self.thisptr.set(reader.thisptr) - - cpdef set_as_text(self, text): - self.thisptr.setAsText(text) - - cpdef init_as_list(self, schema, size): - cdef _ListSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicListBuilder()._init(self.thisptr.initAsList(s.thisptr, size), self._parent) - - cpdef as_text(self): - return (self.thisptr.getAsText().cStr())[:] - - cpdef as_reader(self): - return _DynamicObjectReader()._init(self.thisptr.asReader(), self._parent) - -cdef kjloop_runnable_callback(void* data): - cdef AsyncIoEventPort *port = data - assert port.runHandle is not None - port.kjLoop.run() - -cdef cppclass AsyncIoEventPort(EventPort): - EventLoop *kjLoop - object asyncioLoop; - object runHandle; - - __init__(object asyncioLoop): - this.kjLoop = new EventLoop(deref(this)) - this.runHandle = None - this.asyncioLoop = asyncioLoop - - __dealloc__(): - if this.runHandle is not None: - this.runHandle.cancel() - del this.kjLoop - - cbool wait() except* with gil: - raise KjException("Currently you cannot wait for promises while pycapnp is running in asyncio mode. " + - "You should instead use 'await'. If you have a use-case to start the asyncio loop " + - "using wait(), please report") - - cbool poll() except* with gil: - raise KjException("Currently you cannot poll promises while pycapnp is running in asyncio mode. " + - "If you have a use-case to poll the asyncio loop using poll(), please report") - - void setRunnable(cbool runnable) except* with gil: - if runnable: - assert this.runHandle is None - us = this; - while True: - # TODO: This loop is a workaround for the following occasional nondeterministic bug - # that appears on Python 3.8 and 3.9: - # AttributeError: '_UnixSelectorEventLoop' object has no attribute 'call_soon' - # The cause of this is unknown (either a bug in our code, Cython, or Python). - # It appears to no longer exist in Python 3.10. This can be removed once 3.9 is EOL. - try: - this.runHandle = this.asyncioLoop.call_soon(lambda: kjloop_runnable_callback(us)) - break - except AttributeError: - pass - else: - assert this.runHandle is not None - this.runHandle.cancel() - this.runHandle = None - - EventLoop *getKjLoop(): - return this.kjLoop - -cdef class _EventLoop: - cdef Own[WaitScope] wait_scope - cdef Own[AsyncIoEventPort] event_port - cdef object active_streams - cdef object active_rpcs - cdef object active_tasks - cdef cbool closed - - cdef _init(self, asyncio_loop): - self.event_port = capnp.heap[AsyncIoEventPort](asyncio_loop) - kj_loop = deref(self.event_port).getKjLoop() - self.wait_scope = capnp.heap[WaitScope](deref(kj_loop)) - self.active_streams = _weakref.WeakSet() - self.active_rpcs = _weakref.WeakSet() - self.active_tasks = _weakref.WeakSet() - self.closed = False - return self - - def __dealloc__(self): - self.close() - - cdef close(self): - if not self.closed: - self.closed = True - deref(self.event_port).kjLoop.run() - self.wait_scope = Own[WaitScope]() - self.event_port = Own[AsyncIoEventPort]() - - cdef check(self): - if self.closed: - raise RuntimeError( - "The KJ event-loop is not running (on this thread). Please start it through 'capnp.kj_loop()'") - -@_asynccontextmanager -async def kj_loop(): - """Context manager for running the KJ event loop - - As long as the context manager is active it is guaranteed that the KJ event - loop is running. When the context manager is exited, the KJ event loop is - shut down properly and pending tasks are cancelled. - - :raises [RuntimeError]: If the KJ event loop is already running (on this thread). - - .. warning:: Every capnp rpc call required a running KJ event loop. - """ - asyncio_loop = asyncio.get_running_loop() - if hasattr(asyncio_loop, '_kj_loop'): - raise RuntimeError("The KJ event-loop is already running (on this thread).") - cdef _EventLoop kj_loop = _EventLoop()._init(asyncio_loop) - asyncio_loop._kj_loop = kj_loop - try: - yield - finally: - # Close any asynciostream that has not been closed - for stream in list(kj_loop.active_streams): stream.close() - - # Shut down all the RPC clients and servers - for rpc in list(kj_loop.active_rpcs): rpc.close() - - # Cancel any pending task that is a RPC call - # TODO: What if the cancellation is inhibited? - tasks = list(kj_loop.active_tasks) - for task in tasks: task.cancel() - await asyncio.gather(*tasks, return_exceptions=True) - - try: - del asyncio_loop._kj_loop - except AttributeError: pass - kj_loop.close() - -async def run(coro): - """Ensure that the coroutine runs while the KJ event loop is running - - This is a shortcut for wrapping the coroutine in a :py:meth:`capnp.kj_loop` context manager. - - :param coro: Coroutine to run - """ - async with kj_loop(): - return await coro - -cdef _EventLoop C_DEFAULT_EVENT_LOOP_GETTER(): - asyncio_loop = asyncio.get_running_loop() - kj_loop = getattr(asyncio_loop, '_kj_loop', None) - if kj_loop is None: - raise RuntimeError( - "The KJ event-loop is not running (on this thread). Please start it through 'capnp.kj_loop()'") - elif type(kj_loop) is _EventLoop: return kj_loop - else: raise RuntimeError("Someone meddled with the KJ event loop!") - - -cdef class _CallContext: - cdef CallContext * thisptr - - cdef _init(self, CallContext other): - helpers.allowCancellation(other) - self.thisptr = new CallContext(move(other)) - return self - - def __dealloc__(self): - del self.thisptr - - property params: - def __get__(self): - return _DynamicStructReader()._init(self.thisptr.getParams(), self) - - cpdef _get_results(self, uint word_count=0): - return _DynamicStructBuilder()._init(self.thisptr.getResults(), self) # TODO: pass firstSegmentWordSize - - property results: - def __get__(self): - return self._get_results() - - cpdef release_params(self): - self.thisptr.releaseParams() - - cpdef tail_call(self, _Request tailRequest): - return _voidpromise_to_asyncio(self.thisptr.tailCall(move(deref(tailRequest.thisptr_child)))) - - -cdef _promise_to_asyncio(PyPromise promise): - C_DEFAULT_EVENT_LOOP_GETTER() # Make sure the event loop is running - fut = asyncio.get_running_loop().create_future() - def success(res): return fut.set_result(res) if not fut.cancelled() else None - def exception(err): return fut.set_exception(err) if not fut.cancelled() else None - def done(fut): return fut.kjpromise.cancel() if fut.cancelled() else None - # Attach the promise to the future, so that it doesn't get destroyed - fut.kjpromise = _Promise()._init(helpers.then( - move(promise), - capnp.heap[PyRefCounter](success), - capnp.heap[PyRefCounter](exception))) - fut.add_done_callback(done) - return fut - -cdef _voidpromise_to_asyncio(VoidPromise promise): - return _promise_to_asyncio(helpers.convert_to_pypromise(move(promise))) - -cdef class _Promise: - cdef Own[PyPromise] thisptr - - cdef _init(self, PyPromise other): - self.thisptr = capnp.heap[PyPromise](move(other)) - return self - - cpdef cancel(self): - self.thisptr = Own[PyPromise]() - - -cdef class _RemotePromise: - cdef object _parent - """A pointer to a parent object that needs to be kept alive for this promise to function.""" - - cdef Own[RemotePromise] thisptr - - cdef _init(self, RemotePromise other, object parent=None): - self.thisptr = capnp.heap[RemotePromise](move(other)) - self._parent = parent - return self - - cdef _check_consumed(self): - if self.thisptr.get() == NULL: - raise KjException( - "Promise was already used in a consuming operation. You can no longer use this Promise object") - - def __await__(self): - self._check_consumed() - cdef Own[RemotePromise] thisptr = move(self.thisptr) - return _promise_to_asyncio( - helpers.convert_to_pypromise(move(deref(thisptr))) - .attach(capnp.heap[PyRefCounter](self._parent)) - ).__await__() - - cpdef _get(self, field): - self._check_consumed() - cdef int type = (self.thisptr.get().get(field)).getType() - if type == capnp.TYPE_CAPABILITY: - return _DynamicCapabilityClient()._init( - (self.thisptr.get().get(field)).asCapability(), self._parent) - elif type == capnp.TYPE_STRUCT: - return _DynamicStructPipeline()._init( - new C_DynamicStruct.Pipeline( - (self.thisptr.get().get(field)).asStruct()), self._parent) - elif type == capnp.TYPE_UNKNOWN: - raise KjException("Cannot convert type to Python. Type is unknown by capnproto library") - else: - raise KjException("Cannot convert type to Python. Type is unhandled by capnproto library") - - def __getattr__(self, field): - try: - return self._get(field) - except KjException as e: - raise e._to_python() from None - - property schema: - """A property that returns the _StructSchema object matching this reader""" - def __get__(self): - self._check_consumed() - return _StructSchema()._init_child(self.thisptr.get().getSchema()) - - def __dir__(self): - return list(set(self.schema.fieldnames + tuple(dir(self.__class__)))) - - def to_dict(self, verbose=False, ordered=False, encode_bytes_as_base64=False): - return _to_dict(self, verbose, ordered, encode_bytes_as_base64) - - cpdef cancel(self): - self.thisptr = Own[RemotePromise]() - self._parent = None # We don't need parent anymore. Setting to none allows quicker garbage collection - - -cdef class _Request(_DynamicStructBuilder): - cdef Request * thisptr_child - cdef public bint is_consumed - - cdef _init_child(self, Request other, parent): - self.thisptr_child = new Request(move(other)) - self._init(deref(self.thisptr_child), parent) - self.is_consumed = False - return self - - def __dealloc__(self): - del self.thisptr_child - - cpdef send(self): - C_DEFAULT_EVENT_LOOP_GETTER() # Make sure the event loop is running - if self.is_consumed: - raise KjException('Request has already been sent. You can only send a request once.') - self.is_consumed = True - return _RemotePromise()._init(self.thisptr_child.send(), self._parent) - - -cdef class _Response(_DynamicStructReader): - cdef Response * thisptr_child - - cdef _init_child(self, Response other, parent): - self.thisptr_child = new Response(move(other)) - self._init(deref(self.thisptr_child), parent) - return self - - def __dealloc__(self): - del self.thisptr_child - - cdef _init_childptr(self, Response * other, parent): - self.thisptr_child = other - self._init(deref(self.thisptr_child), parent) - return self - -cdef class _DynamicCapabilityServer: - pass - -cdef class _DynamicCapabilityClient: - cdef C_DynamicCapability.Client thisptr - cdef public object _parent, _cached_schema - - def __dealloc__(self): - # Needed to make Python <=3.9 happy, which seems to have trouble deallocating stack objects - # appropriately - self.thisptr = C_DynamicCapability.Client() - - cdef _init(self, C_DynamicCapability.Client other, object parent): - self.thisptr = other - self._parent = parent - return self - - cdef _init_vals(self, schema, server): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - kj_loop = C_DEFAULT_EVENT_LOOP_GETTER() - self.thisptr = C_DynamicCapability.Client( - capnp.heap[PythonInterfaceDynamicImpl]( - s.thisptr, - capnp.heap[PyRefCounter](server), - capnp.heap[PyRefCounter](kj_loop))) - self._parent = server - return self - - cpdef _find_method_args(self, method_name): - s = self.schema - meth = s.methods_inherited.get(method_name, None) - if meth is None: - raise AttributeError("Method named %s not found." % method_name) - - params = meth.param_type.node - if params.scopeId != 0: - raise KjException( - "Cannot call method `{}` with positional args, since its param struct is not " - "implicitly defined and thus does not have a set order of arguments".format(method_name)) - - return _find_field_order(params.struct) - - cdef _set_fields(self, Request * request, name, args, kwargs): - if args is not None and len(args) > 0: - arg_names = self._find_method_args(name) - if len(args) > len(arg_names): - raise KjException( - "Too many arguments passed to `{}`. Expected {} and got {}" - .format(name, len(arg_names), len(args))) - for arg_name, arg_val in zip(arg_names, args): - _setDynamicField(deref(request), arg_name, arg_val, self) - - if kwargs is not None: - for key, val in kwargs.items(): - _setDynamicField(deref(request), key, val, self) - - cpdef _send_helper(self, name, word_count, args, kwargs): - # if word_count is None: - # word_count = 0 - C_DEFAULT_EVENT_LOOP_GETTER() # Make sure the event loop is running - cdef Request * request = new Request(self.thisptr.newRequest(name)) # TODO: pass word_count - - self._set_fields(request, name, args, kwargs) - - cdef _RemotePromise result = _RemotePromise()._init(request.send(), self) - del request - return result - - cpdef _request_helper(self, name, firstSegmentWordSize, args, kwargs): - # if word_count is None: - # word_count = 0 - cdef _Request req = _Request()._init_child(self.thisptr.newRequest(name), self) - - self._set_fields(req.thisptr_child, name, args, kwargs) - - return req - - def _request(self, name, *args, word_count=None, **kwargs): - return self._request_helper(name, word_count, args, kwargs) - - def _send(self, name, *args, word_count=None, **kwargs): - return self._send_helper(name, word_count, args, kwargs) - - def __getattr__(self, name): - try: - if name.endswith('_request'): - short_name = name[:-8] - if short_name not in self.schema.method_names_inherited: - raise AttributeError('Method named %s not found' % short_name) - return _partial(self._request, short_name) - - if name not in self.schema.method_names_inherited: - raise AttributeError('Method named %s not found' % name) - return _partial(self._send, name) - except KjException as e: - raise e._to_python() from None - - cpdef upcast(self, schema): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - return _DynamicCapabilityClient()._init(self.thisptr.upcast(s.thisptr), self._parent) - - cpdef cast_as(self, schema): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - return _DynamicCapabilityClient()._init(self.thisptr.castAs(s.thisptr), self._parent) - - property schema: - """A property that returns the _InterfaceSchema object matching this client""" - def __get__(self): - if self._cached_schema is None: - self._cached_schema = _InterfaceSchema()._init(self.thisptr.getSchema()) - return self._cached_schema - - def __dir__(self): - return list(set(self.schema.method_names_inherited) | set(dir(self.__class__))) - - -cdef class _CapabilityClient: - cdef Own[C_Capability.Client] thisptr - cdef public object _parent - - cdef _init(self, Own[C_Capability.Client] other, object parent): - self.thisptr = move(other) - self._parent = parent - return self - - cpdef cast_as(self, schema): - cdef _InterfaceSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - return _DynamicCapabilityClient()._init(deref(self.thisptr).castAs(s.thisptr), self._parent) - - -cdef class _TwoPartyVatNetwork: - cdef Own[C_TwoPartyVatNetwork] thisptr - cdef _AsyncIoStream stream - - def close(self): - self.thisptr = Own[C_TwoPartyVatNetwork]() - - cdef _init(self, _AsyncIoStream stream, Side side, schema_cpp.ReaderOptions opts): - self.stream = stream - self.thisptr = capnp.heap[C_TwoPartyVatNetwork](deref(stream.thisptr), side, opts) - return self - - cpdef on_disconnect(self): - return _voidpromise_to_asyncio(deref(self.thisptr).onDisconnect()) - - -cdef class TwoPartyClient: - """ - TwoPartyClient for RPC Communication - - :param socket: AsyncIoStream - :param traversal_limit_in_words: Pointer derefence limit (see https://capnproto.org/cxx.html). - :param nesting_limit: Recursive limit when reading types (see https://capnproto.org/cxx.html). - """ - cdef object __weakref__ # Needed to make this class weak-referenceable - cdef Own[RpcSystem] thisptr - cdef _TwoPartyVatNetwork _network - cdef cbool closed - - def __dealloc__(self): - # Needed to make Python <=3.9 happy, which seems to have trouble deallocating stack objects - # appropriately - self.thisptr = Own[RpcSystem]() - - def close(self): - self.closed = True - self.thisptr = Own[RpcSystem]() - self._network.close() - - def __init__(self, socket=None, traversal_limit_in_words=None, nesting_limit=None): - cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() - loop.active_rpcs.add(self) - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - if isinstance(socket, _AsyncIoStream): - self._network = _TwoPartyVatNetwork()._init(socket, capnp.CLIENT, opts) - else: - raise ValueError(f"Argument socket should be a AsyncIoStream, was {type(socket)}") - - self.thisptr = capnp.heap[RpcSystem](makeRpcClient(deref(self._network.thisptr))) - - cpdef bootstrap(self): - if self.closed: - raise RuntimeError("This client is closed") - return _CapabilityClient()._init(helpers.bootstrapHelper(deref(self.thisptr)), self) - - cpdef on_disconnect(self): - if self.closed: - raise RuntimeError("This client is closed") - return self._network.on_disconnect() - - -cdef class TwoPartyServer: - """ - TwoPartyServer for RPC Communication - - :param socket: AsyncIoStream - :param bootstrap: Class object defining the implementation of the Cap'n'proto interface. - :param traversal_limit_in_words: Pointer derefence limit (see https://capnproto.org/cxx.html). - :param nesting_limit: Recursive limit when reading types (see https://capnproto.org/cxx.html). - """ - cdef object __weakref__ # Needed to make this class weak-referenceable - cdef Own[RpcSystem] thisptr - cdef _TwoPartyVatNetwork _network - cdef cbool closed - - def __dealloc__(self): - # Needed to make Python <=3.9 happy, which seems to have trouble deallocating stack objects - # appropriately - self.thisptr = Own[RpcSystem]() - - def close(self): - self.closed = True - self.thisptr = Own[RpcSystem]() - self._network.close() - - def __init__(self, socket=None, bootstrap=None, traversal_limit_in_words=None, nesting_limit=None): - cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() - loop.active_rpcs.add(self) - if not bootstrap: - raise KjException("You must provide a bootstrap interface to a server constructor.") - - opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - if isinstance(socket, _AsyncIoStream): - self._network = _TwoPartyVatNetwork()._init(socket, capnp.SERVER, opts) - else: - raise ValueError(f"Argument socket should be a AsyncIoStream, was {type(socket)}") - - cdef _InterfaceSchema schema = bootstrap.schema - self.thisptr = capnp.heap[RpcSystem](makeRpcServer( - deref(self._network.thisptr), - C_DynamicCapability.Client(capnp.heap[PythonInterfaceDynamicImpl]( - schema.thisptr, - capnp.heap[PyRefCounter](bootstrap), - capnp.heap[PyRefCounter](loop))))) - - cpdef bootstrap(self): - if self.closed: - raise RuntimeError("This server is closed") - return _CapabilityClient()._init(helpers.bootstrapHelperServer(deref(self.thisptr)), self) - - cpdef on_disconnect(self): - if self.closed: - raise RuntimeError("This server is closed") - return _voidpromise_to_asyncio(deref(self._network.thisptr).onDisconnect() - .attach(capnp.heap[PyRefCounter](self))) - - -cdef class _AsyncIoStream: - cdef object __weakref__ # Needed to make this class weak-referenceable - cdef Own[AsyncIoStream] thisptr - cdef cbool close_called - cdef object protocol - - def __init__(self): - cdef _EventLoop loop = C_DEFAULT_EVENT_LOOP_GETTER() - loop.active_streams.add(self) - self.close_called = False - - def _post_init(self, protocol): - if not self.close_called: - self.thisptr = capnp.heap[PyAsyncIoStream]( - capnp.heap[PyRefCounter](protocol)) - self.protocol = protocol - else: - protocol.transport.close() - - def __dealloc__(self): - # Needed to make Python <=3.9 happy, which seems to have trouble deallocating stack objects - # appropriately - self.thisptr = Own[AsyncIoStream]() - - def close(self): - if self.protocol is None: # _post_init wasn't called yet - self.close_called = True - elif self.protocol.transport is not None and hasattr(self.protocol.transport, "close"): - self.protocol.transport.close() - # Call connection_lost immediately, instead of waiting for the transport to do it. - # TODO: This might be a questionable thing to do... - self.protocol.connection_lost("Stream is closing") - - async def wait_closed(self): - return await self.protocol.closed_future - - @staticmethod - async def create_connection(host = None, port = None, **kwargs): - """Create a TCP connection. - - All parameters given to this function are passed to `asyncio.get_running_loop().create_connection()`. - See that function for documentation on the possible arguments. - """ - cdef _AsyncIoStream self = _AsyncIoStream() - loop = asyncio.get_running_loop() - transport, protocol = await loop.create_connection( - lambda: _PyAsyncIoStreamProtocol(), host, port, **kwargs) - self._post_init(protocol) - return self - - @staticmethod - async def create_unix_connection(path = None, **kwargs): - """Create a Unix socket connection. - - All parameters given to this function are passed to `asyncio.get_running_loop().create_unix_connection()`. - See that function for documentation on the possible arguments. - """ - cdef _AsyncIoStream self = _AsyncIoStream() - loop = asyncio.get_running_loop() - transport, protocol = await loop.create_unix_connection( - lambda: _PyAsyncIoStreamProtocol(), path, **kwargs) - self._post_init(protocol) - return self - - @staticmethod - def _connect(callback): - cdef _AsyncIoStream self = _AsyncIoStream() - loop = asyncio.get_running_loop() - protocol = _PyAsyncIoStreamProtocol(callback, self) - self._post_init(protocol) - return protocol - - @staticmethod - async def create_server(callback, host = None, port = None, **kwargs): - """Create a TCP connection server. - - The `callback` parameter will be called whenever a new connection is made. It receives a `AsyncIoStream` - instance as its only argument. If the result of `callback` is a coroutine, it will be scheduled as a task. - - This function behaves similarly to `asyncio.get_running_loop().create_server()`. All arguments except - for `callback` will be passed directly to that function, and the server returned is similar as well. - See that function for documentation on the possible arguments. - """ - # Fail early in case the kj loop is not running. Without this, the error is thrown when a connection is made. - # Unless asyncio is in debug mode, that error is swallowed. - C_DEFAULT_EVENT_LOOP_GETTER() - loop = asyncio.get_running_loop() - return await loop.create_server(lambda: _AsyncIoStream._connect(callback), host, port, **kwargs) - - @staticmethod - async def create_unix_server(callback, path = None, **kwargs): - """Create a unix connection server. - - The `callback` parameter will be called whenever a new connection is made. It receives a `AsyncIoStream` - instance as its only argument. If the result of `callback` is a coroutine, it will be scheduled as a task. - - This function behaves similarly to `asyncio.get_running_loop().create_server()`. All arguments except - for `callback` will be passed directly to that function, and the server returned is similar as well. - See that function for documentation on the possible arguments. - """ - # Fail early in case the kj loop is not running. Without this, the error is thrown when a connection is made. - # Unless asyncio is in debug mode, that error is swallowed. - C_DEFAULT_EVENT_LOOP_GETTER() - loop = asyncio.get_running_loop() - return await loop.create_unix_server(lambda: _AsyncIoStream._connect(callback), path, **kwargs) - -cdef class DummyBaseClass: - pass - -cdef class _PyAsyncIoStreamProtocol(DummyBaseClass, asyncio.BufferedProtocol): - cdef object _task - - cdef public object transport - cdef object connected_callback - cdef object callback_arg - - # State for reading data from the transport - cdef char* read_buffer - cdef int32_t read_min_bytes - cdef size_t read_max_bytes - cdef size_t read_already_read - cdef PromiseFulfiller[size_t]* read_fulfiller - cdef cbool read_eof - - # TODO: Temporary. This is an overflow buffer, which is needed for two blatant violations of the protocol. - # The first violation is in the the SSL transport implementation. - # See https://github.com/python/cpython/issues/89322, fixed in Python 3.11. This bug causes the - # SSL transport to force data upon us even when we've asked it to pause sending us data. Therefore, - # we have to store the data in a overflow buffer. - # - # The second violation is that a transport cannot be paused immediately after it is connected. - # See https://github.com/python/cpython/issues/103607. This also causes the need to be prepared - # for unexpected data. - # - # This extra code can be removed once both bugs are fixed in all supported python versions. - cdef bytearray read_overflow_buffer - cdef bytearray read_overflow_buffer_current - - # State for writing data to the transport - cdef cbool write_paused - cdef cbool write_in_progress - cdef ArrayPtr[const ArrayPtr[const uint8_t]] write_pieces - cdef size_t write_index - cdef VoidPromiseFulfiller* write_fulfiller - - def __init__(self, connected_callback = None, callback_arg = None): - self.connected_callback = connected_callback - self.callback_arg = callback_arg - - def connection_made(self, transport): - self.transport = transport - - # TODO: BUG. We want to immediately pause reading, but Python's transport implementation does not - # allow this. See https://github.com/python/cpython/issues/103607. - # To work around this, we also insert pause_reading() in get_buffer() when appropriate. - transport.pause_reading() - - self.write_paused = False - self.write_in_progress = False - self.read_eof = False - self.read_overflow_buffer = bytearray() - def done(task): - if self.transport is not None: - self.transport.close() - exc = task.exception() - if exc is not None: - context = { - 'message': "Exception in pycapnp server callback", - 'exception': exc, - 'task': task, - 'protocol': self, - 'transport': self.transport - } - asyncio.get_running_loop().call_exception_handler(context) - if self.connected_callback is not None: - callback_res = self.connected_callback(self.callback_arg) - if asyncio.iscoroutine(callback_res): - self._task = asyncio.create_task(callback_res) - self._task.add_done_callback(done) - self.connected_callback = None - self.callback_arg = None - - def connection_lost(self, exc): - if self.read_fulfiller != NULL: - capnp.rejectDisconnected[size_t](deref(self.read_fulfiller), StringPtr(str(exc))) - self.read_buffer = NULL - self.read_fulfiller = NULL - if self.write_fulfiller != NULL: - capnp.rejectVoidDisconnected(deref(self.write_fulfiller), StringPtr(str(exc))) - self.write_reset() - self.write_paused = True - self.transport = None - self._task = None - - def get_buffer(self, size_hint): - if self.read_buffer == NULL: # Should not happen, but for SSL it does, see comment above - - # TODO: Bug. Workaround for the transport ignoring pause_reading() in connection_made() - self.transport.pause_reading() - - size = size_hint if size_hint > 0 else 100 - self.read_overflow_buffer_current = bytearray(size) - return self.read_overflow_buffer_current - else: - return PyMemoryView_FromMemory(self.read_buffer, self.read_max_bytes, PyBUF_WRITE) - - def buffer_updated(self, size): - if self.read_buffer == NULL: # Should not happen, but for SSL it does, see comment above - self.read_overflow_buffer.extend(self.read_overflow_buffer_current[0:size]) - else: - self.read_buffer += size - self.read_min_bytes -= size - self.read_max_bytes -= size - self.read_already_read += size - if self.read_min_bytes <= 0: - self.read_fulfiller.fulfill(move(self.read_already_read)) - self.read_reset() - - def pause_writing(self): - self.write_paused = True - - def resume_writing(self): - self.write_paused = False - self.write_loop() - - def eof_received(self): - self.read_eof = True - if self.read_buffer != NULL: - self.read_fulfiller.fulfill(move(self.read_already_read)) - self.read_reset() - - cdef write_loop(self): - if self.write_paused or not self.write_in_progress: return - cdef const ArrayPtr[const uint8_t]* piece - for i in range(self.write_index, self.write_pieces.size()): - piece = &self.write_pieces[i] - # Copy data to Python bytes to avoid use-after-free. - # transport.write() is non-blocking and buffers data asynchronously. - # The memoryview would point to C++ memory that gets freed when - # fulfill() is called below, but asyncio may not have sent the data - # yet, causing memory corruption with large payloads. - data = PyBytes_FromStringAndSize(piece.begin(), piece.size()) - self.transport.write(data) - if self.write_paused: - self.write_index = i+1 - break - if not self.write_paused: - self.write_fulfiller.fulfill() - self.write_reset() - - cdef read_reset(self): - self.transport.pause_reading() - self.read_buffer = NULL - self.read_fulfiller = NULL - - cdef write_reset(self): - self.write_in_progress = False - self.write_fulfiller = NULL - - -cdef api void _asyncio_stream_write_start( - object thisptr, ArrayPtr[const ArrayPtr[const uint8_t]] pieces, - VoidPromiseFulfiller& fulfiller) except*: - cdef _PyAsyncIoStreamProtocol self = <_PyAsyncIoStreamProtocol>thisptr - if self.transport is None or self.transport.is_closing(): - capnp.rejectVoidDisconnected(fulfiller, StringPtr("Socket is closing.")) - return - self.write_pieces = pieces - self.write_index = 0 - self.write_fulfiller = &fulfiller - self.write_in_progress = True - self.write_loop() - -cdef api void _asyncio_stream_write_stop(object thisptr): - (<_PyAsyncIoStreamProtocol>thisptr).write_reset() - -cdef api void _asyncio_stream_read_start( - object thisptr, void* buffer, size_t min_bytes, size_t max_bytes, - PromiseFulfiller[size_t]& fulfiller) except*: - cdef _PyAsyncIoStreamProtocol self = <_PyAsyncIoStreamProtocol>thisptr - if self.transport is None or self.transport.is_closing(): - capnp.rejectDisconnected(fulfiller, StringPtr("Socket is closing")) - return - if self.read_eof: - self.read_fulfiller.fulfill(0) - return - self.read_buffer = buffer - self.read_min_bytes = min_bytes - self.read_max_bytes = max_bytes - self.read_already_read = 0 - self.read_fulfiller = &fulfiller - - # Begin of draining the overflow buffer, which is created because of a bug in SSL, see comment above. - # Can be removed once Python < 3.11 is not longer supported. - if self.read_overflow_buffer: - to_copy = min(len(self.read_overflow_buffer), max_bytes) - memcpy(buffer, self.read_overflow_buffer, to_copy) - del self.read_overflow_buffer[:to_copy] - self.read_buffer += to_copy - self.read_min_bytes -= to_copy - self.read_max_bytes -= to_copy - self.read_already_read += to_copy - if self.read_min_bytes <= 0: - self.read_fulfiller.fulfill(move(self.read_already_read)) - self.read_reset() - return # resume_reading no longer needed - # End of draining the overflow buffer. - - self.transport.resume_reading() - -cdef api void _asyncio_stream_read_stop(object thisptr): - cdef _PyAsyncIoStreamProtocol self = <_PyAsyncIoStreamProtocol>thisptr - if self.transport is not None: self.read_reset() - -cdef api void _asyncio_stream_shutdown_write(object thisptr) except*: - cdef _PyAsyncIoStreamProtocol self = <_PyAsyncIoStreamProtocol>thisptr - if self.transport is not None and self.transport.can_write_eof(): - self.transport.write_eof() - -cdef api void _asyncio_stream_close(object thisptr) except*: - cdef _PyAsyncIoStreamProtocol self = <_PyAsyncIoStreamProtocol>thisptr - # Careful, the transport object may have already been partially destroyed here. - if self.transport is not None and hasattr(self.transport, "close"): - self.transport.close() - - cdef class _Schema: cdef _init(self, C_Schema other): self.thisptr = other @@ -2994,9 +1233,6 @@ cdef class _Schema: cpdef as_struct(self): return _StructSchema()._init_child(self.thisptr.asStruct()) - cpdef as_interface(self): - return _InterfaceSchema()._init(self.thisptr.asInterface()) - cpdef as_enum(self): return _EnumSchema()._init(self.thisptr.asEnum()) @@ -3101,9 +1337,7 @@ cdef class _StructSchema(_Schema): cdef typeAsSchema(capnp.SchemaType fieldType): # TODO(soon): make sure this is memory safe - if fieldType.isInterface(): - return _InterfaceSchema()._init(fieldType.asInterface()) - elif fieldType.isStruct(): + if fieldType.isStruct(): return _StructSchema()._init_child(fieldType.asStruct()) elif fieldType.isEnum(): return _EnumSchema()._init(fieldType.asEnum()) @@ -3133,105 +1367,6 @@ cdef class _StructSchemaField: return '' % self.proto.name -cdef class _InterfaceMethod: - cdef C_InterfaceSchema.Method thisptr - - cdef _init(self, C_InterfaceSchema.Method other): - self.thisptr = other - return self - - property param_type: - """The type of this method's parameter struct""" - def __get__(self): - # TODO(soon): make sure this is memory safe - return _StructSchema()._init_child(self.thisptr.getParamType()) - - property result_type: - """The type of this method's result struct""" - def __get__(self): - # TODO(soon): make sure this is memory safe - return _StructSchema()._init_child(self.thisptr.getResultType()) - - -cdef class _InterfaceSchema: - cdef _init(self, C_InterfaceSchema other): - self.thisptr = other - return self - - property method_names: - """A tuple of the function names in the interface.""" - def __get__(self): - if self.__method_names is not None: - return self.__method_names - fieldlist = self.thisptr.getMethods() - nfields = fieldlist.size() - self.__method_names = tuple( - fieldlist[i].getProto().getName().cStr() for i in xrange(nfields)) - return self.__method_names - - property method_names_inherited: - """A set of the function names in the interface, including inherited methods""" - def __get__(self): - if self.__method_names_inherited is not None: - return self.__method_names_inherited - - fieldlist = self.thisptr.getMethods() - nfields = fieldlist.size() - self.__method_names_inherited = set( - fieldlist[i].getProto().getName().cStr() for i in xrange(nfields)) - for interface in self.superclasses: - self.__method_names_inherited |= interface.method_names_inherited - - return self.__method_names_inherited - - property methods: - """A mapping of method names to their respective _InterfaceMethod""" - def __get__(self): - if self.__methods is not None: - return self.__methods - - fieldlist = self.thisptr.getMethods() - nfields = fieldlist.size() - # TODO(soon): make sure this is memory safe - self.__methods = { - fieldlist[i].getProto().getName().cStr(): _InterfaceMethod()._init(fieldlist[i]) - for i in xrange(nfields) - } - return self.__methods - - property methods_inherited: - """A mapping of method names to their respective _InterfaceMethod, including inherited methods""" - def __get__(self): - if self.__methods_inherited is not None: - return self.__methods_inherited - - fieldlist = self.thisptr.getMethods() - nfields = fieldlist.size() - # TODO(soon): make sure this is memory safe - self.__methods_inherited = { - fieldlist[i].getProto().getName().cStr(): _InterfaceMethod()._init(fieldlist[i]) - for i in xrange(nfields) - } - for interface in self.superclasses: - self.__methods_inherited.update(interface.methods_inherited) - - return self.__methods_inherited - - property superclasses: - """A list of superclasses for this interface""" - def __get__(self): - cdef C_InterfaceSchema.SuperclassList classes = self.thisptr.getSuperclasses() - return [_InterfaceSchema()._init(classes[i]) for i in range(classes.size())] - - property node: - """The raw schema node""" - def __get__(self): - return _DynamicStructReader()._init(self.thisptr.getProto(), self) - - def __repr__(self): - return '' % self.node.displayName - - cdef class _EnumSchema: cdef C_EnumSchema thisptr @@ -3256,123 +1391,9 @@ cdef class _EnumSchema: return _DynamicStructReader()._init(self.thisptr.getProto(), self) -cdef class _SchemaType: - cdef capnp.SchemaType thisptr - - -types = _ModuleType('capnp.types') -cdef _SchemaType _void = _SchemaType() -_void.thisptr = capnp.SchemaType(capnp.TypeWhichVOID) -types.Void = _void - -cdef _SchemaType _bool = _SchemaType() -_bool.thisptr = capnp.SchemaType(capnp.TypeWhichBOOL) -types.Bool = _bool - -cdef _SchemaType _int8 = _SchemaType() -_int8.thisptr = capnp.SchemaType(capnp.TypeWhichINT8) -types.Int8 = _int8 - -cdef _SchemaType _int16 = _SchemaType() -_int16.thisptr = capnp.SchemaType(capnp.TypeWhichINT16) -types.Int16 = _int16 - -cdef _SchemaType _int32 = _SchemaType() -_int32.thisptr = capnp.SchemaType(capnp.TypeWhichINT32) -types.Int32 = _int32 - -cdef _SchemaType _int64 = _SchemaType() -_int64.thisptr = capnp.SchemaType(capnp.TypeWhichINT64) -types.Int64 = _int64 - -cdef _SchemaType _uint8 = _SchemaType() -_uint8.thisptr = capnp.SchemaType(capnp.TypeWhichUINT8) -types.UInt8 = _uint8 - -cdef _SchemaType _uint16 = _SchemaType() -_uint16.thisptr = capnp.SchemaType(capnp.TypeWhichUINT16) -types.UInt16 = _uint16 - -cdef _SchemaType _uint32 = _SchemaType() -_uint32.thisptr = capnp.SchemaType(capnp.TypeWhichUINT32) -types.UInt32 = _uint32 - -cdef _SchemaType _uint64 = _SchemaType() -_uint64.thisptr = capnp.SchemaType(capnp.TypeWhichUINT64) -types.UInt64 = _uint64 - -cdef _SchemaType _float32 = _SchemaType() -_float32.thisptr = capnp.SchemaType(capnp.TypeWhichFLOAT32) -types.Float32 = _float32 - -cdef _SchemaType _float64 = _SchemaType() -_float64.thisptr = capnp.SchemaType(capnp.TypeWhichFLOAT64) -types.Float64 = _float64 - -cdef _SchemaType _text = _SchemaType() -_text.thisptr = capnp.SchemaType(capnp.TypeWhichTEXT) -types.Text = _text - -cdef _SchemaType _data = _SchemaType() -_data.thisptr = capnp.SchemaType(capnp.TypeWhichDATA) -types.Data = _data - -# cdef _SchemaType _list = _SchemaType() -# _list.thisptr = capnp.SchemaType(capnp.TypeWhichLIST) -# types.list = _list - -# cdef _SchemaType _enum = _SchemaType() -# _enum.thisptr = capnp.SchemaType(capnp.TypeWhichENUM) -# types.Enum = _enum - -# cdef _SchemaType _struct = _SchemaType() -# _struct.thisptr = capnp.SchemaType(capnp.TypeWhichSTRUCT) -# types.struct = _struct - -# cdef _SchemaType _interface = _SchemaType() -# _interface.thisptr = capnp.SchemaType(capnp.TypeWhichINTERFACE) -# types.interface = _interface - -cdef _SchemaType _any_pointer = _SchemaType() -_any_pointer.thisptr = capnp.SchemaType(capnp.TypeWhichANY_POINTER) -types.AnyPointer = _any_pointer - - cdef class _ListSchema: cdef C_ListSchema thisptr - def __init__(self, schema=None): - cdef _StructSchema ss - cdef _EnumSchema es - cdef _InterfaceSchema iis - cdef _ListSchema ls - cdef _SchemaType st - - if schema is not None: - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - - typeSchema = type(s) - if typeSchema is _StructSchema: - ss = s - self.thisptr = capnp.listSchemaOfStruct(ss._thisptr()) - elif typeSchema is _EnumSchema: - es = s - self.thisptr = capnp.listSchemaOfEnum(es.thisptr) - elif typeSchema is _InterfaceSchema: - iis = s - self.thisptr = capnp.listSchemaOfInterface(iis.thisptr) - elif typeSchema is _ListSchema: - ls = s - self.thisptr = capnp.listSchemaOfList(ls.thisptr) - elif typeSchema is _SchemaType: - st = s - self.thisptr = capnp.listSchemaOfType(st.thisptr) - else: - raise KjException("Unknown schema type") - cdef _init(self, C_ListSchema other): self.thisptr = other return self @@ -3400,12 +1421,9 @@ class _StructABCMeta(type): return isinstance(obj, cls.__base__) and obj.schema == cls._schema -cdef _new_message(self, kwargs, num_first_segment_words, allocate_seg_callable): +cdef _new_message(self, kwargs, num_first_segment_words): cdef _MessageBuilder builder - if allocate_seg_callable is None: - builder = _MallocMessageBuilder(num_first_segment_words) - else: - builder = _PyCustomMessageBuilder(allocate_seg_callable, num_first_segment_words) + builder = _MallocMessageBuilder(num_first_segment_words) msg = builder.init_root(self.schema) if kwargs is not None: msg.from_dict(kwargs) @@ -3449,103 +1467,6 @@ class _StructModule(object): sub_module = _StructModuleWhich("StructModuleWhich", mapping) setattr(self, 'Union', sub_module) - def read(self, file, traversal_limit_in_words=None, nesting_limit=None): - """Returns a Reader for the unpacked object read from file. - - :type file: file - :param file: A python file-like object. It must be a "real" file, with a `fileno()` method. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: :class:`_DynamicStructReader`""" - reader = _StreamFdMessageReader(file, traversal_limit_in_words, nesting_limit) - return reader.get_root(self.schema) - - async def read_async(self, _AsyncIoStream stream, traversal_limit_in_words=None, nesting_limit=None): - """Async version of read(). Returns either a message, or None in case of EOF. - - :type file: AsyncIoStream - :param file: A AsyncIoStream - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: :class:`_DynamicStructReader`""" - C_DEFAULT_EVENT_LOOP_GETTER() # Make sure the event loop is running - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - reader = await _promise_to_asyncio(tryReadMessage(deref(stream.thisptr), opts)) - if reader is None: - return - return reader.get_root(self.schema) - - def read_multiple(self, file, traversal_limit_in_words=None, nesting_limit=None, skip_copy=False): - """Returns an iterable, that when traversed will return Readers for messages. - - :type file: file - :param file: A python file-like object. It must be a "real" file, with a `fileno()` method. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :type skip_copy: bool - :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is - never read completely. Skip this only if you know what you're doing. - - :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultipleMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) - return reader - - def read_packed(self, file, traversal_limit_in_words=None, nesting_limit=None): - """Returns a Reader for the packed object read from file. - - :type file: file - :param file: A python file-like object. It must be a "real" file, with a `fileno()` method. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: :class:`_DynamicStructReader`""" - reader = _PackedFdMessageReader(file, traversal_limit_in_words, nesting_limit) - return reader.get_root(self.schema) - - def read_multiple_packed(self, file, traversal_limit_in_words=None, nesting_limit=None, skip_copy=False): - """Returns an iterable, that when traversed will return Readers for messages. - - :type file: file - :param file: A python file-like object. It must be a "real" file, with a `fileno()` method. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :type skip_copy: bool - :param skip_copy: By default, each message is copied because the file needs to advance, even if the message is - never read completely. Skip this only if you know what you're doing. - - :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultiplePackedMessageReader(file, self.schema, traversal_limit_in_words, nesting_limit, skip_copy) - return reader - def read_multiple_bytes(self, buf, traversal_limit_in_words=None, nesting_limit=None): """Returns an iterable, that when traversed will return Readers for messages. @@ -3563,23 +1484,6 @@ class _StructModule(object): reader = _MultipleBytesMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) return reader - def read_multiple_bytes_packed(self, buf, traversal_limit_in_words=None, nesting_limit=None): - """Returns an iterable, that when traversed will return Readers for messages. - - :type buf: buffer - :param buf: Any Python object that supports the buffer interface. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - reader = _MultipleBytesPackedMessageReader(buf, self.schema, traversal_limit_in_words, nesting_limit) - return reader - @contextlib.contextmanager def from_bytes(self, buf, traversal_limit_in_words=None, nesting_limit=None, builder=False): """Returns a Reader for the unpacked object in buf. @@ -3597,14 +1501,13 @@ class _StructModule(object): :type builder: bool :param builder: If true, return a builder object. - Enabling `builder` will allow you to change the contents of `buf`, so do this with care. + Enabling `builder` returns a writable copy of the message. :rtype: :class:`_DynamicStructReader` or :class:`_DynamicStructBuilder` """ message = None try: if builder: - # message = _FlatMessageBuilder(buf) message = _FlatArrayMessageReader(buf, traversal_limit_in_words, nesting_limit) yield message.get_root(self.schema).as_builder() else: @@ -3614,49 +1517,15 @@ class _StructModule(object): if message: message.close() - def from_segments(self, segments, traversal_limit_in_words=None, nesting_limit=None): - """Returns a Reader for a list of segment bytes. - - This avoids making copies. - - NB: This is not currently supported on PyPy. - - :rtype: list - """ - message = _SegmentArrayMessageReader(segments, traversal_limit_in_words, nesting_limit) - return message.get_root(self.schema) - - def from_bytes_packed(self, buf, traversal_limit_in_words=None, nesting_limit=None): - """Returns a Reader for the packed object in buf. - - :type buf: buffer - :param buf: Any Python object that supports the readable buffer interface. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: :class:`_DynamicStructReader` - """ - return _PackedMessageReaderBytes(buf, traversal_limit_in_words, nesting_limit).get_root(self.schema) - def __call__(self, num_first_segment_words=None, **kwargs): return self.new_message(num_first_segment_words=num_first_segment_words, **kwargs) - def new_message(self, num_first_segment_words=None, allocate_seg_callable=None, **kwargs): + def new_message(self, num_first_segment_words=None, **kwargs): """Returns a newly allocated builder message. :type num_first_segment_words: int :param num_first_segment_words: Size of the first segment to allocate (in words ie. 8 byte increments) - :type allocate_seg_callable: Callable[[int], bytearray] - :param allocate_seg_callable: A python callable object that takes the minimum number of 8-byte - words to allocate (as an `int`) and returns a `bytearray`. This is used to customize the memory - allocation strategy. - :type kwargs: dict :param kwargs: A list of fields and their values to initialize in the struct. @@ -3665,18 +1534,7 @@ class _StructModule(object): :rtype: :class:`_DynamicStructBuilder` """ - return _new_message(self, kwargs, num_first_segment_words, allocate_seg_callable) - - -class _InterfaceModule(object): - def __init__(self, schema, name): - def server_init(server_self): - pass - self.schema = schema - self.Server = type(name + '.Server', (_DynamicCapabilityServer,), {'__init__': server_init, 'schema':schema}) - - def _new_client(self, server): - return _DynamicCapabilityClient()._init_vals(self.schema, server) + return _new_message(self, kwargs, num_first_segment_words) class _EnumModule(object): @@ -3699,34 +1557,6 @@ cdef class _StringArrayPtr: return ArrayPtr[StringPtr](self.thisptr, self.size) -cdef class SchemaLoader: - """ Class which can be used to construct Schema objects from schema::Nodes as defined in - schema.capnp. - - This class wraps capnproto/c++/src/capnp/schema-loader.h directly.""" - def __cinit__(self): - self.thisptr = new C_SchemaLoader() - - def __dealloc__(self): - del self.thisptr - - def load(self, _NodeReader reader): - """Loads the given schema node. Validates the node and throws an exception if invalid. This - makes a copy of the schema, so the object passed in can be destroyed after this returns. - - """ - return _Schema()._init(self.thisptr.load(reader.thisptr)) - - def load_dynamic(self, _DynamicStructReader reader): - """Loads the given schema node with self.load, but converts from a _DynamicStructReader - first.""" - return _Schema()._init(self.thisptr.load(helpers.toReader(reader.thisptr))) - - def get(self, id_): - """Gets the schema for the given ID, throwing an exception if it isn't present.""" - return _Schema()._init(self.thisptr.get(id_)) - - cdef class SchemaParser: """A class for loading Cap'n Proto schema files. @@ -3836,9 +1666,7 @@ cdef class SchemaParser: elif proto.isConst: module.__dict__[node.name] = schema.as_const_value() elif proto.isInterface: - local_module = _InterfaceModule(schema.as_interface(), node.name) - - module.__dict__[node.name] = local_module + continue elif proto.isEnum: local_module = _EnumModule(schema.as_enum(), node.name) @@ -3859,8 +1687,7 @@ cdef class SchemaParser: module._parser = parser - # Some systems (Windows running pytest) add non-directories to the sys.path used for imports - # Filter these out so kj doesn't implode when searching paths + # Only pass directories to the schema parser. filtered_imports = [] for imp in imports: if _os.path.isdir(imp): @@ -3937,16 +1764,6 @@ cdef class _MessageBuilder: ptr = s._thisptr() return _DynamicStructBuilder()._init(self.thisptr.getRootDynamicStruct(ptr), self, True) - cpdef get_root_as_any(self): - """A method for getting a Cap'n Proto AnyPointer, from an already pre-written buffer - - Don't use this method unless you know what you're doing. - - :rtype: :class:`_DynamicObjectBuilder` - :return: An AnyPointer that you can set fields in - """ - return _DynamicObjectBuilder()._init(self.thisptr.getRootAnyPointer(), self) - cpdef set_root(self, value): """A method for instantiating Cap'n Proto structs by copying from an existing struct @@ -3963,43 +1780,6 @@ cdef class _MessageBuilder: self.thisptr.setRootDynamicStruct((<_DynamicStructReader>value).thisptr) return self.get_root(value.schema) - cpdef get_segments_for_output(self): - segments = self.thisptr.getSegmentsForOutput() - res = [] - cdef const char* ptr - cdef bytes segment_bytes - for i in range(0, segments.size()): - segment = segments[i] - ptr = segment.begin() - segment_bytes = ptr[:8*segment.size()] - res.append(segment_bytes) - return res - - cpdef new_orphan(self, schema): - """A method for instantiating Cap'n Proto orphans - - Don't use this method unless you know what you're doing. - Orphans are useful for dynamically allocating objects for an unknown sized list, ie:: - - addressbook = capnp.load('addressbook.capnp') - m = capnp._MallocMessageBuilder() - alice = m.new_orphan(addressbook.Person) - - :type schema: Schema - :param schema: A Cap'n proto schema specifying which struct to instantiate - - :rtype: :class:`_DynamicOrphan` - :return: An orphan representing a :class:`_DynamicStructBuilder` - """ - cdef _StructSchema s - if hasattr(schema, 'schema'): - s = schema.schema - else: - s = schema - ptr = s._thisptr() - return _DynamicOrphan()._init(self.thisptr.newOrphan(ptr), self) - - cdef class _MallocMessageBuilder(_MessageBuilder): """The main class for building Cap'n Proto messages @@ -4012,8 +1792,7 @@ cdef class _MallocMessageBuilder(_MessageBuilder): person = message.init_root(addressbook.Person) person.name = 'alice' ... - f = open('out.txt', 'w') - _write_message_to_fd(f.fileno(), message) + data = person.to_bytes() """ def __init__(self, size=None): if size is None: @@ -4022,85 +1801,6 @@ cdef class _MallocMessageBuilder(_MessageBuilder): self.thisptr = new schema_cpp.MallocMessageBuilder(size) -cdef class _PyCustomMessageBuilder(_MessageBuilder): - """The class for building Cap'n Proto messages, - with customised memory allocation strategy - - You will use this class if you want to customise the allocateSegment method, - and define your own memory allocation strategy. - """ - def __init__(self, allocate_seg_callable, size=None): - """ The constructor requires you to provide a Python callable object as a parameter. - This callable object will be invoked in the allocateSegment method of the MessageBuilder - to allocate memory. The allocated memory will be managed within the MessageBuilder. - - :type allocate_seg_callable: Callable[[int], Buffer] - :param allocate_seg_callable: A python callable object that takes the minimum number of 8-byte - words to allocate (as an `int`) and returns any object supporting the writable buffer protocol - (e.g., `bytearray`, `memoryview`, `numpy.ndarray`). This enables custom memory allocation - strategies including shared memory. - - Required function signature is like this: - def __call__(self, minimum_size: int) -> Buffer: - - Where `Buffer` is any object that: - - Supports the Python buffer protocol (PyObject_GetBuffer) - - Is writable - Note that the unit of minimum_size is words, ie. 8 byte increments. - - The underlying memory must remain valid for the lifetime of the MessageBuilder. - If returning a view (e.g., `memoryview`, `numpy.ndarray`) that wraps external memory, - the allocator is responsible for properly managing the memory lifecycle。 - - Examples: - - # Example 1: Simple bytearray allocator - class Allocator: - def __init__(self): - self.cur_size = 0 - def __call__(self, minimum_size: int) -> bytearray: - size = max(minimum_size, self.cur_size) - self.cur_size += size - WORD_SIZE = 8 - byte_count = size * WORD_SIZE - return bytearray(byte_count) - - addressbook = capnp.load('addressbook.capnp') - allocator = Allocator() - message = capnp._PyCustomMessageBuilder(allocator) - person = message.init_root(addressbook.Person) - - # Example 2: Shared memory allocator (zero-copy) - import ctypes - - class ShmAllocator: - def __init__(self, shm_pool): - self.shm = shm_pool - self.buffers = [] - - def __call__(self, minimum_size: int) -> memoryview: - size = minimum_size * 8 - ptr = self.shm.allocate(size) - buffer = (ctypes.c_uint8 * size).from_address(ptr) - self.buffers.append(buffer) - return memoryview(buffer) - - def release(self): - for buffer in self.buffers: - ptr = ctypes.addressof(buffer) - size = ctypes.sizeof(buffer) - self.shm.deallocate(ptr, size) - self.buffers.clear() - - :type size: int - :param size: Size of the first segment to allocate (in words ie. 8 byte increments) - """ - if size is None: - self.thisptr = new schema_cpp.PyCustomMessageBuilder(allocate_seg_callable) - else: - self.thisptr = new schema_cpp.PyCustomMessageBuilder(allocate_seg_callable, size) - - cdef class _MessageReader: """An abstract base class for reading Cap'n Proto messages @@ -4137,243 +1837,6 @@ cdef class _MessageReader: ptr = s._thisptr() return _DynamicStructReader()._init(self.thisptr.getRootDynamicStruct(ptr), self) - cpdef get_root_as_any(self): - """A method for getting a Cap'n Proto AnyPointer, from an already pre-written buffer - - Don't use this method unless you know what you're doing. - - :rtype: :class:`_DynamicObjectReader` - :return: An AnyPointer that you can read from - """ - return _DynamicObjectReader()._init(self.thisptr.getRootAnyPointer(), self) - - -cdef class _StreamFdMessageReader(_MessageReader): - """Read a Cap'n Proto message from a file descriptor - - You use this class to for reading message(s) from a file. It's analagous to the inverse of - :func:`_write_message_to_fd` and :class:`_MessageBuilder`, but in one class:: - - f = open('out.txt') - message = _StreamFdMessageReader(f) - person = message.get_root(addressbook.Person) - print person.name - - :Parameters: - fd (`int`) - A file descriptor - """ - def __init__(self, file, traversal_limit_in_words=None, nesting_limit=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - self._parent = file - cdef int fd = file.fileno() - with nogil: - self.thisptr = new schema_cpp.StreamFdMessageReader(fd, opts) - - def __dealloc__(self): - del self.thisptr - - -cdef class _PackedMessageReader(_MessageReader): - """Read a Cap'n Proto message from a file descriptor in a packed manner - - You use this class to for reading message(s) from a file. It's analagous to the inverse of - :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: - - f = open('out.txt') - message = _PackedFdMessageReader(f) - person = message.get_root(addressbook.Person) - print person.name - - :Parameters: - fd (`int`) - A file descriptor - """ - def __init__(self): - pass - - cdef _init(self, schema_cpp.BufferedInputStream & stream, - traversal_limit_in_words=None, nesting_limit=None, parent=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - self._parent = parent - with nogil: - self.thisptr = new schema_cpp.PackedMessageReader(stream, opts) - return self - - def __dealloc__(self): - del self.thisptr - - -cdef class _PackedMessageReaderBytes(_MessageReader): - cdef schema_cpp.ArrayInputStream * stream - cdef Py_buffer view - - def __init__(self, buf, traversal_limit_in_words=None, nesting_limit=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - self._parent = buf - - if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: - raise KjException("could not get read buffer") - - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) - - self.thisptr = new schema_cpp.PackedMessageReader(deref(self.stream), opts) - - def __dealloc__(self): - del self.thisptr - del self.stream - PyBuffer_Release(&self.view) - - -cdef class _InputMessageReader(_MessageReader): - """Read a Cap'n Proto message from a file descriptor in a packed manner - - You use this class to for reading message(s) from a file. It's analagous to the inverse of - :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: - - f = open('out.txt') - message = _PackedFdMessageReader(f) - person = message.get_root(addressbook.Person) - print person.name - - :Parameters: - fd (`int`) - A file descriptor - """ - def __init__(self): - pass - - cdef _init(self, schema_cpp.BufferedInputStream & stream, - traversal_limit_in_words=None, nesting_limit=None, parent=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - self._parent = parent - with nogil: - self.thisptr = new schema_cpp.InputStreamMessageReader(stream, opts) - return self - - def __dealloc__(self): - del self.thisptr - - -cdef class _PackedFdMessageReader(_MessageReader): - """Read a Cap'n Proto message from a file descriptor in a packed manner - - You use this class to for reading message(s) from a file. It's analagous to the inverse of - :func:`_write_packed_message_to_fd` and :class:`_MessageBuilder`, but in one class.:: - - f = open('out.txt') - message = _PackedFdMessageReader(f) - person = message.get_root(addressbook.Person) - print person.name - - :Parameters: - fd (`int`) - A file descriptor - """ - def __init__(self, file, traversal_limit_in_words=None, nesting_limit=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - - self._parent = file - cdef int fd = file.fileno() - with nogil: - self.thisptr = new schema_cpp.PackedFdMessageReader(fd, opts) - - def __dealloc__(self): - del self.thisptr - -cdef class _AsyncMessageReader(_MessageReader): - """Read a Cap'n Proto message from a AsyncIoStream class. - - Do not use directly - """ - - def __init__(self): - pass - - cdef Own[MessageReader] reader - cdef _init(self, Own[MessageReader] reader): - self.reader = move(reader) - self.thisptr = self.reader.get() - return self - -cdef api object make_async_message_reader(Own[MessageReader] reader): - return _AsyncMessageReader()._init(move(reader)) - - -cdef class _MultipleMessageReader: - cdef schema_cpp.FdInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - cdef cbool skip_copy - - cdef public object traversal_limit_in_words, nesting_limit, schema, file - - def __init__(self, file, schema, traversal_limit_in_words=None, nesting_limit=None, skip_copy=False): - self.file = file - self.schema = schema - self.traversal_limit_in_words = traversal_limit_in_words - self.nesting_limit = nesting_limit - self.skip_copy = skip_copy - - self.stream = new schema_cpp.FdInputStream(file.fileno()) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - del self.stream - del self.buffered_stream - - def __next__(self): - try: - reader = _InputMessageReader()._init( - deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - ret = reader.get_root(self.schema) - if not self.skip_copy: - ret = ret.as_builder().as_reader() - return ret - except KjException as e: - if 'EOF' in str(e): - raise StopIteration - else: - raise - - def __iter__(self): - return self - - -cdef class _MultiplePackedMessageReader: - cdef schema_cpp.FdInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - cdef cbool skip_copy - - cdef public object traversal_limit_in_words, nesting_limit, schema, file - - def __init__(self, file, schema, traversal_limit_in_words=None, nesting_limit=None, skip_copy=False): - self.file = file - self.schema = schema - self.traversal_limit_in_words = traversal_limit_in_words - self.nesting_limit = nesting_limit - self.skip_copy = skip_copy - - self.stream = new schema_cpp.FdInputStream(file.fileno()) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - del self.stream - del self.buffered_stream - - def __next__(self): - try: - reader = _PackedMessageReader()._init( - deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - ret = reader.get_root(self.schema) - if not self.skip_copy: - ret = ret.as_builder().as_reader() - return ret - except KjException as e: - if 'EOF' in str(e): - raise StopIteration - else: - raise - - def __iter__(self): - return self - - cdef class _MultipleBytesMessageReader: cdef Py_ssize_t offset, sz cdef const char *ptr @@ -4423,84 +1886,6 @@ cdef class _MultipleBytesMessageReader: return self -cdef class _MultipleBytesPackedMessageReader: - cdef schema_cpp.ArrayInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - cdef Py_buffer view - - cdef public object traversal_limit_in_words, nesting_limit, schema, buf - - def __init__(self, buf, schema, traversal_limit_in_words=None, nesting_limit=None): - self.schema = schema - self.traversal_limit_in_words = traversal_limit_in_words - self.nesting_limit = nesting_limit - - if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: - raise KjException("could not get read buffer") - - self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - PyBuffer_Release(&self.view) - del self.buffered_stream - del self.stream - - def __next__(self): - try: - reader = _PackedMessageReader()._init( - deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root(self.schema) - except KjException as e: - if 'EOF' in str(e): - raise StopIteration - else: - raise - - def __iter__(self): - return self - - -cdef class _MultipleBytesPackedAnyMessageReader: - cdef schema_cpp.ArrayInputStream * stream - cdef schema_cpp.BufferedInputStream * buffered_stream - cdef Py_buffer view - - cdef public object traversal_limit_in_words, nesting_limit, schema, buf - - def __init__(self, buf, traversal_limit_in_words=None, nesting_limit=None): - self.traversal_limit_in_words = traversal_limit_in_words - self.nesting_limit = nesting_limit - - if PyObject_GetBuffer(buf, &self.view, PyBUF_SIMPLE) != 0: - raise KjException("could not get read buffer") - - self.buf = buf - self.stream = new schema_cpp.ArrayInputStream(schema_cpp.ByteArrayPtr(self.view.buf, self.view.len)) - self.buffered_stream = new schema_cpp.BufferedInputStreamWrapper(deref(self.stream)) - - def __dealloc__(self): - PyBuffer_Release(&self.view) - del self.buffered_stream - del self.stream - - def __next__(self): - try: - reader = _PackedMessageReader()._init( - deref(self.buffered_stream), self.traversal_limit_in_words, self.nesting_limit, self) - return reader.get_root_as_any() - except KjException as e: - if 'EOF' in str(e): - raise StopIteration - else: - raise - - def __iter__(self): - return self - - -@cython.internal cdef class _AlignedBuffer: cdef char * buf cdef bint allocated @@ -4621,136 +2006,6 @@ cdef class _FlatArrayMessageReader(_MessageReader): del self.thisptr -@cython.internal -cdef class _SegmentArrayMessageReader(_MessageReader): - - cdef object _objects_to_pin - cdef uint num_segments - cdef schema_cpp.ConstWordArrayPtr* _seg_ptrs - cdef Py_buffer* views - - def __init__(self, segments, traversal_limit_in_words=None, nesting_limit=None): - cdef schema_cpp.ReaderOptions opts = make_reader_opts(traversal_limit_in_words, nesting_limit) - # take a Python array of bytes and constructs a ConstWordArrayArrayPtr - num_segments = len(segments) - cdef schema_cpp.ConstWordArrayPtr seg_ptr - self._seg_ptrs = malloc(num_segments * sizeof(schema_cpp.ConstWordArrayPtr)) - self.views = malloc(num_segments * sizeof(Py_buffer)) - self.num_segments = num_segments - self._objects_to_pin = [] - for i in range(0, num_segments): - if PyObject_GetBuffer(segments[i], &self.views[i], PyBUF_SIMPLE) != 0: - raise KjException("could not get read buffer") - - if (self.views[i].buf) % 8 != 0: - aligned = _AlignedBuffer(segments[i]) - self.views[i].buf = aligned.buf - self._objects_to_pin.append(aligned) - else: - self._objects_to_pin.append(segments[i]) - seg_ptr = schema_cpp.ConstWordArrayPtr(self.views[i].buf, self.views[i].len//8) - self._seg_ptrs[i] = seg_ptr - self.thisptr = new schema_cpp.SegmentArrayMessageReader( - schema_cpp.ConstWordArrayArrayPtr(self._seg_ptrs, num_segments), - opts) - - def __dealloc__(self): - free(self._seg_ptrs) - for i in range(0, self.num_segments): - PyBuffer_Release(&self.views[i]) - free(self.views) - del self.thisptr - - -@cython.internal -cdef class _FlatMessageBuilder(_MessageBuilder): - cdef object _object_to_pin - cdef Py_buffer view - - def __init__(self, buf): - if PyObject_GetBuffer(buf, &self.view, PyBUF_WRITABLE) != 0: - raise KjException("expected variable length string object") - if self.view.len % 8 != 0: - raise KjException("input length must be a multiple of eight bytes") - self._object_to_pin = buf - self.thisptr = new schema_cpp.FlatMessageBuilder( - schema_cpp.WordArrayPtr(self.view.buf, self.view.len // 8)) - - def __dealloc__(self): - PyBuffer_Release(&self.view) - - -def _message_to_packed_bytes(_MessageBuilder message): - r, w = _os.pipe() - - writer = new schema_cpp.FdOutputStream(w) - schema_cpp.writePackedMessage(deref(writer), deref(message.thisptr)) - _os.close(w) - - reader = _os.fdopen(r, 'rb') - ret = reader.read() - - del writer - reader.close() - - return ret - - -def _write_message_to_fd(int fd, _MessageBuilder message): - """Serialize a Cap'n Proto message to a file descriptor - - You use this method to serialize your message to a file. Please note that - you must pass a file descriptor (ie. an int), not a file object. Make sure - you use the proper reader to match this (ie. don't use _PackedFdMessageReader):: - - message = capnp._MallocMessageBuilder() - ... - f = open('out.txt', 'w') - _write_message_to_fd(f.fileno(), message) - ... - f = open('out.txt') - _StreamFdMessageReader(f) - - :type fd: int - :param fd: A file descriptor - - :type message: :class:`_MessageBuilder` - :param message: The Cap'n Proto message to serialize - - :rtype: void - """ - with nogil: - schema_cpp.writeMessageToFd(fd, deref(message.thisptr)) - - -def _write_packed_message_to_fd(int fd, _MessageBuilder message): - """Serialize a Cap'n Proto message to a file descriptor in a packed manner - - You use this method to serialize your message to a file. Please note that - you must pass a file descriptor (ie. an int), not a file object. Also, note - the difference in names with _write_message_to_fd. This method uses a different - serialization specification, and your reader will need to match.:: - - message = capnp._MallocMessageBuilder() - ... - f = open('out.txt', 'w') - _write_packed_message_to_fd(f.fileno(), message) - ... - f = open('out.txt') - _PackedFdMessageReader(f) - - :type fd: int - :param fd: A file descriptor - - :type message: :class:`_MessageBuilder` - :param message: The Cap'n Proto message to serialize - - :rtype: void - """ - with nogil: - schema_cpp.writePackedMessageToFd(fd, deref(message.thisptr)) - - _global_schema_parser = None @@ -4798,125 +2053,9 @@ def load(file_name, display_name=None, imports=[]): return _global_schema_parser.load(file_name, display_name, imports) - -def read_multiple_bytes_packed(buf, traversal_limit_in_words=None, nesting_limit=None): - """Returns an iterable, that when traversed will return Readers for AnyPointer messages. - - :type buf: buffer - :param buf: Any Python object that supports the buffer interface. - - :type traversal_limit_in_words: int - :param traversal_limit_in_words: Limits how many total words of data are allowed to be traversed. - Is actually a uint64_t, and values can be up to 2^64-1. Default is 8*1024*1024. - - :type nesting_limit: int - :param nesting_limit: Limits how many total words of data are allowed to be traversed. Default is 64. - - :rtype: Iterable with elements of :class:`_DynamicStructReader`""" - - reader = _MultipleBytesPackedAnyMessageReader(buf, traversal_limit_in_words, nesting_limit) - return reader - - -# Automatically include the system and built-in capnp paths -# Highest priority at position 0 -_capnp_paths = [ - # Common macOS brew location - '/usr/local/include', - # Common posix location - '/usr/include', -] - -class _Loader: - def __init__(self, fullname, path): - self.fullname = fullname - self.path = path - - def create_module(self, _spec): - imports = _capnp_paths + [path if path != '' else '.' for path in _sys.path] - module = load(self.path, self.fullname, imports=imports) - return module - - def exec_module(self, _module): - pass - - -class _Importer: - - def find_spec(self, fullname, package_path, target=None): - if fullname in _sys.modules: # Don't allow re-imports - return None - - if '.' in fullname: # only when package_path anyway? - mod_parts = fullname.split('.') - module_name = mod_parts[-1] - else: - module_name = fullname - - if not module_name.endswith('_capnp'): - return None - - module_name = module_name[:-len('_capnp')] - capnp_module_name = module_name + '.capnp' - - capnp_module_names = set() - capnp_module_names.add(capnp_module_name) - if '_' in capnp_module_name: - capnp_module_names.add(capnp_module_name.replace('_', '-')) - capnp_module_names.add(capnp_module_name.replace('_', ' ')) - - if package_path: - paths = list(package_path) - else: - paths = _sys.path - - # Special case for the 'capnp' namespace, which can be resolved to system paths - if fullname.startswith('capnp.'): - paths += [path + '/capnp' for path in _capnp_paths] - - for path in paths: - if not path: - path = _os.getcwd() - elif not _os.path.isabs(path): - path = _os.path.abspath(path) - - for capnp_module_name in capnp_module_names: - if _os.path.isfile(path+_os.path.sep+capnp_module_name): - return ModuleSpec(fullname, _Loader(fullname, _os.path.join(path, capnp_module_name))) - - -_importer = None - - -def add_import_hook(): - """Add a hook to the python import system, so that Cap'n Proto modules are directly importable - - After calling this function, you can use the python import syntax to directly import capnproto schemas. - This function is automatically called upon first import of `capnp`, - so you will typically never need to use this function.:: - - import capnp - capnp.add_import_hook() - - import addressbook_capnp - # equivalent to capnp.load('addressbook.capnp', 'addressbook', sys.path), - # except it will search for 'addressbook.capnp' in all directories of sys.path - - """ - global _importer - if _importer is not None: - remove_import_hook() - - _importer = _Importer() - _sys.meta_path.append(_importer) - - def remove_import_hook(): - """Remove the import hook, and return python's import to normal""" - global _importer - if _importer is not None: - _sys.meta_path.remove(_importer) - _importer = None + """Compatibility with cereal: this build never installs an import hook.""" + pass def _init_capnp_api(): diff --git a/capnp/lib/pickle_helper.py b/capnp/lib/pickle_helper.py deleted file mode 100644 index fbf6a91..0000000 --- a/capnp/lib/pickle_helper.py +++ /dev/null @@ -1,7 +0,0 @@ -import capnp - - -def _struct_reducer(schema_id, data): - 'Hack to deal with pypy not allowing reduce functions to be "built-in" methods (ie. compiled from a .pyx)' - with capnp._global_schema_parser.modules_by_id[schema_id].from_bytes(data) as msg: - return msg diff --git a/capnp/templates/module.pyx b/capnp/templates/module.pyx deleted file mode 100644 index 024b9d8..0000000 --- a/capnp/templates/module.pyx +++ /dev/null @@ -1,249 +0,0 @@ -# addressbook_fast.pyx -# distutils: language = c++ -# distutils: include_dirs = {{include_dir}} -# distutils: libraries = capnpc capnp capnp-rpc -# distutils: sources = {{file.filename}}.cpp -# cython: c_string_type = str -# cython: c_string_encoding = default -# cython: embedsignature = True - -{% macro getter(field, type) -%} - {% if 'uint' in field['type'] -%} -uint64_t get{{field.c_name}}() except +reraise_kj_exception - {% elif 'int' in field['type'] -%} -int64_t get{{field.c_name}}() except +reraise_kj_exception - {% elif 'void' == field['type'] -%} -void get{{field.c_name}}() except +reraise_kj_exception - {% elif 'bool' == field['type'] -%} -cbool get{{field.c_name}}() except +reraise_kj_exception - {% elif 'text' == field['type'] -%} -StringPtr get{{field.c_name}}() except +reraise_kj_exception - {% elif 'data' == field['type'] -%} -Data.{{type}} get{{field.c_name}}() except +reraise_kj_exception - {% else -%} -DynamicValue.{{type}} get{{field.c_name}}() except +reraise_kj_exception - {%- endif %} -{%- endmacro %} -# TODO: add struct/enum/list types - -{% macro getfield(field, type) -%} -cpdef _get_{{field.name}}(self): - {% if 'int' in field['type'] -%} - return self.thisptr_child.get{{field.c_name}}() - {% elif 'void' == field['type'] -%} - self.thisptr_child.get{{field.c_name}}() - return None - {% elif 'bool' == field['type'] -%} - return self.thisptr_child.get{{field.c_name}}() - {% elif 'text' == field['type'] -%} - temp = self.thisptr_child.get{{field.c_name}}() - return (temp.begin())[:temp.size()] - {% elif 'data' == field['type'] -%} - temp = self.thisptr_child.get{{field.c_name}}() - return ((temp.begin())[:temp.size()]) - {% else -%} - cdef DynamicValue.{{type}} temp = self.thisptr_child.get{{field.c_name}}() - return to_python_{{type | lower}}(temp, self._parent) - {% endif -%} -{%- endmacro %} - -{% macro setter(field) -%} - {% if 'int' in field['type'] -%} -void set{{field.c_name}}({{field.type}}_t) except +reraise_kj_exception - {% elif 'bool' == field['type'] -%} -void set{{field.c_name}}(cbool) except +reraise_kj_exception - {% elif 'text' == field['type'] -%} -void set{{field.c_name}}(StringPtr) except +reraise_kj_exception - {% elif 'data' == field['type'] -%} -void set{{field.c_name}}(ArrayPtr[byte]) except +reraise_kj_exception - {% else -%} -void set{{field.c_name}}(DynamicValue.Reader) except +reraise_kj_exception - {%- endif %} -{%- endmacro %} - -{% macro setfield(field) -%} - {% if 'int' in field['type'] -%} -cpdef _set_{{field.name}}(self, {{field.type}}_t value): - self.thisptr_child.set{{field.c_name}}(value) - {% elif 'void' == field['type'] -%} -cpdef _set_{{field.name}}(self, value=None): - pass - {% elif 'bool' == field['type'] -%} -cpdef _set_{{field.name}}(self, cbool value): - self.thisptr_child.set{{field.c_name}}(value) - {% elif 'list' == field['type'] -%} -cpdef _set_{{field.name}}(self, list value): - cdef uint i = 0 - self.init("{{field.name}}", len(value)) - cdef _DynamicListBuilder temp = self._get_{{field.name}}() - for elem in value: - {% if 'struct' in field['sub_type'] -%} - temp._get(i).from_dict(elem) - {% else -%} - temp[i] = elem - {% endif -%} - i += 1 - {% elif 'text' == field['type'] -%} -cpdef _set_{{field.name}}(self, value): - cdef StringPtr temp_string - if type(value) is bytes: - temp_string = StringPtr(value, len(value)) - else: - encoded_value = value.encode('utf-8') - temp_string = StringPtr(encoded_value, len(encoded_value)) - self.thisptr_child.set{{field.c_name}}(temp_string) - {% elif 'data' == field['type'] -%} -cpdef _set_{{field.name}}(self, value): - cdef StringPtr temp_string - if type(value) is bytes: - temp_string = StringPtr(value, len(value)) - else: - encoded_value = value.encode('utf-8') - temp_string = StringPtr(encoded_value, len(encoded_value)) - self.thisptr_child.set{{field.c_name}}(ArrayPtr[byte](temp_string.begin(), temp_string.size())) - {% else -%} -cpdef _set_{{field.name}}(self, value): - _setDynamicFieldStatic(self.thisptr, "{{field.name}}", value, self._parent) - {% endif -%} -{%- endmacro %} - -import capnp -import {{file.filename | replace('.', '_')}} - -from capnp.includes.types cimport * -from capnp cimport helpers -from capnp.includes.capnp_cpp cimport DynamicValue, Schema, VOID, StringPtr, ArrayPtr, Data -from capnp.lib.capnp cimport _DynamicStructReader, _DynamicStructBuilder, _DynamicListBuilder, _DynamicEnum, _StructSchemaField, to_python_builder, to_python_reader, _to_dict, _setDynamicFieldStatic, _Schema, _InterfaceSchema - -from capnp.helpers.non_circular cimport reraise_kj_exception - -cdef DynamicValue.Reader _extract_dynamic_struct_builder(_DynamicStructBuilder value): - return DynamicValue.Reader(value.thisptr.asReader()) - -cdef DynamicValue.Reader _extract_dynamic_struct_reader(_DynamicStructReader value): - return DynamicValue.Reader(value.thisptr) - -cdef DynamicValue.Reader _extract_dynamic_enum(_DynamicEnum value): - return DynamicValue.Reader(value.thisptr) - -cdef _from_list(_DynamicListBuilder msg, list d): - cdef size_t count = 0 - for val in d: - msg._set(count, val) - count += 1 - - -cdef extern from "{{file.filename}}.h": - {%- for node in code.nodes %} - Schema get{{node.module_name}}Schema"capnp::Schema::from<{{node.c_module_path}}>"() - - cdef cppclass {{node.module_name}}"{{node.c_module_path}}": - cppclass Reader: - {%- for field in node.struct.fields %} - {{ getter(field, "Reader")|indent(12)}} - {%- endfor %} - cppclass Builder: - {%- for field in node.struct.fields %} - {{ getter(field, "Builder")|indent(12)}} - {{ setter(field)|indent(12)}} - {%- endfor %} - {%- endfor %} - - cdef cppclass C_DynamicStruct_Reader" ::capnp::DynamicStruct::Reader": - {%- for node in code.nodes %} - {{node.module_name}}.Reader as{{node.module_name}}"as<{{node.c_module_path}}>"() - {%- endfor %} - - cdef cppclass C_DynamicStruct_Builder" ::capnp::DynamicStruct::Builder": - {%- for node in code.nodes %} - {{node.module_name}}.Builder as{{node.module_name}}"as<{{node.c_module_path}}>"() - {%- endfor %} - -{%- for node in code.nodes %} - -{{node.schema}} = _Schema()._init(get{{node.module_name}}Schema()).as_struct() -{{node.module_path}}.schema = {{node.schema}} - -cdef class {{node.module_name}}_Reader(_DynamicStructReader): - cdef {{node.module_name}}.Reader thisptr_child - def __init__(self, _DynamicStructReader struct): - self._init(struct.thisptr, struct._parent, struct.is_root, False) - self.thisptr_child = (struct.thisptr).as{{node.module_name}}() - {% for field in node.struct.fields %} - - {{ getfield(field, "Reader")|indent(4) }} - - property {{field.name}}: - def __get__(self): - return self._get_{{field.name}}() - {%- endfor %} - - def to_dict(self, verbose=False, ordered=False): - ret = { - {% for field in node.struct.fields %} - {% if field.discriminantValue == 65535 %} - '{{field.name}}': _to_dict(self.{{field.name}}, verbose, ordered), - {% endif %} - {%- endfor %} - } - - {% if node.is_union %} - which = self._which_str() - ret[which] = getattr(self, which) - {% endif %} - - return ret - -cdef class {{node.module_name}}_Builder(_DynamicStructBuilder): - cdef {{node.module_name}}.Builder thisptr_child - def __init__(self, _DynamicStructBuilder struct): - self._init(struct.thisptr, struct._parent, struct.is_root, False) - self.thisptr_child = (struct.thisptr).as{{node.module_name}}() - {% for field in node.struct.fields %} - {{ getfield(field, "Builder")|indent(4) }} - {{ setfield(field)|indent(4) }} - - property {{field.name}}: - def __get__(self): - return self._get_{{field.name}}() - def __set__(self, value): - self._set_{{field.name}}(value) - {%- endfor %} - - def to_dict(self, verbose=False, ordered=False): - ret = { - {% for field in node.struct.fields %} - {% if field.discriminantValue == 65535 %} - '{{field.name}}': _to_dict(self.{{field.name}}, verbose, ordered), - {% endif %} - {%- endfor %} - } - - {% if node.is_union %} - which = self._which_str() - ret[which] = getattr(self, which) - {% endif %} - - return ret - - def from_dict(self, dict d): - cdef str key - for key, val in d.iteritems(): - if False: pass - {% for field in node.struct.fields %} - elif key == "{{field.name}}": - try: - self._set_{{field.name}}(val) - except Exception as e: - if 'expected isSetInUnion(field)' in str(e): - self.init(key) - self._set_{{field.name}}(val) - else: - raise - {%- endfor %} - else: - raise ValueError('Key not found in struct: ' + key) - - -capnp.register_type({{node.id}}, ({{node.module_name}}_Reader, {{node.module_name}}_Builder)) -{% endfor %} diff --git a/capnp/templates/setup.py.tmpl b/capnp/templates/setup.py.tmpl deleted file mode 100644 index 48b9ef0..0000000 --- a/capnp/templates/setup.py.tmpl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python -from distutils.core import setup -from Cython.Build import cythonize -from shutil import copyfile -import os -import re - - -files = [{% for f in code.requestedFiles %}"{{f.filename}}", {% endfor %}] - -for f in files: - cpp_file = f + '.cpp' - cplus_file = f + '.c++' - cpp_mod = 0 - try: - cpp_mod = os.path.getmtime(cpp_file) - except: - pass - cplus_mod = 0 - try: - cplus_mod = os.path.getmtime(cplus_file) - except: - pass - if not os.path.exists(cpp_file) or cpp_mod < cplus_mod: - if not os.path.exists(cplus_file): - raise RuntimeError("You need to run `capnp compile -oc++` in addition to `-ocython` first.") - copyfile(cplus_file, cpp_file) - - with open(f + '.h', "r") as file: - lines = file.readlines() - with open(f + '.h', "w") as file: - for line in lines: - file.write(re.sub(r'Builder\(\)\s*=\s*delete;', 'Builder() = default;', line)) - -setup( - name="{{code.requestedFiles[0] | replace('.', '_')}}", - ext_modules=cythonize('*_capnp_cython.pyx', language="c++") -) diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index 38d76dd..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/capnp.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/capnp.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/capnp" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/capnp" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/_templates/versioning.html b/docs/_templates/versioning.html deleted file mode 100644 index 7c5ab14..0000000 --- a/docs/_templates/versioning.html +++ /dev/null @@ -1,8 +0,0 @@ -{% if versions %} -

{{ _('Versions') }}

- -{% endif %} diff --git a/docs/capnp.rst b/docs/capnp.rst deleted file mode 100644 index a4f4f80..0000000 --- a/docs/capnp.rst +++ /dev/null @@ -1,178 +0,0 @@ -.. _api: - -API Reference -============= - -.. automodule:: capnp - -.. currentmodule:: capnp - - -Classes -------- - -RPC -~~~ - -.. autoclass:: capnp.lib.capnp._RemotePromise - :members: - :undoc-members: - :inherited-members: - -Communication -############# - -.. autoclass:: TwoPartyClient - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: TwoPartyServer - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: AsyncIoStream - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: capnp.lib.capnp._AsyncIoStream - :members: - :undoc-members: - :inherited-members: - -Capability -########## - -.. autoclass:: capnp.lib.capnp._DynamicCapabilityClient - :members: - :undoc-members: - :inherited-members: - - -Response -######## - -.. autoclass:: capnp.lib.capnp._Response - :members: - :undoc-members: - :inherited-members: - - -Miscellaneous -~~~~~~~~~~~~~ -.. autoclass:: KjException - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: SchemaParser - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: SchemaLoader - :members: - :undoc-members: - :inherited-members: - -Functions ---------- -.. autofunction:: add_import_hook -.. autofunction:: remove_import_hook -.. autofunction:: cleanup_global_schema_parser - -.. autofunction:: kj_loop -.. autofunction:: run - -.. autofunction:: load - - - -Internal Classes ----------------- -These classes are internal to the library. You will never need to allocate -one yourself, but you may end up using some of their member methods. - -Modules -~~~~~~~ -These are classes that are made for you when you import a Cap'n Proto file:: - - import capnp - import addressbook_capnp - - print type(addressbook_capnp.Person) # capnp.capnp._StructModule - -.. autoclass:: _InterfaceModule - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _StructModule - :members: - :undoc-members: - :inherited-members: - -Readers -~~~~~~~ -.. autoclass:: _DynamicListReader - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _DynamicStructReader - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _PackedFdMessageReader - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _StreamFdMessageReader - :members: - :undoc-members: - :inherited-members: - -Builders -~~~~~~~~ -.. autoclass:: _DynamicResizableListBuilder - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _DynamicListBuilder - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _DynamicStructBuilder - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _MallocMessageBuilder - :members: - :undoc-members: - :inherited-members: - -RPC -~~~ -.. autoclass:: _CapabilityClient - :members: - :undoc-members: - :inherited-members: - -.. autoclass:: _DynamicCapabilityClient - :members: - :undoc-members: - :inherited-members: - -Miscellaneous -~~~~~~~~~~~~~ -.. autoclass:: _DynamicOrphan - :members: - :undoc-members: - :inherited-members: diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index f3499bd..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,313 +0,0 @@ -""" -Docs configuration -""" - -# -*- coding: utf-8 -*- -# -# capnp documentation build configuration file, created by -# sphinx-quickstart on Sat Aug 17 18:00:25 2013. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import string - -# import sys, os -import capnp - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.viewcode", - "sphinx.ext.intersphinx", - "sphinx_multiversion", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "capnp" -copyright = "2013-2019 (Jason Paryani), 2019-2020 (Jacob Alexander)" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. - -vs = capnp.__version__ -# The short X.Y version. -version = vs.rstrip(string.ascii_letters) -# The full version, including alpha/beta/rc tags. -release = vs - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ["_build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = "nature" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -html_sidebars = { - "**": [ - "globaltoc.html", - "relations.html", - "sourcelink.html", - "searchbox.html", - "versioning.html", - ] -} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_domain_indices = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = "capnpdoc" - - -# -- Options for LaTeX output -------------------------------------------------- - -# The paper size ('letterpaper' or 'a4paper'). -# 'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -# 'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -# 'preamble': '', -latex_elements = {} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ("index", "capnp.tex", "capnp Documentation", "Author", "manual"), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False - -# If true, show page references after internal links. -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [("index", "capnp", "capnp Documentation", ["Author"], 1)] - -# If true, show URL addresses after external links. -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ( - "index", - "capnp", - "capnp Documentation", - "Author", - "capnp", - "One line description of project.", - "Miscellaneous", - ), -] - -# Documents to append as an appendix to all manuals. -# texinfo_appendices = [] - -# If false, no module index is generated. -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# texinfo_show_urls = 'footnote' - - -# -- Options for Epub output --------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = "capnp" -epub_author = "Author" -epub_publisher = "Author" -epub_copyright = "2013, Author" - -# The language of the text. It defaults to the language option -# or en if the language is not set. -# epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -# epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -# epub_identifier = '' - -# A unique identification for the text. -# epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -# epub_cover = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -# epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -# epub_post_files = [] - -# A list of files that should not be packed into the epub file. -# epub_exclude_files = [] - -# The depth of the table of contents in toc.ncx. -# epub_tocdepth = 3 - -# Allow duplicate toc entries. -# epub_tocdup = True - -intersphinx_mapping = {"": ("http://docs.python.org/", None)} - -smv_branch_whitelist = r"^master$" diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index eebd1e5..0000000 --- a/docs/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. capnp documentation master file - -pycapnp -======= - -This is a python wrapping of the C++ implementation of the `Cap'n Proto `_ library. Here is a short description, quoted from its docs: - - Cap’n Proto is an insanely fast data interchange format and capability-based RPC system. Think JSON, except binary. Or think Protocol Buffers, except faster. In fact, in benchmarks, Cap’n Proto is INFINITY TIMES faster than Protocol Buffers. - -Since the python library is just a thin wrapping of the C++ library, we inherit a lot of what makes Cap'n Proto fast. In some simplistic benchmarks (available in the `benchmark directory of the repo `_), pycapnp has proven to be decently faster than Protocol Buffers (both pure python and C++ implementations). Also, the python capnp library can load Cap'n Proto schema files directly, without the need for a seperate compile step like with Protocol Buffers or Thrift. pycapnp is available on `github `_ and `pypi `_. - -Contents: - -.. toctree:: - :maxdepth: 4 - - install - quickstart - capnp diff --git a/docs/install.rst b/docs/install.rst deleted file mode 100644 index 13bd657..0000000 --- a/docs/install.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. _install: - -Installation -============ - -Pip ---- -The pip installation will using the binary versions of the package (if possible). These contain a bundled version of capnproto (on Linux compiled with `manylinux `_). Starting from v1.0.0b1 binary releases are available for Windows, macOS and Linux from `pypi `_:: - - [sudo] pip install pycapnp - -To force rebuilding the pip package from source (you'll need requirments.txt or pipenv):: - - pip install --no-binary :all: pycapnp - -To force bundling libcapnp (or force system libcapnp), just in case pip isn't doing the right thing:: - - pip install --no-binary :all: -C force-bundled-libcapnp=True - pip install --no-binary :all: -C force-system-libcapnp=True - -If you're using an older Linux distro (e.g. CentOS 6) you many need to set `LDFLAGS="-Wl,--no-as-needed -lrt"`:: - - LDFLAGS="-Wl,--no-as-needed -lrt" pip install --no-binary :all: pycapnp - -It's also possible to specify the libcapnp url when bundling (this may not work, there be dragons):: - - pip install --no-binary :all: -C force-bundled-libcapnp=True -C libcapnp-url="https://github.com/capnproto/capnproto/archive/master.tar.gz" - -From Source ------------ -Source installation is generally not needed unless you're looking into an issue with capnproto or pycapnp itself. - -C++ Cap'n Proto Library -~~~~~~~~~~~~~~~~~~~~~~~ -You need to install the C++ Cap'n Proto library first. It requires a C++ compiler with C++14 support, such as GCC 5+ or Clang 5+. Follow installation docs at `https://capnproto.org/install.html `_. - -pycapnp from git -~~~~~~~~~~~~~~~~ -If you want the latest development version, you can clone the github repo:: - - git clone https://github.com/capnproto/pycapnp.git - -For development packages use one of the following to install the python dependencies:: - - cd pycapnp - uv venv - uv sync - source .venv/bin/activate - -And install pycapnp with:: - - uv pip install . - - -Development ------------ -Clone the repo from https://github.com/capnproto/pycapnp.git:: - - git clone https://github.com/capnproto/pycapnp.git - -For development packages use one of the following to install the python dependencies:: - - cd pycapnp - uv venv - uv sync - source .venv/bin/activate - -Building:: - - uv pip install . - -Useful targets for setup.py:: - - python setup.py clean - -Useful command-line arguments are available for pip install:: - - -C force-bundled-libcapnp=True - -C force-system-libcapnp=True - -C libcapnp-url="https://github.com/capnproto/capnproto/archive/master.tar.gz" - -Testing is done through pytest:: - - cd pycapnp - pytest - pytest test/test_rpc_calculator.py - -Once you're done installing, take a look at the :ref:`quickstart` diff --git a/docs/quickstart.rst b/docs/quickstart.rst deleted file mode 100644 index c2e3114..0000000 --- a/docs/quickstart.rst +++ /dev/null @@ -1,597 +0,0 @@ -.. _quickstart: - -Quickstart -========== - -This assumes you already have the capnp library installed. If you don't, please follow the instructions at :ref:`Installation ` first. - -In general, this library is a very light wrapping of the `Cap'n Proto C++ library `_. You can refer to its docs for more advanced concepts, or just to get a basic idea of how the python library is structured. - - -Load a Cap'n Proto Schema -------------------------- -First you need to import the library:: - - import capnp - -Then you can load the Cap'n Proto schema with:: - - import addressbook_capnp - -This will look all through all the directories in your sys.path/PYTHONPATH, and try to find a file of the form 'addressbook.capnp'. If you want to disable the import hook magic that `import capnp` adds, and load manually, here's how:: - - capnp.remove_import_hook() - addressbook_capnp = capnp.load('addressbook.capnp') - -For future reference, here is the Cap'n Proto schema. Also available in the github repository under `examples/addressbook.capnp `_:: - - # addressbook.capnp - @0x934efea7f017fff0; - - const qux :UInt32 = 123; - - struct Person { - id @0 :UInt32; - name @1 :Text; - email @2 :Text; - phones @3 :List(PhoneNumber); - - struct PhoneNumber { - number @0 :Text; - type @1 :Type; - - enum Type { - mobile @0; - home @1; - work @2; - } - } - - employment :union { - unemployed @4 :Void; - employer @5 :Text; - school @6 :Text; - selfEmployed @7 :Void; - # We assume that a person is only one of these. - } - } - - struct AddressBook { - people @0 :List(Person); - } - - -Const values -~~~~~~~~~~~~ -Const values show up just as you'd expect under the loaded schema. For example:: - - print(addressbook_capnp.qux) - # 123 - - -Build a message ---------------- - -Initialize a New Cap'n Proto Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Now that you've imported your schema, you need to allocate an actual struct from that schema. In this case, we will allocate an `AddressBook`:: - - addresses = addressbook_capnp.AddressBook.new_message() - -Notice that we used `addressbook_capnp` from the previous section: `Load a Cap'n Proto Schema`_. - -Also as a shortcut, you can pass keyword arguments to the `new_message` function, and those fields will be set in the new message:: - - person = addressbook_capnp.Person.new_message(name='alice') - # is equivalent to: - person = addressbook_capnp.Person.new_message() - person.name = 'alice' - - -List -~~~~ -Allocating a list inside of an object requires use of the `init` function:: - - people = addresses.init('people', 2) - -For now, let's grab the first element out of this list and assign it to a variable named `alice`:: - - alice = people[0] - -.. note:: It is a very bad idea to call `init` more than once on a single field. Every call to `init` allocates new memory inside your Cap'n Proto message, and if you call it more than once, the previous memory is left as dead space in the message. See `Tips and Best Practices `_ for more details. - - -Primitive Types -~~~~~~~~~~~~~~~ -For all primitive types, from the Cap'n Proto docs: - -- Boolean: Bool -- Integers: Int8, Int16, Int32, Int64 -- Unsigned integers: UInt8, UInt16, UInt32, UInt64 -- Floating-point: Float32, Float64 -- Blobs: Text, Data - -You can assign straight to the variable with the corresponding Python type. For Blobs, you use strings. Assignment happens just by using the `.` syntax on the object you contstructed above:: - - alice.id = 123 - alice.name = 'Alice' - alice.email = 'alice@example.com' - -.. note:: Text fields will behave differently depending on your version of Python. In Python 2.x, Text fields will expect and return a `bytes` string, while in Python 3.x, they will expect and return a `unicode` string. Data fields will always a return `bytes` string. - - -Enums -~~~~~ -First we'll allocate a length one list of phonenumbers for `alice`:: - - alicePhone = alice.init('phones', 1)[0] - -Note that even though it was a length 1 list, it was still a list that was returned, and we extracted the first (and only) element with `[0]`. - -Enums are treated like strings, and you assign to them like they were a Text field:: - - alicePhone.type = 'mobile' - -If you assign an invalid value to one, you will get a ValueError:: - - alicePhone.type = 'foo' - --------------------------------------------------------------------------- - KjException Traceback (most recent call last) - ... - AttributeError: capnp/schema.c++:566: failed: enum has no such enumerant; name = foo - - -Unions -~~~~~~ -For the most part, you just treat them like structs:: - - alice.employment.school = "MIT" - -Now the `school` field is the active part of the union, and we've assigned `'MIT'` to it. You can query which field is set in a union with `which()`, shown in `Reading Unions`_ - -Also, one weird case is for Void types in Unions (and in general, but Void is really only used in Unions). For these, you will have to assign `None` to them:: - - bob.employment.unemployed = None - -.. note:: One caveat for unions is having structs as union members. Let us assume `employment.school` was actually a struct with a field of type `Text` called `name` - - alice.employment.school.name = "MIT" - # Raises a KjException - -The problem is that a struct within a union isn't initialized automatically. You have to do the following:: - - school = alice.employment.init('school') - school.name = "MIT" - -Note that this is similar to `init` for lists, but you don't pass a size. Requiring the `init` makes it more clear that a memory allocation is occurring, and will hopefully make you mindful that you shouldn't set more than 1 field inside of a union, else you risk a memory leak - - -Writing to a File -~~~~~~~~~~~~~~~~~ -Once you're done assigning to all the fields in a message, you can write it to a file like so:: - - with open('example.bin', 'wb') as f: - addresses.write(f) - -There is also a `write_packed` function, that writes out the message more space-efficientally. If you use write_packed, make sure to use read_packed when reading the message. - -Writing to a socket -~~~~~~~~~~~~~~~~~~~ -Alternatively, you can write to a socket. This is useful if you want to send the message over the network or to another process. -A full example of this is available on GitHub `examples/async_socket_message_client.py `_.:: - - stream = await capnp.AsyncIoStream.create_connection(host="localhost", port=6000) - await addresses.write_async(stream) - -.. important:: Writing to a socket is implemented using asyncio and requires a running event loop both for the python part (asyncio) and the C++ part (KJ). See :ref:`RPC ` for more information. - -Read a message --------------- - -Reading from a file -~~~~~~~~~~~~~~~~~~~ -Much like before, you will have to de-serialize the message from a file descriptor:: - - with open('example.bin', 'rb') as f: - addresses = addressbook_capnp.AddressBook.read(f) - -Note that this very much needs to match the type you wrote out. In general, you will always be sending the same message types out over a given channel or you should wrap all your types in an unnamed union. Unnamed unions are defined in the .capnp file like so:: - - struct Message { - union { - person @0 :Person; - addressbook @1 :AddressBook; - } - } - -Reading from a socket -~~~~~~~~~~~~~~~~~~~~~ - -The same as for writing, you can read from a socket. This is useful if you want to receive the message over the network or from another process. -A full example of this is available on GitHub `examples/async_socket_message_client.py `_.:: - - stream = await capnp.AsyncIoStream.create_connection(host="localhost", port=6000) - message = await addressbook_capnp.AddressBook.read_async(stream) - -.. important:: Reading from a socket is implemented using asyncio and requires a running event loop both for the python part (asyncio) and the C++ part (KJ). See :ref:`RPC ` for more information. - - -Reading Fields -~~~~~~~~~~~~~~ -Fields are very easy to read. You just use the `.` syntax as before. Lists behave just like normal Python lists:: - - for person in addresses.people: - print(person.name, ':', person.email) - for phone in person.phones: - print(phone.type, ':', phone.number) - - -Reading Unions -~~~~~~~~~~~~~~ -The only tricky one is unions, where you need to call `.which()` to determine the union type. The `.which()` call returns an enum, ie. a string, corresponding to the field name:: - - which = person.employment.which() - print(which) - - if which == 'unemployed': - print('unemployed') - elif which == 'employer': - print('employer:', person.employment.employer) - elif which == 'school': - print('student at:', person.employment.school) - elif which == 'selfEmployed': - print('self employed') - print() - - -Serializing/Deserializing -------------------------- - -Files -~~~~~ -As shown in the examples above, there is file serialization with `write()`:: - - addresses = addressbook_capnp.AddressBook.new_message() - ... - with open('example.bin', 'wb') as f: - addresses.write(f) - -And similarly for reading:: - - with open('example.bin', 'rb') as f: - addresses = addressbook_capnp.AddressBook.read(f) - -There are packed versions as well:: - - with open('example.bin', 'wb') as f: - addresses.write_packed(f) - ... - with open('example.bin', 'rb') as f: - addresses = addressbook_capnp.AddressBook.read_packed(f) - - -Multi-message files -~~~~~~~~~~~~~~~~~~~ -The above methods only guaranteed to work if your file contains a single message. If you have more than 1 message serialized sequentially in your file, then you need to use these convenience functions:: - - addresses = addressbook_capnp.AddressBook.new_message() - ... - with open('example.bin', 'wb') as f: - addresses.write(f) - addresses.write(f) - addresses.write(f) # write 3 messages - - with open('example.bin', 'rb') as f: - for addresses in addressbook_capnp.AddressBook.read_multiple(f): - print(addresses) - -There is also a packed version:: - - for addresses in addressbook_capnp.AddressBook.read_multiple_packed(f): - print addresses - -Dictionaries -~~~~~~~~~~~~ -There is a convenience method for converting Cap'n Proto messages to a dictionary. This works for both Builder and Reader type messages:: - - alice.to_dict() - -For the reverse, all you have to do is pass keyword arguments to the new_message constructor:: - - my_dict = {'name' : 'alice'} - alice = addressbook_capnp.Person.new_message(**my_dict) - # equivalent to: alice = addressbook_capnp.Person.new_message(name='alice') - -It's also worth noting, you can use python lists/dictionaries interchangably with their Cap'n Proto equivalent types:: - - book = addressbook_capnp.AddressBook.new_message(people=[{'name': 'Alice'}]) - ... - book = addressbook_capnp.AddressBook.new_message() - book.init('people', 1) - book.people[0] = {'name': 'Bob'} - - -Byte Strings/Buffers -~~~~~~~~~~~~~~~~~~~~ -There is serialization to a byte string available:: - - encoded_message = alice.to_bytes() - -And a corresponding from_bytes function:: - - with addressbook_capnp.Person.from_bytes(encoded_message) as alice: - # something with alice - -There are also packed versions:: - - alice2 = addressbook_capnp.Person.from_bytes_packed(alice.to_bytes_packed()) - - -Byte Segments -~~~~~~~~~~~~~ -.. note:: This feature is not supported in PyPy at the moment, pending investigation. - -Cap'n Proto supports a serialization mode which minimizes object copies. In the C++ interface, ``capnp::MessageBuilder::getSegmentsForOutput()`` returns an array of pointers to segments of the message's content without copying. ``capnp::SegmentArrayMessageReader`` performs the reverse operation, i.e., takes an array of pointers to segments and uses the underlying data, again without copying. This produces a different wire serialization format from ``to_bytes()`` serialization, which uses ``capnp::messageToFlatArray()`` and ``capnp::FlatArrayMessageReader`` (both of which use segments internally, but write them in an incompatible way). - -For compatibility on the Python side, use the ``to_segments()`` and ``from_segments()`` functions:: - - segments = alice.to_segments() - -This returns a list of copied, Python-owned ``bytes`` objects. Each segment can be, e.g., turned into a ZeroMQ message frame. The list of segments can also be turned back into an object:: - - alice = addressbook_capnp.Person.from_segments(segments) - -For high-throughput code that can safely consume borrowed buffers, ``to_segment_views()`` exposes the same output segments without copying them into Python ``bytes`` objects:: - - segment_views = alice.to_segment_views() - for segment in segment_views: - transport.send(segment) - -Each segment view supports the Python buffer protocol and is read-only. The returned views borrow memory from the message builder's arena, so do not mutate, reset, or reuse the builder while any segment view is still in use. If you need data that remains independent of the builder lifetime, use ``to_segments()`` instead. - -For more information, please refer to the following links: - -- `Advice on minimizing copies from Cap'n Proto `_ (from the author of Cap'n Proto) -- `Advice on using Cap'n Proto over ZeroMQ `_ (from the author of Cap'n Proto) -- `Discussion about sending and reassembling Cap'n Proto message segments in C++ `_ (from the Cap'n Proto mailing list; includes sample code) - - -RPC ---- - -Cap'n Proto has a rich RPC protocol. You should read the `RPC specification `_ as well as the `C++ RPC documentation `_ before using pycapnp's RPC features. As with the serialization part of this library, the RPC component tries to be a very thin wrapper on top of the C++ API. - -The examples below will be using `calculator.capnp `_. Please refer to it to understand the interfaces that will be used. - -Asyncio support was added to pycapnp in v1.0.0. Since v2.0.0, the usage of asyncio is mandatory for all RPC calls. This guarantees a more robust and flexible RPC implementation. - -KJ Event Loop -~~~~~~~~~~~~~ - -.. _kj-event-loop: - -Cap'n Proto uses the KJ event loop for its RPC implementation. Pycapnp handles all the required mapping between the asyncio event loop and the KJ event loop. -To ensure proper creation, usage, and cleanup of the KJ event loop, a context manager called :py:meth:`capnp.kj_loop` is exposed by pycapnp . All RPC calls must be made within this context:: - - import capnp - import asyncio - - async def main(): - async with capnp.kj_loop(): - # RPC calls here - - asyncio.run(main()) - -To simplify the usage, the helper function:py:meth:`capnp.run` can execute a asyncio coroutine within the :py:meth:`capnp.kj_loop` context manager:: - - import capnp - import asyncio - - async def main(): - # RPC calls here - - asyncio.run(capnp.run(main())) - -Client -~~~~~~ - -.. _rpc-asyncio-client: - -Thanks to the integration into the asyncio library, most of the boiler plate code is handled by pycapnp directly. The only thing that needs to be done is to create a client object and bootstrap the server capability. - -Starting a Client -################# - -The first step is to open a socket to the server. For now this needs to be done -through :py:meth:`~._AsyncIoStream.create_connection`. A thin wrapper around :py:meth:`asyncio.get_running_loop().create_connection()` -that adds all required Protocol handling:: - - async def main(): - host = 'localhost' - port = '6000' - connection = await capnp.AsyncIoStream.create_connection(host=host, port=port) - - asyncio.run(capnp.run(main())) - -.. note:: :py:meth:`~._AsyncIoStream.create_connection` forwards all calls to the underlying asyncio create_connection function. - -In the next step, this created connection can be passed to :py:meth:`capnp.TwoPartyClient` to create the client object:: - - async def main(): - host = 'localhost' - port = '6000' - connection = await capnp.AsyncIoStream.create_connection(host=host, port=port) - client = capnp.TwoPartyClient(connection) - ## Bootstrap Here ## - - asyncio.run(capnp.run(main())) - - -SSL/TLS Client -^^^^^^^^^^^^^^ -SSL/TLS setup effectively wraps the socket transport. You'll need an SSL certificate, for this example, we'll use a self-signed certificate. Since we wrap around the asyncio connection interface, the SSL/TLS setup is done through the :py:obj:`ssl`` parameter of :py:meth:`~._AsyncIoStream.create_connection`:: - - async def main(): - host = 'localhost' - port = '6000' - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, "selfsigned.cert")) - - connection = await capnp.AsyncIoStream.create_connection(host=host, port=port, ssl=ctx, family=socket.AF_INET) - client = capnp.TwoPartyClient(connection) - ## Bootstrap Here ## - - asyncio.run(capnp.run(main())) - - -Due to a `bug `_ in Python 3.8 asyncio client needs to be initialized in a slightly different way:: - - if __name__ == '__main__': - loop = asyncio.new_event_loop() - loop.run_until_complete(capnp.run(main(parse_args().host))) - - -Bootstrap -######### -Before calling any methods you'll need to bootstrap the Calculator interface:: - - calculator = client.bootstrap().cast_as(calculator_capnp.Calculator) - -There's two things worth noting here. First, we are asking for the server capability. Secondly, you see that we are casting the capability that we receive. This is because capabilities are intrinsically dynamic, and they hold no run time type information, so we need to pick what interface to interpret them as. - - -Calling methods -############### -There are 2 ways to call RPC methods. First the more verbose `request` syntax:: - - request = calculator.evaluate_request() - request.expression.literal = 123 - eval_promise = request.send() - -This creates a request for the method named 'evaluate', sets `expression.literal` in that call's parameters to 123, and then sends the request and returns a promise (all non-blocking). - -The shorter syntax for calling methods is:: - - eval_promise = calculator.evaluate({"literal": 123}) - -The major shortcoming with this method is that expressing complex fields with many nested sub-structs can become very tedious. - -The returned promise can be handled like any other asyncio promise:: - - result = await eval_promise() - - -Pipelining -########## -If a method returns values that are themselves capabilites, then you can access these fields before awaiting the promise. Doing this is called pipelining, and it allows Cap'n Proto to chain the calls without a round-trip occurring to the server:: - - # evaluate returns `value` which is itself an interface. - # You can call a new method on `value` without having to call wait first - read_promise = eval_promise.value.read() - read_result = await read_promise # only 1 await call - -Server -~~~~~~ - -Starting a Server -########################### - -Like the client, the server uses an asyncio server that can be created with :py:meth:`~._AsyncIoStream.create_server`. - -.. note:: :py:meth:`~._AsyncIoStream.create_server`, similar to :py:meth:`~._AsyncIoStream.create_connection`, forwards all arguments to the underlying asyncio create_connection function (with the exception of the first argument). - -The first argument to :py:meth:`~._AsyncIoStream.create_server` must be a callback -used by the pycapnp protocol implementation. The :py:obj:`callback` parameter will be called -whenever a new connection is made. It receives a py:obj:`AsyncIoStream` instance as its -only argument. If the result of py:obj:`callback` is a coroutine, it will be scheduled as a -task. At minimum, the callback should create a :py:class:`capnp.TwoPartyServer` for the -passed stream. :py:class:`capnp.TwoPartyServer` also exposes a -:py:meth:`~.TwoPartyServer.on_disconnect()` function, which can be used as a task to handle -the lifetime properly:: - - async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=CalculatorImpl()).on_disconnect() - - async def main(): - host = 'localhost' - port = '6000' - server = await capnp.AsyncIoStream.create_server(new_connection, host, port) - async with server: - await server.serve_forever() - - if __name__ == "__main__": - asyncio.run(capnp.run(main())) - -.. note:: On systems that have both IPv4 and IPv6 addresses, IPv6 is often resolved first and needs to be handled separately. If you're certain IPv6 won't be used, you can remove it (you should also avoid localhost, and stick to something like 127.0.0.1). If you're broadcasting in general, you'll probably want to use `0.0.0.0` (IPv4) or `::/0` (IPv6). - - -SSL/TLS Server -^^^^^^^^^^^^^^ -Adding SSL/TLS support for a pycapnp asyncio server is fairly straight-forward. Just create an SSL context before starting the asyncio server:: - - async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=CalculatorImpl()).on_disconnect() - - async def main(): - host = 'localhost' - port = '6000' - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ctx.load_cert_chain( - os.path.join(this_dir, "selfsigned.cert"), - os.path.join(this_dir, "selfsigned.key"), - ) - - server = await capnp.AsyncIoStream.create_server( - new_connection, host, port, ssl=ctx, family=socket.AF_INET - ) - async with server: - await server.serve_forever() - - if __name__ == "__main__": - asyncio.run(capnp.run(main())) - - -Implementing a Server -##################### -Here's a part of how you would implement a Calculator server:: - - class CalculatorImpl(calculator_capnp.Calculator.Server): - - "Implementation of the Calculator Cap'n Proto interface." - - def evaluate(self, expression, _context, **kwargs): - return evaluate_impl(expression).then(lambda value: setattr(_context.results, 'value', ValueImpl(value))) - - def defFunction_context(self, context): - params = context.params - context.results.func = FunctionImpl(params.paramCount, params.body) - - def getOperator(self, op, **kwargs): - return OperatorImpl(op) - -Some major things worth noting. - -- You must inherit from `your_module_capnp.YourInterface.Server`, but don't worry about calling __super__ in your __init__ -- Method names of your class must either match the interface exactly, or have '_context' appended to it -- If your method name is exactly the same as the interface, then you will be passed all the arguments from the interface as keyword arguments, so your argument names must match the interface spec exactly. You will also receive a `_context` parameter which is equivalent to the C++ API's Context. I highly recommend having `**kwargs` as well, so that even if your interface spec is upgraded and arguments were added, your server will still operate fine. -- Returns work with a bit of magic as well. If you return a promise, then it will be handled the same as if you returned a promise from a server method in the C++ API. Otherwise, your return statement will be filled into the results struct following the ordering in your spec, for example:: - - # capability.capnp file - interface TestInterface { - foo @0 (i :UInt32, j :Bool) -> (x: Text, i:UInt32); - } - - # python code - class TestInterface(capability_capnp.TestInterface.Server): - def foo(self, i, j, **kwargs): - return str(j), i - -- If your method ends in _context, then you will only be passed a context parameter. You will have to access params and set results yourself manually. Returning promises still works as above, but you can't return anything else from a method. - - -Full Examples -------------- -`Full examples `_ are available on github. There is also an example of a very simplistic RPC available in `test_rpc.py `_. diff --git a/examples/addressbook.capnp b/examples/addressbook.capnp deleted file mode 100644 index 95fb26a..0000000 --- a/examples/addressbook.capnp +++ /dev/null @@ -1,40 +0,0 @@ -@0x934efea7f017fff0; - -const qux :UInt32 = 123; - -struct Person { - id @0 :UInt32; - name @1 :Text; - email @2 :Text; - phones @3 :List(PhoneNumber); - - struct PhoneNumber { - number @0 :Text; - type @1 :Type; - - enum Type { - mobile @0; - home @1; - work @2; - } - } - employment :union { - unemployed @4 :Void; - employer @5 :Text; - school @6 :Text; - selfEmployed @7 :Void; - # We assume that a person is only one of these. - } - - testGroup :group { - field1 @8 :UInt32; - field2 @9 :UInt32; - field3 @10 :UInt32; - } - extraData @11 :Data; -} - -struct AddressBook { - people @0 :List(Person); -} - diff --git a/examples/addressbook.py b/examples/addressbook.py deleted file mode 100755 index 462f20c..0000000 --- a/examples/addressbook.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 - -import capnp # noqa: F401 - -import addressbook_capnp - - -def writeAddressBook(file): - addresses = addressbook_capnp.AddressBook.new_message() - people = addresses.init("people", 2) - - alice = people[0] - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - alice.employment.school = "MIT" - - bob = people[1] - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - bob.employment.unemployed = None - - addresses.write(file) - - -def printAddressBook(file): - addresses = addressbook_capnp.AddressBook.read(file) - - for person in addresses.people: - print(person.name, ":", person.email) - for phone in person.phones: - print(phone.type, ":", phone.number) - - which = person.employment.which() - print(which) - - if which == "unemployed": - print("unemployed") - elif which == "employer": - print("employer:", person.employment.employer) - elif which == "school": - print("student at:", person.employment.school) - elif which == "selfEmployed": - print("self employed") - print() - - -if __name__ == "__main__": - f = open("example", "w") - writeAddressBook(f) - - f = open("example", "r") - printAddressBook(f) diff --git a/examples/async_calculator_client.py b/examples/async_calculator_client.py deleted file mode 100755 index 5d817da..0000000 --- a/examples/async_calculator_client.py +++ /dev/null @@ -1,306 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import capnp - -import calculator_capnp - - -class PowerFunction(calculator_capnp.Calculator.Function.Server): - """An implementation of the Function interface wrapping pow(). Note that - we're implementing this on the client side and will pass a reference to - the server. The server will then be able to make calls back to the client.""" - - async def call(self, params, **kwargs): - """Note the **kwargs. This is very necessary to include, since - protocols can add parameters over time. Also, by default, a _context - variable is passed to all server methods, but you can also return - results directly as python objects, and they'll be added to the - results struct in the correct order""" - - return pow(params[0], params[1]) - - -def parse_args(): - parser = argparse.ArgumentParser(usage="Connects to the Calculator server at the given address and does some RPCs") - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -async def main(connection): - client = capnp.TwoPartyClient(connection) - - # Bootstrap the Calculator interface - calculator = client.bootstrap().cast_as(calculator_capnp.Calculator) - - """Make a request that just evaluates the literal value 123. - - What's interesting here is that evaluate() returns a "Value", which is - another interface and therefore points back to an object living on the - server. We then have to call read() on that object to read it. - However, even though we are making two RPC's, this block executes in - *one* network round trip because of promise pipelining: we do not wait - for the first call to complete before we send the second call to the - server.""" - - print("Evaluating a literal... ", end="") - - # Make the request. Note we are using the shorter function form (instead - # of evaluate_request), and we are passing a dictionary that represents a - # struct and its member to evaluate - eval_promise = calculator.evaluate({"literal": 123}) - - # This is equivalent to: - """ - request = calculator.evaluate_request() - request.expression.literal = 123 - - # Send it, which returns a promise for the result (without blocking). - eval_promise = request.send() - """ - - # Using the promise, create a pipelined request to call read() on the - # returned object. Note that here we are using the shortened method call - # syntax read(), which is mostly just sugar for read_request().send() - read_promise = eval_promise.value.read() - - # Now that we've sent all the requests, wait for the response. Until this - # point, we haven't waited at all! - response = await read_promise - assert response.value == 123 - - print("PASS") - - """Make a request to evaluate 123 + 45 - 67. - - The Calculator interface requires that we first call getOperator() to - get the addition and subtraction functions, then call evaluate() to use - them. But, once again, we can get both functions, call evaluate(), and - then read() the result -- four RPCs -- in the time of *one* network - round trip, because of promise pipelining.""" - - print("Using add and subtract... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "subtract" function from the server. - subtract = calculator.getOperator(op="subtract").func - - # Build the request to evaluate 123 + 45 - 67. Note the form is 'evaluate' - # + '_request', where 'evaluate' is the name of the method we want to call - request = calculator.evaluate_request() - subtract_call = request.expression.init("call") - subtract_call.function = subtract - subtract_params = subtract_call.init("params", 2) - subtract_params[1].literal = 67.0 - - add_call = subtract_params[0].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].literal = 123 - add_params[1].literal = 45 - - # Send the evaluate() request, read() the result, and wait for read() to finish. - eval_promise = request.send() - read_promise = eval_promise.value.read() - - response = await read_promise - assert response.value == 101 - - print("PASS") - - """ - Note: a one liner version of building the previous request (I highly - recommend not doing it this way for such a complicated structure, but I - just wanted to demonstrate it is possible to set all of the fields with a - dictionary): - - eval_promise = calculator.evaluate( -{'call': {'function': subtract, - 'params': [{'call': {'function': add, - 'params': [{'literal': 123}, - {'literal': 45}]}}, - {'literal': 67.0}]}}) - """ - - """Make a request to evaluate 4 * 6, then use the result in two more - requests that add 3 and 5. - - Since evaluate() returns its result wrapped in a `Value`, we can pass - that `Value` back to the server in subsequent requests before the first - `evaluate()` has actually returned. Thus, this example again does only - one network round trip.""" - - print("Pipelining eval() calls... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "multiply" function from the server. - multiply = calculator.getOperator(op="multiply").func - - # Build the request to evaluate 4 * 6 - request = calculator.evaluate_request() - - multiply_call = request.expression.init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[0].literal = 4 - multiply_params[1].literal = 6 - - multiply_result = request.send().value - - # Use the result in two calls that add 3 and add 5. - - add_3_request = calculator.evaluate_request() - add_3_call = add_3_request.expression.init("call") - add_3_call.function = add - add_3_params = add_3_call.init("params", 2) - add_3_params[0].previousResult = multiply_result - add_3_params[1].literal = 3 - add_3_promise = add_3_request.send().value.read() - - add_5_request = calculator.evaluate_request() - add_5_call = add_5_request.expression.init("call") - add_5_call.function = add - add_5_params = add_5_call.init("params", 2) - add_5_params[0].previousResult = multiply_result - add_5_params[1].literal = 5 - add_5_promise = add_5_request.send().value.read() - - # Now wait for the results. - assert (await add_3_promise).value == 27 - assert (await add_5_promise).value == 29 - - print("PASS") - - """Our calculator interface supports defining functions. Here we use it - to define two functions and then make calls to them as follows: - - f(x, y) = x * 100 + y - g(x) = f(x, x + 1) * 2; - f(12, 34) - g(21) - - Once again, the whole thing takes only one network round trip.""" - - print("Defining functions... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "multiply" function from the server. - multiply = calculator.getOperator(op="multiply").func - - # Define f. - request = calculator.defFunction_request() - request.paramCount = 2 - - # Build the function body. - add_call = request.body.init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[1].parameter = 1 # y - - multiply_call = add_params[0].init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[0].parameter = 0 # x - multiply_params[1].literal = 100 - - f = request.send().func - - # Define g. - request = calculator.defFunction_request() - request.paramCount = 1 - - # Build the function body. - multiply_call = request.body.init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[1].literal = 2 - - f_call = multiply_params[0].init("call") - f_call.function = f - f_params = f_call.init("params", 2) - f_params[0].parameter = 0 - - add_call = f_params[1].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].parameter = 0 - add_params[1].literal = 1 - - g = request.send().func - - # OK, we've defined all our functions. Now create our eval requests. - - # f(12, 34) - f_eval_request = calculator.evaluate_request() - f_call = f_eval_request.expression.init("call") - f_call.function = f - f_params = f_call.init("params", 2) - f_params[0].literal = 12 - f_params[1].literal = 34 - f_eval_promise = f_eval_request.send().value.read() - - # g(21) - g_eval_request = calculator.evaluate_request() - g_call = g_eval_request.expression.init("call") - g_call.function = g - g_call.init("params", 1)[0].literal = 21 - g_eval_promise = g_eval_request.send().value.read() - - # Wait for the results. - assert (await f_eval_promise).value == 1234 - assert (await g_eval_promise).value == 4244 - - print("PASS") - - """Make a request that will call back to a function defined locally. - - Specifically, we will compute 2^(4 + 5). However, exponent is not - defined by the Calculator server. So, we'll implement the Function - interface locally and pass it to the server for it to use when - evaluating the expression. - - This example requires two network round trips to complete, because the - server calls back to the client once before finishing. In this - particular case, this could potentially be optimized by using a tail - call on the server side -- see CallContext::tailCall(). However, to - keep the example simpler, we haven't implemented this optimization in - the sample server.""" - - print("Using a callback... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - - # Build the eval request for 2^(4+5). - request = calculator.evaluate_request() - - pow_call = request.expression.init("call") - pow_call.function = PowerFunction() - pow_params = pow_call.init("params", 2) - pow_params[0].literal = 2 - - add_call = pow_params[1].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].literal = 4 - add_params[1].literal = 5 - - # Send the request and wait. - response = await request.send().value.read() - assert response.value == 512 - - print("PASS") - - -async def cmd_main(host): - host, port = host.split(":") - await main(await capnp.AsyncIoStream.create_connection(host=host, port=port)) - - -if __name__ == "__main__": - asyncio.run(capnp.run(cmd_main(parse_args().host))) diff --git a/examples/async_calculator_server.py b/examples/async_calculator_server.py deleted file mode 100755 index f758cd8..0000000 --- a/examples/async_calculator_server.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import logging - -import capnp -import calculator_capnp - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -async def evaluate_impl(expression, params=None): - """Implementation of CalculatorImpl::evaluate(), also shared by - FunctionImpl::call(). In the latter case, `params` are the parameter - values passed to the function; in the former case, `params` is just an - empty list.""" - - which = expression.which() - - if which == "literal": - return expression.literal - elif which == "previousResult": - return (await expression.previousResult.read()).value - elif which == "parameter": - assert expression.parameter < len(params) - return params[expression.parameter] - elif which == "call": - call = expression.call - func = call.function - - # Evaluate each parameter. - paramPromises = [evaluate_impl(param, params) for param in call.params] - vals = await asyncio.gather(*paramPromises) - - # When the parameters are complete, call the function. - result = await func.call(vals) - return result.value - else: - raise ValueError("Unknown expression type: " + which) - - -class ValueImpl(calculator_capnp.Calculator.Value.Server): - "Simple implementation of the Calculator.Value Cap'n Proto interface." - - def __init__(self, value): - self.value = value - - async def read(self, **kwargs): - return self.value - - -class FunctionImpl(calculator_capnp.Calculator.Function.Server): - """Implementation of the Calculator.Function Cap'n Proto interface, where the - function is defined by a Calculator.Expression.""" - - def __init__(self, paramCount, body): - self.paramCount = paramCount - self.body = body.as_builder() - - async def call(self, params, _context, **kwargs): - """Note that we're returning a Promise object here, and bypassing the - helper functionality that normally sets the results struct from the - returned object. Instead, we set _context.results directly inside of - another promise""" - - assert len(params) == self.paramCount - return await evaluate_impl(self.body, params) - - -class OperatorImpl(calculator_capnp.Calculator.Function.Server): - """Implementation of the Calculator.Function Cap'n Proto interface, wrapping - basic binary arithmetic operators.""" - - def __init__(self, op): - self.op = op - - async def call(self, params, **kwargs): - assert len(params) == 2 - - op = self.op - - if op == "add": - return params[0] + params[1] - elif op == "subtract": - return params[0] - params[1] - elif op == "multiply": - return params[0] * params[1] - elif op == "divide": - return params[0] / params[1] - else: - raise ValueError("Unknown operator") - - -class CalculatorImpl(calculator_capnp.Calculator.Server): - "Implementation of the Calculator Cap'n Proto interface." - - async def evaluate(self, expression, _context, **kwargs): - return ValueImpl(await evaluate_impl(expression)) - - async def defFunction(self, paramCount, body, _context, **kwargs): - return FunctionImpl(paramCount, body) - - async def getOperator(self, op, **kwargs): - return OperatorImpl(op) - - -async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=CalculatorImpl()).on_disconnect() - - -def parse_args(): - parser = argparse.ArgumentParser(usage="""Runs the server bound to the given address/port ADDRESS. """) - - parser.add_argument("address", help="ADDRESS:PORT") - - return parser.parse_args() - - -async def main(): - host, port = parse_args().address.split(":") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port) - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(capnp.run(main())) diff --git a/examples/async_client.py b/examples/async_client.py deleted file mode 100755 index ab4286f..0000000 --- a/examples/async_client.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import time -import capnp - -import thread_capnp - - -def parse_args(): - parser = argparse.ArgumentParser( - usage="Connects to the Example thread server at the given address and does some RPCs" - ) - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): - """An implementation of the StatusSubscriber interface""" - - async def status(self, value, **kwargs): - print("status: {}".format(time.time())) - - -async def main(host): - host, port = host.split(":") - connection = await capnp.AsyncIoStream.create_connection(host=host, port=port) - client = capnp.TwoPartyClient(connection) - cap = client.bootstrap().cast_as(thread_capnp.Example) - - # Start background task for subscriber - task = asyncio.ensure_future(cap.subscribeStatus(StatusSubscriber())) - - # Run blocking tasks - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - - task.cancel() - - -if __name__ == "__main__": - args = parse_args() - asyncio.run(capnp.run(main(args.host))) - - # Test that we can run multiple asyncio loops in sequence. This is particularly tricky, because - # main contains a background task that we never cancel. The entire loop gets cleaned up anyways, - # and we can start a new loop. - asyncio.run(capnp.run(main(args.host))) diff --git a/examples/async_reconnecting_ssl_client.py b/examples/async_reconnecting_ssl_client.py deleted file mode 100755 index b628dfb..0000000 --- a/examples/async_reconnecting_ssl_client.py +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import os -import time -import ssl -import socket - -import capnp - -import thread_capnp - -this_dir = os.path.dirname(os.path.abspath(__file__)) - - -def parse_args(): - parser = argparse.ArgumentParser( - usage="Connects to the Example thread server at the given address and does some RPCs" - ) - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): - """An implementation of the StatusSubscriber interface""" - - def status(self, value, **kwargs): - print("status: {}".format(time.time())) - - -async def watch_connection(cap): - while True: - try: - await asyncio.wait_for(cap.alive(), timeout=5) - await asyncio.sleep(1) - except asyncio.TimeoutError: - print("Watch timeout!") - asyncio.get_running_loop().stop() - return False - - -async def main(host): - addr, port = host.split(":") - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, "selfsigned.cert")) - - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET) - except Exception: - print("Try IPv6") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET6) - - client = capnp.TwoPartyClient(stream) - cap = client.bootstrap().cast_as(thread_capnp.Example) - - # Start watcher to restart socket connection if it is lost and subscriber background task - background_tasks = asyncio.gather( - cap.subscribeStatus(StatusSubscriber()), - watch_connection(cap), - return_exceptions=True, - ) - - # Run blocking tasks - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - - background_tasks.cancel() - - return True - - -if __name__ == "__main__": - # Using asyncio.run hits an asyncio ssl bug - # https://bugs.python.org/issue36709 - # asyncio.run(main(parse_args().host), loop=loop, debug=True) - retry = True - while retry: - loop = asyncio.new_event_loop() - try: - retry = not loop.run_until_complete(capnp.run(main(parse_args().host))) - except RuntimeError: - # If an IO is hung, the event loop will be stopped - # and will throw RuntimeError exception - continue - if retry: - time.sleep(1) - print("Retrying...") - -# How this works -# - There are two retry mechanisms -# 1. Connection retry -# 2. alive RPC verification -# - The connection retry just loops the connection (IPv4+IPv6 until there is a connection or Ctrl+C) -# - The alive RPC verification attempts a very basic rpc call with a timeout -# * If there is a timeout, stop the current event loop -# * Use the RuntimeError exception to force a reconnect -# * myreader and mywriter must also be wrapped in wait_for in order for the events to get triggered correctly diff --git a/examples/async_server.py b/examples/async_server.py deleted file mode 100755 index 4ac275a..0000000 --- a/examples/async_server.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import logging - -import capnp -import thread_capnp - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class ExampleImpl(thread_capnp.Example.Server): - "Implementation of the Example threading Cap'n Proto interface." - - async def subscribeStatus(self, subscriber, **kwargs): - await asyncio.sleep(0.1) - await subscriber.status(True) - await self.subscribeStatus(subscriber) - - async def longRunning(self, **kwargs): - await asyncio.sleep(0.1) - - -async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=ExampleImpl()).on_disconnect() - - -def parse_args(): - parser = argparse.ArgumentParser(usage="""Runs the server bound to the given address/port ADDRESS. """) - - parser.add_argument("address", help="ADDRESS:PORT") - - return parser.parse_args() - - -async def main(): - host, port = parse_args().address.split(":") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port) - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(capnp.run(main())) diff --git a/examples/async_socket_message_client.py b/examples/async_socket_message_client.py deleted file mode 100644 index e656851..0000000 --- a/examples/async_socket_message_client.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import capnp - -import addressbook_capnp - - -def parse_args(): - parser = argparse.ArgumentParser( - usage="Connects to the Example thread server at the given address and does some RPCs" - ) - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -async def writeAddressBook(stream, bob_id): - addresses = addressbook_capnp.AddressBook.new_message() - people = addresses.init("people", 1) - - bob = people[0] - bob.id = bob_id - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - bob.employment.unemployed = None - - await addresses.write_async(stream) - - -async def main(host): - host, port = host.split(":") - stream = await capnp.AsyncIoStream.create_connection(host=host, port=port) - - await writeAddressBook(stream, 0) - - message = await addressbook_capnp.AddressBook.read_async(stream) - print(message) - assert message.people[0].name == "Alice" - assert message.people[0].id == 0 - - await writeAddressBook(stream, 1) - - -if __name__ == "__main__": - args = parse_args() - asyncio.run(capnp.run(main(args.host))) diff --git a/examples/async_socket_message_server.py b/examples/async_socket_message_server.py deleted file mode 100644 index e128c56..0000000 --- a/examples/async_socket_message_server.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio - -import capnp -import addressbook_capnp - - -async def writeAddressBook(stream, alice_id): - addresses = addressbook_capnp.AddressBook.new_message() - people = addresses.init("people", 1) - - alice = people[0] - alice.id = alice_id - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - alice.employment.school = "MIT" - - await addresses.write_async(stream) - - -async def new_connection(stream): - message = await addressbook_capnp.AddressBook.read_async(stream) - print(message) - assert message.people[0].name == "Bob" - assert message.people[0].id == 0 - - await writeAddressBook(stream, 0) - - message = await addressbook_capnp.AddressBook.read_async(stream) - print(message) - assert message.people[0].name == "Bob" - assert message.people[0].id == 1 - - message = await addressbook_capnp.AddressBook.read_async(stream) - print(message) - assert message is None - - -def parse_args(): - parser = argparse.ArgumentParser(usage="""Runs the server bound to the given address/port ADDRESS. """) - - parser.add_argument("address", help="ADDRESS:PORT") - - return parser.parse_args() - - -async def main(): - host, port = parse_args().address.split(":") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port) - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(capnp.run(main())) diff --git a/examples/async_ssl_calculator_client.py b/examples/async_ssl_calculator_client.py deleted file mode 100755 index 627b8a3..0000000 --- a/examples/async_ssl_calculator_client.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import os -import ssl -import socket - -import capnp -import calculator_capnp - -this_dir = os.path.dirname(os.path.abspath(__file__)) - - -class PowerFunction(calculator_capnp.Calculator.Function.Server): - """An implementation of the Function interface wrapping pow(). Note that - we're implementing this on the client side and will pass a reference to - the server. The server will then be able to make calls back to the client.""" - - async def call(self, params, **kwargs): - """Note the **kwargs. This is very necessary to include, since - protocols can add parameters over time. Also, by default, a _context - variable is passed to all server methods, but you can also return - results directly as python objects, and they'll be added to the - results struct in the correct order""" - - return pow(params[0], params[1]) - - -def parse_args(): - parser = argparse.ArgumentParser(usage="Connects to the Calculator server at the given address and does some RPCs") - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -async def main(host): - addr, port = host.split(":") - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, "selfsigned.cert")) - - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET) - except Exception: - print("Try IPv6") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET6) - - client = capnp.TwoPartyClient(stream) - - # Bootstrap the Calculator interface - calculator = client.bootstrap().cast_as(calculator_capnp.Calculator) - - """Make a request that just evaluates the literal value 123. - - What's interesting here is that evaluate() returns a "Value", which is - another interface and therefore points back to an object living on the - server. We then have to call read() on that object to read it. - However, even though we are making two RPC's, this block executes in - *one* network round trip because of promise pipelining: we do not wait - for the first call to complete before we send the second call to the - server.""" - - print("Evaluating a literal... ", end="") - - # Make the request. Note we are using the shorter function form (instead - # of evaluate_request), and we are passing a dictionary that represents a - # struct and its member to evaluate - eval_promise = calculator.evaluate({"literal": 123}) - - # This is equivalent to: - """ - request = calculator.evaluate_request() - request.expression.literal = 123 - - # Send it, which returns a promise for the result (without blocking). - eval_promise = request.send() - """ - - # Using the promise, create a pipelined request to call read() on the - # returned object. Note that here we are using the shortened method call - # syntax read(), which is mostly just sugar for read_request().send() - read_promise = eval_promise.value.read() - - # Now that we've sent all the requests, wait for the response. Until this - # point, we haven't waited at all! - response = await read_promise - assert response.value == 123 - - print("PASS") - - """Make a request to evaluate 123 + 45 - 67. - - The Calculator interface requires that we first call getOperator() to - get the addition and subtraction functions, then call evaluate() to use - them. But, once again, we can get both functions, call evaluate(), and - then read() the result -- four RPCs -- in the time of *one* network - round trip, because of promise pipelining.""" - - print("Using add and subtract... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "subtract" function from the server. - subtract = calculator.getOperator(op="subtract").func - - # Build the request to evaluate 123 + 45 - 67. Note the form is 'evaluate' - # + '_request', where 'evaluate' is the name of the method we want to call - request = calculator.evaluate_request() - subtract_call = request.expression.init("call") - subtract_call.function = subtract - subtract_params = subtract_call.init("params", 2) - subtract_params[1].literal = 67.0 - - add_call = subtract_params[0].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].literal = 123 - add_params[1].literal = 45 - - # Send the evaluate() request, read() the result, and wait for read() to finish. - eval_promise = request.send() - read_promise = eval_promise.value.read() - - response = await read_promise - assert response.value == 101 - - print("PASS") - - """ - Note: a one liner version of building the previous request (I highly - recommend not doing it this way for such a complicated structure, but I - just wanted to demonstrate it is possible to set all of the fields with a - dictionary): - - eval_promise = calculator.evaluate( -{'call': {'function': subtract, - 'params': [{'call': {'function': add, - 'params': [{'literal': 123}, - {'literal': 45}]}}, - {'literal': 67.0}]}}) - """ - - """Make a request to evaluate 4 * 6, then use the result in two more - requests that add 3 and 5. - - Since evaluate() returns its result wrapped in a `Value`, we can pass - that `Value` back to the server in subsequent requests before the first - `evaluate()` has actually returned. Thus, this example again does only - one network round trip.""" - - print("Pipelining eval() calls... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "multiply" function from the server. - multiply = calculator.getOperator(op="multiply").func - - # Build the request to evaluate 4 * 6 - request = calculator.evaluate_request() - - multiply_call = request.expression.init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[0].literal = 4 - multiply_params[1].literal = 6 - - multiply_result = request.send().value - - # Use the result in two calls that add 3 and add 5. - - add_3_request = calculator.evaluate_request() - add_3_call = add_3_request.expression.init("call") - add_3_call.function = add - add_3_params = add_3_call.init("params", 2) - add_3_params[0].previousResult = multiply_result - add_3_params[1].literal = 3 - add_3_promise = add_3_request.send().value.read() - - add_5_request = calculator.evaluate_request() - add_5_call = add_5_request.expression.init("call") - add_5_call.function = add - add_5_params = add_5_call.init("params", 2) - add_5_params[0].previousResult = multiply_result - add_5_params[1].literal = 5 - add_5_promise = add_5_request.send().value.read() - - # Now wait for the results. - assert (await add_3_promise).value == 27 - assert (await add_5_promise).value == 29 - - print("PASS") - - """Our calculator interface supports defining functions. Here we use it - to define two functions and then make calls to them as follows: - - f(x, y) = x * 100 + y - g(x) = f(x, x + 1) * 2; - f(12, 34) - g(21) - - Once again, the whole thing takes only one network round trip.""" - - print("Defining functions... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - # Get the "multiply" function from the server. - multiply = calculator.getOperator(op="multiply").func - - # Define f. - request = calculator.defFunction_request() - request.paramCount = 2 - - # Build the function body. - add_call = request.body.init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[1].parameter = 1 # y - - multiply_call = add_params[0].init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[0].parameter = 0 # x - multiply_params[1].literal = 100 - - f = request.send().func - - # Define g. - request = calculator.defFunction_request() - request.paramCount = 1 - - # Build the function body. - multiply_call = request.body.init("call") - multiply_call.function = multiply - multiply_params = multiply_call.init("params", 2) - multiply_params[1].literal = 2 - - f_call = multiply_params[0].init("call") - f_call.function = f - f_params = f_call.init("params", 2) - f_params[0].parameter = 0 - - add_call = f_params[1].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].parameter = 0 - add_params[1].literal = 1 - - g = request.send().func - - # OK, we've defined all our functions. Now create our eval requests. - - # f(12, 34) - f_eval_request = calculator.evaluate_request() - f_call = f_eval_request.expression.init("call") - f_call.function = f - f_params = f_call.init("params", 2) - f_params[0].literal = 12 - f_params[1].literal = 34 - f_eval_promise = f_eval_request.send().value.read() - - # g(21) - g_eval_request = calculator.evaluate_request() - g_call = g_eval_request.expression.init("call") - g_call.function = g - g_call.init("params", 1)[0].literal = 21 - g_eval_promise = g_eval_request.send().value.read() - - # Wait for the results. - assert (await f_eval_promise).value == 1234 - assert (await g_eval_promise).value == 4244 - - print("PASS") - - """Make a request that will call back to a function defined locally. - - Specifically, we will compute 2^(4 + 5). However, exponent is not - defined by the Calculator server. So, we'll implement the Function - interface locally and pass it to the server for it to use when - evaluating the expression. - - This example requires two network round trips to complete, because the - server calls back to the client once before finishing. In this - particular case, this could potentially be optimized by using a tail - call on the server side -- see CallContext::tailCall(). However, to - keep the example simpler, we haven't implemented this optimization in - the sample server.""" - - print("Using a callback... ", end="") - - # Get the "add" function from the server. - add = calculator.getOperator(op="add").func - - # Build the eval request for 2^(4+5). - request = calculator.evaluate_request() - - pow_call = request.expression.init("call") - pow_call.function = PowerFunction() - pow_params = pow_call.init("params", 2) - pow_params[0].literal = 2 - - add_call = pow_params[1].init("call") - add_call.function = add - add_params = add_call.init("params", 2) - add_params[0].literal = 4 - add_params[1].literal = 5 - - # Send the request and wait. - response = await request.send().value.read() - assert response.value == 512 - - print("PASS") - - -if __name__ == "__main__": - # Using asyncio.run hits an asyncio ssl bug - # https://bugs.python.org/issue36709 - # asyncio.run(main(parse_args().host), loop=loop, debug=True) - loop = asyncio.new_event_loop() - loop.run_until_complete(capnp.run(main(parse_args().host))) diff --git a/examples/async_ssl_calculator_server.py b/examples/async_ssl_calculator_server.py deleted file mode 100755 index 046ead7..0000000 --- a/examples/async_ssl_calculator_server.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import logging -import os -import ssl -import socket - -import capnp -import calculator_capnp - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -this_dir = os.path.dirname(os.path.abspath(__file__)) - - -async def evaluate_impl(expression, params=None): - """Implementation of CalculatorImpl::evaluate(), also shared by - FunctionImpl::call(). In the latter case, `params` are the parameter - values passed to the function; in the former case, `params` is just an - empty list.""" - - which = expression.which() - - if which == "literal": - return expression.literal - elif which == "previousResult": - return (await expression.previousResult.read()).value - elif which == "parameter": - assert expression.parameter < len(params) - return params[expression.parameter] - elif which == "call": - call = expression.call - func = call.function - - # Evaluate each parameter. - paramPromises = [evaluate_impl(param, params) for param in call.params] - vals = await asyncio.gather(*paramPromises) - - # When the parameters are complete, call the function. - result = await func.call(vals) - return result.value - else: - raise ValueError("Unknown expression type: " + which) - - -class ValueImpl(calculator_capnp.Calculator.Value.Server): - "Simple implementation of the Calculator.Value Cap'n Proto interface." - - def __init__(self, value): - self.value = value - - async def read(self, **kwargs): - return self.value - - -class FunctionImpl(calculator_capnp.Calculator.Function.Server): - """Implementation of the Calculator.Function Cap'n Proto interface, where the - function is defined by a Calculator.Expression.""" - - def __init__(self, paramCount, body): - self.paramCount = paramCount - self.body = body.as_builder() - - async def call(self, params, _context, **kwargs): - """Note that we're returning a Promise object here, and bypassing the - helper functionality that normally sets the results struct from the - returned object. Instead, we set _context.results directly inside of - another promise""" - - assert len(params) == self.paramCount - return await evaluate_impl(self.body, params) - - -class OperatorImpl(calculator_capnp.Calculator.Function.Server): - """Implementation of the Calculator.Function Cap'n Proto interface, wrapping - basic binary arithmetic operators.""" - - def __init__(self, op): - self.op = op - - async def call(self, params, **kwargs): - assert len(params) == 2 - - op = self.op - - if op == "add": - return params[0] + params[1] - elif op == "subtract": - return params[0] - params[1] - elif op == "multiply": - return params[0] * params[1] - elif op == "divide": - return params[0] / params[1] - else: - raise ValueError("Unknown operator") - - -class CalculatorImpl(calculator_capnp.Calculator.Server): - "Implementation of the Calculator Cap'n Proto interface." - - async def evaluate(self, expression, _context, **kwargs): - return ValueImpl(await evaluate_impl(expression)) - - async def defFunction(self, paramCount, body, _context, **kwargs): - return FunctionImpl(paramCount, body) - - async def getOperator(self, op, **kwargs): - return OperatorImpl(op) - - -def parse_args(): - parser = argparse.ArgumentParser(usage="""Runs the server bound to the given address/port ADDRESS. """) - - parser.add_argument("address", help="ADDRESS:PORT") - - return parser.parse_args() - - -async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=CalculatorImpl()).on_disconnect() - - -async def main(): - host, port = parse_args().address.split(":") - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ctx.load_cert_chain( - os.path.join(this_dir, "selfsigned.cert"), - os.path.join(this_dir, "selfsigned.key"), - ) - - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port, ssl=ctx, family=socket.AF_INET) - except Exception: - print("Try IPv6") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port, ssl=ctx, family=socket.AF_INET6) - - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(capnp.run(main())) diff --git a/examples/async_ssl_client.py b/examples/async_ssl_client.py deleted file mode 100755 index 00aad4e..0000000 --- a/examples/async_ssl_client.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import os -import ssl -import time -import socket - -import capnp -import thread_capnp - -this_dir = os.path.dirname(os.path.abspath(__file__)) - - -def parse_args(): - parser = argparse.ArgumentParser( - usage="Connects to the Example thread server at the given address and does some RPCs" - ) - parser.add_argument("host", help="HOST:PORT") - - return parser.parse_args() - - -class StatusSubscriber(thread_capnp.Example.StatusSubscriber.Server): - """An implementation of the StatusSubscriber interface""" - - async def status(self, value, **kwargs): - print("status: {}".format(time.time())) - - -async def main(host): - addr, port = host.split(":") - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=os.path.join(this_dir, "selfsigned.cert")) - - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET) - except Exception: - print("Try IPv6") - stream = await capnp.AsyncIoStream.create_connection(addr, port, ssl=ctx, family=socket.AF_INET6) - - client = capnp.TwoPartyClient(stream) - cap = client.bootstrap().cast_as(thread_capnp.Example) - - # Start background task for subscriber - task = asyncio.ensure_future(cap.subscribeStatus(StatusSubscriber())) - - # Run blocking tasks - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - await cap.longRunning() - print("main: {}".format(time.time())) - - task.cancel() - - -if __name__ == "__main__": - # Using asyncio.run hits an asyncio ssl bug - # https://bugs.python.org/issue36709 - # asyncio.run(main(parse_args().host), loop=loop, debug=True) - loop = asyncio.new_event_loop() - loop.run_until_complete(capnp.run(main(parse_args().host))) diff --git a/examples/async_ssl_server.py b/examples/async_ssl_server.py deleted file mode 100755 index 4ebc0b3..0000000 --- a/examples/async_ssl_server.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import asyncio -import logging -import os -import ssl -import socket - -import capnp -import thread_capnp - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -this_dir = os.path.dirname(os.path.abspath(__file__)) - - -class ExampleImpl(thread_capnp.Example.Server): - "Implementation of the Example threading Cap'n Proto interface." - - async def subscribeStatus(self, subscriber, **kwargs): - await asyncio.sleep(0.1) - await subscriber.status(True) - await self.subscribeStatus(subscriber) - - async def longRunning(self, **kwargs): - await asyncio.sleep(0.1) - - async def alive(self, **kwargs): - return True - - -async def new_connection(stream): - await capnp.TwoPartyServer(stream, bootstrap=ExampleImpl()).on_disconnect() - - -def parse_args(): - parser = argparse.ArgumentParser(usage="""Runs the server bound to the given address/port ADDRESS. """) - parser.add_argument("address", help="ADDRESS:PORT") - return parser.parse_args() - - -async def main(): - host, port = parse_args().address.split(":") - - # Setup SSL context - ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - ctx.load_cert_chain( - os.path.join(this_dir, "selfsigned.cert"), - os.path.join(this_dir, "selfsigned.key"), - ) - - # Handle both IPv4 and IPv6 cases - try: - print("Try IPv4") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port, ssl=ctx, family=socket.AF_INET) - except Exception: - print("Try IPv6") - server = await capnp.AsyncIoStream.create_server(new_connection, host, port, ssl=ctx, family=socket.AF_INET6) - - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(capnp.run(main())) diff --git a/examples/calculator.capnp b/examples/calculator.capnp deleted file mode 100644 index b30a2c4..0000000 --- a/examples/calculator.capnp +++ /dev/null @@ -1,97 +0,0 @@ -@0x85150b117366d14b; - -interface Calculator { - # A "simple" mathematical calculator, callable via RPC. - # - # But, to show off Cap'n Proto, we add some twists: - # - # - You can use the result from one call as the input to the next - # without a network round trip. To accomplish this, evaluate() - # returns a `Value` object wrapping the actual numeric value. - # This object may be used in a subsequent expression. With - # promise pipelining, the Value can actually be used before - # the evaluate() call that creates it returns! - # - # - You can define new functions, and then call them. This again - # shows off pipelining, but it also gives the client the - # opportunity to define a function on the client side and have - # the server call back to it. - # - # - The basic arithmetic operators are exposed as Functions, and - # you have to call getOperator() to obtain them from the server. - # This again demonstrates pipelining -- using getOperator() to - # get each operator and then using them in evaluate() still - # only takes one network round trip. - - evaluate @0 (expression: Expression) -> (value: Value); - # Evaluate the given expression and return the result. The - # result is returned wrapped in a Value interface so that you - # may pass it back to the server in a pipelined request. To - # actually get the numeric value, you must call read() on the - # Value -- but again, this can be pipelined so that it incurs - # no additional latency. - - struct Expression { - # A numeric expression. - - union { - literal @0 :Float64; - # A literal numeric value. - - previousResult @1 :Value; - # A value that was (or, will be) returned by a previous - # evaluate(). - - parameter @2 :UInt32; - # A parameter to the function (only valid in function bodies; - # see defFunction). - - call :group { - # Call a function on a list of parameters. - function @3 :Function; - params @4 :List(Expression); - } - } - } - - interface Value { - # Wraps a numeric value in an RPC object. This allows the value - # to be used in subsequent evaluate() requests without the client - # waiting for the evaluate() that returns the Value to finish. - - read @0 () -> (value :Float64); - # Read back the raw numeric value. - } - - defFunction @1 (paramCount :Int32, body :Expression) - -> (func :Function); - # Define a function that takes `paramCount` parameters and returns the - # evaluation of `body` after substituting these parameters. - - interface Function { - # An algebraic function. Can be called directly, or can be used inside - # an Expression. - # - # A client can create a Function that runs on the server side using - # `defFunction()` or `getOperator()`. Alternatively, a client can - # implement a Function on the client side and the server will call back - # to it. However, a function defined on the client side will require a - # network round trip whenever the server needs to call it, whereas - # functions defined on the server and then passed back to it are called - # locally. - - call @0 (params :List(Float64)) -> (value: Float64); - # Call the function on the given parameters. - } - - getOperator @2 (op: Operator) -> (func: Function); - # Get a Function representing an arithmetic operator, which can then be - # used in Expressions. - - enum Operator { - add @0; - subtract @1; - multiply @2; - divide @3; - } -} diff --git a/examples/py_custom_message_builder.py b/examples/py_custom_message_builder.py deleted file mode 100644 index 983c65f..0000000 --- a/examples/py_custom_message_builder.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -import capnp # noqa: F401 -import addressbook_capnp - - -class Allocator: - def __init__(self): - self.cur_size = 0 - self.last_size = 0 - - def __call__(self, minimum_size: int) -> bytearray: - actual_size = max(minimum_size, self.cur_size) - print( - f"minimum_size: {minimum_size}, last_size: {self.last_size}, " - f"actual_size: {actual_size}, cur_size: {self.cur_size}" - ) - self.last_size = actual_size - self.cur_size += actual_size - - WORD_SIZE = 8 - byte_count = actual_size * WORD_SIZE - return bytearray(byte_count) - - -class MemoryViewAllocator: - def __init__(self): - self.buffers = [] - - def __call__(self, minimum_size: int) -> memoryview: - WORD_SIZE = 8 - buffer = bytearray(minimum_size * WORD_SIZE) - self.buffers.append(buffer) - return memoryview(buffer) - - -person = addressbook_capnp.Person.new_message(allocate_seg_callable=Allocator()) - -person.init("extraData", 5) -print(person.extraData) -print(type(person.extraData)) -print() - -person.extraData = b"hello" -print(person.extraData) -print(type(person.extraData)) -print() - -person = person.as_reader() -print(person.extraData) -print(type(person.extraData)) -print() - -person = addressbook_capnp.Person.new_message(allocate_seg_callable=MemoryViewAllocator()) - -person.init("extraData", 5) -print(person.extraData) -print(type(person.extraData)) -print() - -person.extraData = b"world" -print(person.extraData) -print(type(person.extraData)) diff --git a/examples/selfsigned.cert b/examples/selfsigned.cert deleted file mode 100644 index aa04b9a..0000000 --- a/examples/selfsigned.cert +++ /dev/null @@ -1,29 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFCTCCAvGgAwIBAgIUAd1KPs1sZ9NMbVRLXRM+XKSQ/gUwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI1MDUxMjE4MjEwOFoXDTM1MDUx -MDE4MjEwOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF -AAOCAg8AMIICCgKCAgEAz52p5xMMrjqRtwp9oH6m7TOVNkIU2WohUQfhbPgBdpkj -ZJnv6BBJyeUnDfJHayYGA7VPNh3MEWCIzFUok98fSZzIh4qqfSTg+2YdNIfM/WP8 -6jA6+YXutLl+dBZfTLW0DNUsCElgTWFpmi5AD2oqlMqdJKqOh5e8e6LBlpukXcc5 -ykNzFbTpkpc99f7k/2zMBe1208EjHXMscXDxuETdlTySo9FtetUxLuc7p6a6g472 -296/HYDaqjYZextKCTpZb4YXAG/+IcRqPi0qfSy4BKsYImo0G6lTIm28zvQBOkU9 -hpWw5GtKeWjdsF/Gr780QBY3AeUMX0cC4iU+goq4NLChRC9dde+yHZOUL7UgPAPD -tQYjLZ9pbfi3pkJZ4879YIkDD/M0Zh6LPgstfG9PRNBFEXdb9Ima8ZsxN8YeelsM -ZfmHKcWN8PAnH2ETGxVsXPXUuz1PoIxEh/MWWx1lASqF0bQtLsJP39XGWiRkhg4v -djMv9lk4+uZwZ7oDiXqGEBTzdxGtvFYNjAOWtspumdtP9MwC+oI1IR6S/bG45jtz -kkeiWz/sW1nktjFkSgdvECOBJAQM+RsNEs/9bpQV+Y8di12a/7imsSQacsvkf3Mw -m127zYcdMJVfg84AZCz+EmDaFqfO9OQdsLPtQhL6Qpixd3Mtn1WoxIhhtb+blTUC -AwEAAaNTMFEwHQYDVR0OBBYEFGBQtFxHoLu3xrGrXiulqFXhPHQTMB8GA1UdIwQY -MBaAFGBQtFxHoLu3xrGrXiulqFXhPHQTMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI -hvcNAQELBQADggIBAI84QsV4nUP6nHhHnmrUBDRpUy7VcJVD5rfq3LBSd4j2sKtd -Kv0REUDjYQVAjP6XvrX8Q+pHBrQmtoNoUWCPkp8dl8CpnjB5Jl2QPy2HOyZMmL/7 -LJN2KziZ/JprGxX1sYKKu7OQQ45ZwVByTwJ1JQRtQNhCYSV7IdA6nKbaWpnU98Xs -usq0JZQ97HNCYqOHicFl0zzSAtRPDMwMc981LYeD31Pn48pnUOpyvVvnOhA+djr6 -kjxysSaDaqq+lemqXk/AcxISFp2AaljdK8eeDnSacULGjLQmdTG1Jpc6ko/kGbD4 -J2yeB4HQJ6cFsZTVicqORffqXV9rPoe+0BUDQweSs2/aTYsyiNOsWixkzdj5ZKFw -xTEOwXBIyTt970TNJPOPRIVG5Fau5H+U/DnhrKVyLxlWYEVoYT35zAHOtlUZNR8E -7NPPsn2zTkaG5S68vZ/w8GGLQkRIl1fC89HEFYiIdPYNbNlyKsVlODQZ53zr39VS -pBTP7igelastMizfIECzkSKU2i+/io6kV9PUQ2oq1mEeEdDBYi0TwGX5d9zXOf15 -yG+PIHzb1EGQRamRC89ij0jTS/uCwgdd8ibWVzekeC43/BiSAIuGWKFF9Sq9XBdb -HEO0V8WoMxRDKe0S3FchXNWbMBcWBUVBFs5YXCmo0O+4KJNp+2XoLA7ayweW ------END CERTIFICATE----- diff --git a/examples/selfsigned.key b/examples/selfsigned.key deleted file mode 100644 index 8ae4bae..0000000 --- a/examples/selfsigned.key +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDPnannEwyuOpG3 -Cn2gfqbtM5U2QhTZaiFRB+Fs+AF2mSNkme/oEEnJ5ScN8kdrJgYDtU82HcwRYIjM -VSiT3x9JnMiHiqp9JOD7Zh00h8z9Y/zqMDr5he60uX50Fl9MtbQM1SwISWBNYWma -LkAPaiqUyp0kqo6Hl7x7osGWm6RdxznKQ3MVtOmSlz31/uT/bMwF7XbTwSMdcyxx -cPG4RN2VPJKj0W161TEu5zunprqDjvbb3r8dgNqqNhl7G0oJOllvhhcAb/4hxGo+ -LSp9LLgEqxgiajQbqVMibbzO9AE6RT2GlbDka0p5aN2wX8avvzRAFjcB5QxfRwLi -JT6Cirg0sKFEL11177Idk5QvtSA8A8O1BiMtn2lt+LemQlnjzv1giQMP8zRmHos+ -Cy18b09E0EURd1v0iZrxmzE3xh56Wwxl+YcpxY3w8CcfYRMbFWxc9dS7PU+gjESH -8xZbHWUBKoXRtC0uwk/f1cZaJGSGDi92My/2WTj65nBnugOJeoYQFPN3Ea28Vg2M -A5a2ym6Z20/0zAL6gjUhHpL9sbjmO3OSR6JbP+xbWeS2MWRKB28QI4EkBAz5Gw0S -z/1ulBX5jx2LXZr/uKaxJBpyy+R/czCbXbvNhx0wlV+DzgBkLP4SYNoWp8705B2w -s+1CEvpCmLF3cy2fVajEiGG1v5uVNQIDAQABAoICACSN+wM/fGUU1OEojLP8eMGc -6nGyMt+Q6yrMO2mnRQKvteaZn/75FzTgDv9KoD6CZF60xqydlHeeypdHiyx2BZk9 -bKVIyfncy2wYL543JuWafEZzlX6nkT7qxhQEeGUWPQxhYC5ZVQZq12AZMphENhka -j46MJSpEkiAmqPUulEMat9cgBxxUTSfNT1CHv6QlcMq+Y8Sm5drik3mpzDWIkocb -Mip7zk3pSY6bkgpTtdVCD77oujekn1uGyPe+90smpeaX8mbWUSV64sXtx+RgQko4 -Ibi1gFU6e/O85Jh/p9Otq0aOBqZBKcy0pQvP4TjCbp22C5tey83ev+g3bIkpiYMT -3JDJ5hF1qcdmuFbmFrXWn/4cdnSXRr+LGacOBMeIsJzgU7mIwsvOFqX89/oarRA/ -aG5RrzhJyt1rHb+tVre8jcgLQnpferENdJ2/UWiSi5J1BhPBYoPUxDeqqABItD31 -n6Rn6UZAOOzkMzpsB1SLSTg/hbTR2Y6kvQC2MZBLssnf+fxOH1/q+iYh8AL/kj57 -oRRbN+Pk5WxAN8+NT/meGhgsjKTeo5GooIHPmDIHBWurLZ5tBqbEQrqQHYWaOLig -5tUhlZ/mD3/6kZQHjbmUwIjebRNeNmJhu0z8qhwpwVpPcUspUoHcDH84eWCDHC0H -OHYWGW1swE4g30W09FsFAoIBAQD+pC3XP/l3yzOPX8/5W2uQWihWSm92+8USrZSm -vbYz9jPb4vq36rRzLduxvkxrVeqF5ldykBYKWXXVJCirpBoTC5+eh1oTAzRGutm5 -+U2zbwtsDSvUmzI6MbH7xx8d2Beh0IgfWGIB3Veh2yxoPvo6QcUHR37I9KMrZsPQ -70hg9+zWAF0Sh0t3ecoBocRqkrk0Lw1DARGOA4uTNbZcDJQAzPDmSWxAlbU2MoRU -HWjTN3WXmOrsIC7VrFcByaFXKxwSL0j1yMlBQyGg4bclYb/8fb3StWVWU2XBUIgS -xqsFGNnXpGkmY9HQ1RJF+wvJUMsWjgXqwf4iTzGJ1TPrJSgLAoIBAQDQuUBafgII -Jc8YHGCn4IDobE5yM9BKHu1M3YnthenoJFu2DYRCiQV9QMNUzF2XrqfGQwAqydQi -2Wfwqnh7tXoZbIRljEyDxfBI1YQmr9icFyoq7JIa+4f2AIuUl0AQk8wYvwELVPC/ -fU/N3LMnVLjq5VajPt9blwSBeUpjGA8y0qJaQ2KV7knFQwtI7OrAeNFH8I/wkIRH -vNy+/Sz0BcDHRRL0120/45DFNCuxyvs5EN/dAuvOfSJqndY1t/fAVoNg2wvAyns/ -sbKQeexd/clTYGjqAMEU/sImtQEwldeCPvuCAhUwASylrcHWoHsCDb4pKcgiFsIN -tPWgsdrbGD+/AoIBAQD2fVh+Z0gGBOYJIFcCatNJbWxkczNIutf+h5ZAfZ202NtE -O4g0pfY9FCP4/1ub/xPAv8Lge8dKB2T/iDvyQiyXSQYe/6hahRyCZvbBhikHyzME -Sg+mgwBwwpAmR47AZeAiW+iYZwagBXGBlNZ8ppGz+NxPeo6o2d5k8doVErs+Wl+g -m8N8XwjXQ0YepEesXhD3CaDNvmgOzzG5syGuIuLVj4yVbndiYUiDiQz9G2bQJnwm -3fhxz4lmfqfObC5IYcuPcsQuX0kpamFQCY4umlusfs9T+xF4Kcxy/5BolHURvweI -LXc3mSKOAuLoaOX03sdoMtxZbaWh8oTihkX2lgYXAoIBAGhhHCOk/FMixUwjdNq/ -VPfmodxOuQ04JifYak+UNoNXG14RqGC1sT8QEh7oDK38M/7cJss/H41F98rNFW+Y -M7VfJV67KNCFPkLONEY8jjCRDQ9mOzKvMzD82NC4Stt/bgO6EUWfdr3sZupmQlma -7tbZVdhRatWc0i4FgAPKVl9uIq7NIBImllHF03DmugcC5HX7gaAmRWCyvBnu9noa -HmwIyRAUY5gdr5pPGsLQ5Y2GOM2H1nDu9zUmNaerloRjP1RCdsA1Aim6Lbg+oMvo -TLQbdJwBQI3FUUaWIkAvzxRddt1vOTVGgRNhr5wrqRg/0yc2s9UIWIcORf/UscP7 -fnUCggEAPi88JcWuzlJmEUSJdgjRzdCnNh3ZyOUG8L8SIQaLs098hsJhR4cpHc6Q -2uXr7kuTkXlNh/vUZmfFMQJe+8Pg+fjUzom7oM4iOEGozJRyY9phjPWvUgE2UWfd -Pa2nLqhP2UIplayvr6gGPYVlxSnaddZPMBB53W641nYxO1iuEHY25vdkAnhQU1K+ -BfaR9E7Rc80AAhK+Fhs2xzoHUKJ/z1ciFtZylqWZNDu2jHSgBLawDljhCqLstJ3q -vx7Qlq+j6dh68BVb60/R4RRsRy6pvUFbYcKPFiuEPK3EYr3Ascy7P2tn9Z8LdXie -flYh3XPcmbDCBLfXFvforZR+CYzBlA== ------END PRIVATE KEY----- diff --git a/examples/thread.capnp b/examples/thread.capnp deleted file mode 100644 index 8caf56f..0000000 --- a/examples/thread.capnp +++ /dev/null @@ -1,12 +0,0 @@ -@0xf5745ea9c82baa3a; - -interface Example { - interface StatusSubscriber { - status @0 (value: Bool); - # Call the function on the given parameters. - } - - longRunning @0 () -> (value: Bool); - subscribeStatus @1 (subscriber: StatusSubscriber); - alive @2 () -> (value: Bool); -} diff --git a/pyproject.toml b/pyproject.toml index 655e6fb..e68a0bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ backend-path = ["_custom_build"] [project] name = "pycapnp" -requires-python = ">=3.9" +requires-python = ">=3.12" dynamic = [ "authors", "classifiers", @@ -13,60 +13,31 @@ dynamic = [ "keywords", "license", "readme", - "scripts", "version", ] -dependencies = [ - "jinja2", -] +dependencies = [] [dependency-groups] -dev = [ - "tox", - "pkgconfig", - "cython>=3.0", - "setuptools", - {include-group = "lint"}, - {include-group = "test"}, - {include-group = "docs"}, -] - -lint = [ - # TODO: Update lint and formatting for newer Ruff releases, then remove the pin. - "ruff==0.15.22", -] - -test = [ - "pytest", - "anyio", - "pytest-asyncio", - "pytest-tornasync", - "pytest-trio", - "pytest-twisted", - "twisted", -] - -docs = [ - "sphinx", - "sphinx-multiversion", -] +dev = ["cython>=3.0", "setuptools", "wheel", "pkgconfig", "build", {include-group = "test"}, {include-group = "lint"}] +test = ["pytest"] +lint = ["ruff==0.16.8"] [tool.pytest.ini_options] -asyncio_mode = "auto" +testpaths = ["test"] [tool.ruff] -target-version = "py37" +target-version = "py312" line-length = 120 -exclude = ["benchmark", "build", "capnp/templates/module.pyx"] +exclude = ["build", "build64", "bundled"] [tool.ruff.lint] +select = ["E4", "E7", "E9", "F"] ignore = [ "E203", "E211", "E225", "E226", "E227", "E231", "E251", "E261", "E262", "E265", "E402", ] [tool.ruff.lint.per-file-ignores] -"test/test_examples.py" = ["C901"] "capnp/__init__.py" = ["F401", "F403", "F405"] [tool.ruff.lint.mccabe] diff --git a/scripts/capnp-json.py b/scripts/capnp-json.py deleted file mode 100755 index 1199e2a..0000000 --- a/scripts/capnp-json.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python - -import argparse -import sys -import json -import capnp - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("command") - parser.add_argument("schema_file") - parser.add_argument("struct_name") - parser.add_argument( - "-d", - "--defaults", - help="include default values in json output", - action="store_true", - ) - - return parser.parse_args() - - -def encode(schema_file, struct_name, **kwargs): - schema = capnp.load(schema_file) - - struct_schema = getattr(schema, struct_name) - - struct_dict = json.load(sys.stdin) - struct = struct_schema.from_dict(struct_dict) - - struct.write(sys.stdout) - - -def decode(schema_file, struct_name, defaults): - schema = capnp.load(schema_file) - - struct_schema = getattr(schema, struct_name) - struct = struct_schema.read(sys.stdin) - - json.dump(struct.to_dict(defaults), sys.stdout) - - -def main(): - args = parse_args() - - command = args.command - kwargs = vars(args) - del kwargs["command"] - - globals()[command](**kwargs) # hacky way to get defined functions, and call function with name=command - - -main() diff --git a/scripts/capnp_test_pycapnp.py b/scripts/capnp_test_pycapnp.py deleted file mode 100755 index 0716779..0000000 --- a/scripts/capnp_test_pycapnp.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -import capnp - -capnp.add_import_hook([os.getcwd(), "/usr/local/include/"]) # change this to be auto-detected? - -import test_capnp # noqa: E402 - - -def decode(name): - class_name = name[0].upper() + name[1:] - with getattr(test_capnp, class_name).from_bytes(sys.stdin.read()) as msg: - print(msg._short_str()) - - -def encode(name): - val = getattr(test_capnp, name) - class_name = name[0].upper() + name[1:] - message = getattr(test_capnp, class_name).from_dict(val.to_dict()) - print(message.to_bytes()) - - -if sys.argv[1] == "decode": - decode(sys.argv[2]) -else: - encode(sys.argv[2]) diff --git a/scripts/release-pypi.sh b/scripts/release-pypi.sh deleted file mode 100755 index f56274e..0000000 --- a/scripts/release-pypi.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash -# Download GitHub Actions build artifacts for a tagged commit (or specific run -# ID) and upload the wheels + sdist to PyPI via twine. -# -# Usage: -# scripts/release-pypi.sh [output-dir] [--force] [--test] -# -# Examples: -# scripts/release-pypi.sh v2.2.1 -# scripts/release-pypi.sh 2.2.1 dist_221 -# scripts/release-pypi.sh 1234567890 dist_run_1234567890 -# scripts/release-pypi.sh v2.2.1 --test # dry-run upload to TestPyPI -# -# Requirements: -# - gh CLI (authenticated; `gh auth status` must succeed) -# - python3 -# - Twine credentials configured in the environment or ~/.pypirc. -# For real uploads: ~/.pypirc [pypi] section, or TWINE_USERNAME/TWINE_PASSWORD. -# For --test uploads: ~/.pypirc [testpypi] section (separate TestPyPI account -# and API token; see https://packaging.python.org/en/latest/guides/using-testpypi/). - -set -euo pipefail - -WORKFLOW_FILE="wheels.yml" -ARTIFACT_PATTERN="cibw-*" -VENV_DIR=".venv-release" - -usage() { - cat >&2 < [output-dir] [--force] [--test] - - Git tag (e.g. v2.2.1 or 2.2.1) or a GitHub Actions run ID. - [output-dir] Directory to place wheels/sdist into. Defaults to - dist_ for a tag, or dist_run_ for a run ID. - --force Allow reusing a non-empty output directory. - --test Dry-run: upload to TestPyPI (https://test.pypi.org) instead - of the real PyPI. Requires a [testpypi] entry in ~/.pypirc - or a TestPyPI API token. -EOF - exit 2 -} - -die() { - echo "error: $*" >&2 - exit 1 -} - -require_cmd() { - command -v "$1" >/dev/null 2>&1 || die "required command '$1' not found in PATH" -} - -FORCE=0 -TEST_UPLOAD=0 -POSITIONAL=() -for arg in "$@"; do - case "$arg" in - --force) FORCE=1 ;; - --test|--testpypi|--dry-run) TEST_UPLOAD=1 ;; - -h|--help) usage ;; - *) POSITIONAL+=("$arg") ;; - esac -done - -[[ ${#POSITIONAL[@]} -ge 1 && ${#POSITIONAL[@]} -le 2 ]] || usage - -INPUT="${POSITIONAL[0]}" -OUTPUT_DIR="${POSITIONAL[1]:-}" - -require_cmd gh -require_cmd python3 - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$REPO_ROOT" - -echo ">> verifying gh authentication" -gh auth status >/dev/null || die "gh is not authenticated; run 'gh auth login'" - -# Resolve run ID. A bare integer is treated as a run ID; otherwise we look up -# the latest successful run on the branch/tag matching the build workflow. -if [[ "$INPUT" =~ ^[0-9]+$ ]]; then - RUN_ID="$INPUT" - DEFAULT_OUT="dist_run_${RUN_ID}" - echo ">> using GitHub Actions run id: $RUN_ID" -elif [[ "$INPUT" =~ ^v?[0-9].* ]]; then - TAG="$INPUT" - TAG_NO_V="${TAG#v}" - DEFAULT_OUT="dist_${TAG_NO_V//./}" - echo ">> resolving latest successful '$WORKFLOW_FILE' run for tag '$TAG'" - RUN_ID="$(gh run list \ - --workflow="$WORKFLOW_FILE" \ - --branch "$TAG" \ - --status success \ - --limit 1 \ - --json databaseId \ - --jq '.[0].databaseId')" - if [[ -z "${RUN_ID:-}" || "$RUN_ID" == "null" ]]; then - die "no successful '$WORKFLOW_FILE' run found for tag '$TAG'" - fi - echo ">> resolved run id: $RUN_ID" -else - die "first arg must be a git tag (vX.Y.Z) or a numeric run ID" -fi - -OUTPUT_DIR="${OUTPUT_DIR:-$DEFAULT_OUT}" - -if [[ -e "$OUTPUT_DIR" ]]; then - if [[ -d "$OUTPUT_DIR" ]]; then - if [[ -n "$(ls -A "$OUTPUT_DIR" 2>/dev/null)" && "$FORCE" -ne 1 ]]; then - die "'$OUTPUT_DIR' is not empty; pass --force to reuse it" - fi - else - die "'$OUTPUT_DIR' exists and is not a directory" - fi -fi -mkdir -p "$OUTPUT_DIR" -OUTPUT_DIR_ABS="$(cd "$OUTPUT_DIR" && pwd)" - -TMP_DIR="$(mktemp -d -t pycapnp-release-XXXXXX)" -trap 'rm -rf "$TMP_DIR"' EXIT - -echo ">> downloading artifacts (pattern '$ARTIFACT_PATTERN') into $TMP_DIR" -gh run download "$RUN_ID" --dir "$TMP_DIR" --pattern "$ARTIFACT_PATTERN" - -echo ">> flattening wheels and sdists into $OUTPUT_DIR_ABS" -shopt -s nullglob globstar -moved=0 -for f in "$TMP_DIR"/**/*.whl "$TMP_DIR"/**/*.tar.gz; do - [[ -f "$f" ]] || continue - mv -n "$f" "$OUTPUT_DIR_ABS/" - moved=$((moved + 1)) -done -shopt -u globstar -[[ "$moved" -gt 0 ]] || die "no .whl or .tar.gz files found in downloaded artifacts" -echo ">> collected $moved files" - -echo ">> setting up release virtualenv at $VENV_DIR" -if [[ ! -x "$VENV_DIR/bin/python" ]]; then - python3 -m venv "$VENV_DIR" -fi -# shellcheck disable=SC1091 -source "$VENV_DIR/bin/activate" -python -m pip install --quiet --upgrade pip -python -m pip install --quiet --upgrade twine - -echo ">> running 'twine check'" -python -m twine check "$OUTPUT_DIR_ABS"/* - -if [[ "$TEST_UPLOAD" -eq 1 ]]; then - TARGET_LABEL="TestPyPI (https://test.pypi.org)" - TWINE_REPO_ARGS=(--repository testpypi) -else - TARGET_LABEL="PyPI (https://pypi.org)" - TWINE_REPO_ARGS=() -fi - -echo -echo "Target: $TARGET_LABEL" -echo "Files to upload from $OUTPUT_DIR_ABS:" -ls -1 "$OUTPUT_DIR_ABS" -echo -read -r -p "Upload these to $TARGET_LABEL? [y/N] " reply -case "$reply" in - [yY]|[yY][eE][sS]) ;; - *) echo "aborted; files remain in $OUTPUT_DIR_ABS"; exit 0 ;; -esac - -echo ">> uploading to $TARGET_LABEL via twine" -python -m twine upload "${TWINE_REPO_ARGS[@]}" "$OUTPUT_DIR_ABS"/* - -echo ">> done" diff --git a/setup.py b/setup.py index 3852123..c1d68d6 100644 --- a/setup.py +++ b/setup.py @@ -55,13 +55,9 @@ short_version = '%s' write_version_py() -# Try to use README.md and CHANGELOG.md as description and changelog +# Use the fork README as the package description with open("README.md", encoding="utf-8") as f: long_description = f.read() -with open("CHANGELOG.md", encoding="utf-8") as f: - changelog = f.read() -changelog = "\nChangelog\n=============\n" + changelog -long_description += changelog class clean(_clean): @@ -154,8 +150,6 @@ class build_libcapnp_ext(build_ext_c): # Check if we've already built capnproto capnp_bin = os.path.join(build_dir, "bin", "capnp") - if os.name == "nt": - capnp_bin = os.path.join(build_dir, "bin", "capnp.exe") if not os.path.exists(capnp_bin): # Not built, fetch and build @@ -170,23 +164,10 @@ class build_libcapnp_ext(build_ext_c): os.path.join(build_dir, "lib"), ] + self.library_dirs - # Copy .capnp files from source - src_glob = glob.glob(os.path.join(build_dir, "include", "capnp", "*.capnp")) - dst_dir = os.path.join(self.build_lib, "capnp") - os.makedirs(dst_dir, exist_ok=True) - for file in src_glob: - print("copying {} -> {}".format(file, dst_dir)) - shutil.copy(file, dst_dir) - return build_ext_c.run(self) extra_compile_args = ["--std=c++14"] -extra_link_args = [] -if os.name == "nt": - extra_compile_args = ["/std:c++14", "/MD"] - extra_link_args = ["/MANIFEST"] - import Cython.Build # noqa: E402 import Cython # noqa: E402 @@ -194,26 +175,24 @@ extensions = [ Extension( "*", [ - "capnp/helpers/capabilityHelper.cpp", - "capnp/includes/PyCustomMessageBuilder.cpp", + "capnp/helpers/exception.cpp", "capnp/lib/*.pyx", ], extra_compile_args=extra_compile_args, - extra_link_args=extra_link_args, language="c++", ) ] setup( - python_requires=">=3.9", + python_requires=">=3.12", name="pycapnp", - packages=["capnp"], + packages=["capnp", "capnp.lib"], + include_package_data=False, version=VERSION, package_data={ "capnp": [ "*.pxd", "*.h", - "*.capnp", "helpers/*.pxd", "helpers/*.h", "includes/*.h", @@ -222,13 +201,11 @@ setup( "lib/*.py", "lib/*.pyx", "lib/*.h", - "templates/*", ] }, ext_modules=Cython.Build.cythonize(extensions), cmdclass={"clean": clean, "build_ext": build_libcapnp_ext}, install_requires=[], - entry_points={"console_scripts": ["capnpc-cython = capnp._gen:main"]}, # PyPi info description="A cython wrapping of the C++ Cap'n Proto library", long_description=long_description, @@ -237,24 +214,19 @@ setup( # (setup.py only supports 1 author...) author="Jacob Alexander", # <- Current maintainer; Original author -> Jason Paryani author_email="haata@kiibohd.com", - url="https://github.com/capnproto/pycapnp", - download_url="https://github.com/capnproto/pycapnp/archive/v%s.zip" % VERSION, + url="https://github.com/commaai/pycapnp", + download_url="https://github.com/commaai/pycapnp/archive/v%s.zip" % VERSION, keywords=["capnp", "capnproto", "Cap'n Proto", "pycapnp"], classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Operating System :: MacOS :: MacOS X", - "Operating System :: Microsoft :: Windows :: Windows 10", "Operating System :: POSIX", "Programming Language :: C++", "Programming Language :: Cython", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Communications", ], ) diff --git a/test/addressbook with spaces.capnp b/test/addressbook with spaces.capnp deleted file mode 100644 index f7c611e..0000000 --- a/test/addressbook with spaces.capnp +++ /dev/null @@ -1,39 +0,0 @@ -@0xc39aee9191aedcf3; - -const qux :UInt32 = 123; - -struct Person { - id @0 :UInt32; - name @1 :Text; - email @2 :Text; - phones @3 :List(PhoneNumber); - - struct PhoneNumber { - number @0 :Text; - type @1 :Type; - - enum Type { - mobile @0; - home @1; - work @2; - } - } - - employment :union { - unemployed @4 :Void; - employer @5 :Employer; - school @6 :Text; - selfEmployed @7 :Void; - # We assume that a person is only one of these. - } -} - -struct Employer { - name @0 :Text; - boss @1 :Person; -} - -struct AddressBook { - people @0 :List(Person); -} - diff --git a/test/addressbook-with-dashes.capnp b/test/addressbook-with-dashes.capnp deleted file mode 100644 index 2295e8e..0000000 --- a/test/addressbook-with-dashes.capnp +++ /dev/null @@ -1,39 +0,0 @@ -@0xd33206731939e03b; - -const qux :UInt32 = 123; - -struct Person { - id @0 :UInt32; - name @1 :Text; - email @2 :Text; - phones @3 :List(PhoneNumber); - - struct PhoneNumber { - number @0 :Text; - type @1 :Type; - - enum Type { - mobile @0; - home @1; - work @2; - } - } - - employment :union { - unemployed @4 :Void; - employer @5 :Employer; - school @6 :Text; - selfEmployed @7 :Void; - # We assume that a person is only one of these. - } -} - -struct Employer { - name @0 :Text; - boss @1 :Person; -} - -struct AddressBook { - people @0 :List(Person); -} - diff --git a/test/all-types.packed b/test/all-types.packed deleted file mode 100644 index acb6086e58e37b121e9ca54bf1f58ff98260eb2f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 831 zcmZ9JZAep57{|}q-Al9C_3q3ttG($ys3C4kQv0y=1{Nv`_9YOzx))!Dce~roZ3OqG z*o#Fen3#ywmq5$}N%~O6if9$0u)7g|rFZ`av|Noo^p8vTejj&d! z!>}hlKQ(#;Qx}ZiA`h-%ybL;~bZr@Pi44Q%D`WNJ`r6^6ia!sRVqKS~UozEZraHH- zN~uFVR0%pqnFv|}BIpR&#McrKK|{zSKVT?sbEh#M#FU#UGX8uoyVQ)1MGjM~Pi zE9ihAgbmFc?{z!8JS+G@X`gh>4lm35eGx+^>vM6mhj+W`G~^`|XfKR{9A+j}8lvPa z!p2~T7n(!B;cE?sjMlA&ZL49=rl1n4DXyRP7Ts9D@%Ed_@`2MoWbweWqH&>}F28zk zp}uz@NpXvGY1haEee(9b=Qtj?J3TpM8wPVD6UQWRR>Cnc8Dk$iV@WZV#IaQ|y*r6j zXz=?4Z;K%uiAE!Is6}v*JdFATlUYG?oIi-&;TBdP{}Vu$$a|XYCFQn%;o4H+Ql+7d Vgj~4000 bytes) causes memory corruption. The root cause was that -_PyAsyncIoStreamProtocol.write_loop() passed a memoryview pointing to C++ -memory to transport.write(), then called fulfill() which freed the C++ memory. -Since transport.write() is non-blocking and buffers data asynchronously, -the data could be corrupted before being sent. - -The fix is to copy the data to Python bytes before passing to transport.write(). - -See: https://github.com/capnproto/pycapnp/pull/392 -""" - -import pytest -import socket - -import capnp -import test_capability_capnp - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -class LargeResponseServer(test_capability_capnp.TestInterface.Server): - """ - Server that returns large text responses to trigger the use-after-free bug. - - The bug manifests when response messages are >~4000 bytes. - """ - - async def foo(self, i, j, **kwargs): - # Generate a large response string based on input - # The size is controlled by the input parameter 'i' - size = i - # Create a deterministic pattern that can be verified - pattern = "".join(chr(65 + (k % 26)) for k in range(size)) - return pattern - - -async def test_large_response_sequential(): - """ - Test that large RPC responses are received correctly when sent sequentially. - - Tests various payload sizes including those >4000 bytes which trigger the bug. - """ - read_sock, write_sock = socket.socketpair() - read_stream = await capnp.AsyncIoStream.create_connection(sock=read_sock) - write_stream = await capnp.AsyncIoStream.create_connection(sock=write_sock) - - _ = capnp.TwoPartyServer(write_stream, bootstrap=LargeResponseServer()) - client = capnp.TwoPartyClient(read_stream) - cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) - - # Test various sizes, including sizes that trigger the bug (>~4000 bytes) - test_sizes = [100, 1000, 4000, 5000, 8000] - - for size in test_sizes: - response = await cap.foo(i=size, j=False) - - # Verify the response has the correct length - assert len(response.x) == size, f"Size mismatch for {size}: expected {size}, got {len(response.x)}" - - # Verify the pattern is correct (not corrupted) - expected = "".join(chr(65 + (k % 26)) for k in range(size)) - assert response.x == expected, ( - f"Data corruption detected for {size} bytes payload! " - f"First 50 chars: expected '{expected[:50]}', got '{response.x[:50]}'" - ) - - -async def test_large_response_pipelined(): - """ - Test that pipelining multiple RPC calls with large responses works correctly. - - This is a more aggressive test that sends multiple requests without awaiting, - then collects all results. This pattern is more likely to trigger the - use-after-free bug because multiple messages are queued in the write buffer. - """ - read_sock, write_sock = socket.socketpair() - read_stream = await capnp.AsyncIoStream.create_connection(sock=read_sock) - write_stream = await capnp.AsyncIoStream.create_connection(sock=write_sock) - - _ = capnp.TwoPartyServer(write_stream, bootstrap=LargeResponseServer()) - client = capnp.TwoPartyClient(read_stream) - cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) - - # Test sizes that trigger the bug - send 3 pipelined requests - test_sizes = [5000, 6000, 8000] - - # Send all requests without awaiting (pipelining) - promises = [] - for size in test_sizes: - promise = cap.foo(i=size, j=False) - promises.append((size, promise)) - - # Now await all responses and verify - for size, promise in promises: - response = await promise - - assert len(response.x) == size, f"Size mismatch for {size}" - - expected = "".join(chr(65 + (k % 26)) for k in range(size)) - assert response.x == expected, f"Data corruption detected for {size} bytes payload!" diff --git a/test/test_capability.capnp b/test/test_capability.capnp deleted file mode 100644 index a735538..0000000 --- a/test/test_capability.capnp +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2013, Kenton Varda -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -@0xd508eefec2dc42b8; - -interface TestInterface { - foo @0 (i :UInt32, j :Bool) -> (x: Text); - bar @1 () -> (); - buz @2 (i: TestSturdyRefHostId) -> (x: Text); - bam @3 (i :UInt32, j :Bool) -> (x: Text, i:UInt32); - bak1 @4 () -> (i:List(UInt32)); - bak2 @5 (i:List(UInt32)) -> (); - # baz @2 (s: TestAllTypes); -} - -interface TestExtends extends(TestInterface) { - qux @0 (); -} - -interface TestPipeline { - getCap @0 (n: UInt32, inCap :TestInterface) -> (s: Text, outBox :Box); - testPointers @1 (cap :TestInterface, obj :AnyPointer, list :List(TestInterface)) -> (); - - struct Box { - cap @0 :TestInterface; - } -} - -struct TestSturdyRefHostId { - host @0 :Text; -} - -struct TestSturdyRefObjectId { - tag @0 :Tag; - enum Tag { - testInterface @0; - testExtends @1; - testPipeline @2; - } -} - -interface TestCallOrder { - getCallSequence @0 (expected: UInt32) -> (n: UInt32); - # First call returns 0, next returns 1, ... - # - # The input `expected` is ignored but useful for disambiguating debug logs. -} - -interface TestTailCallee { - struct TailResult { - i @0 :UInt32; - t @1 :Text; - c @2 :TestCallOrder; - } - - foo @0 (i :Int32, t :Text) -> TailResult; -} - -interface TestTailCaller { - foo @0 (i :Int32, callee :TestTailCallee) -> TestTailCallee.TailResult; -} - -interface TestPassedCap { - foo @0 (cap :TestInterface) -> (x: Text); -} - -interface TestStructArg { - bar @0 BarParams -> (c: Text); -} -struct BarParams { - a @0 :Text; - b @1 :Int32; -} - -interface TestGeneric(MyObject) { - foo @0 (a :MyObject) -> (b: Text); -} diff --git a/test/test_capability.py b/test/test_capability.py deleted file mode 100644 index 4dc5dc3..0000000 --- a/test/test_capability.py +++ /dev/null @@ -1,411 +0,0 @@ -import pytest -import asyncio - -import capnp -import test_capability_capnp as capability - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -class Server(capability.TestInterface.Server): - def __init__(self, val=1): - self.val = val - - async def foo(self, i, j, **kwargs): - extra = 0 - if j: - extra = 1 - return str(i * 5 + extra + self.val) - - async def buz(self, i, **kwargs): - return i.host + "_test" - - async def bam(self, i, **kwargs): - return str(i) + "_test", i - - async def bak1(self, **kwargs): - return [1, 2, 3, 4, 5] - - async def bak2(self, i, **kwargs): - assert i[4] == 5 - - -class PipelineServer(capability.TestPipeline.Server): - async def getCap(self, n, inCap, _context, **kwargs): - response = await inCap.foo(i=n) - _results = _context.results - _results.s = response.x + "_foo" - _results.outBox.cap = Server(100) - - -async def test_client(): - client = capability.TestInterface._new_client(Server()) - - req = client._request("foo") - req.i = 5 - - remote = req.send() - response = await remote - - assert response.x == "26" - - req = client.foo_request() - req.i = 5 - - remote = req.send() - response = await remote - - assert response.x == "26" - - with pytest.raises(AttributeError): - client.foo2_request() - - req = client.foo_request() - - with pytest.raises(Exception): - req.i = "foo" - - req = client.foo_request() - - with pytest.raises(AttributeError): - req.baz = 1 - - resp = await client.bak1() - # Used to fail with - # capnp.lib.capnp.KjException: Tried to set field: 'i' with a value of: '[1, 2, 3, 4, 5]' - # which is an unsupported type: '' - await client.bak2(resp.i) - - -async def test_simple_client(): - client = capability.TestInterface._new_client(Server()) - - remote = client._send("foo", i=5) - response = await remote - - assert response.x == "26" - - remote = client.foo(i=5) - response = await remote - - assert response.x == "26" - - remote = client.foo(i=5, j=True) - response = await remote - - assert response.x == "27" - - remote = client.foo(5) - response = await remote - - assert response.x == "26" - - remote = client.foo(5, True) - response = await remote - - assert response.x == "27" - - remote = client.foo(5, j=True) - response = await remote - - assert response.x == "27" - - remote = client.buz(capability.TestSturdyRefHostId.new_message(host="localhost")) - response = await remote - - assert response.x == "localhost_test" - - remote = client.bam(i=5) - response = await remote - - assert response.x == "5_test" - assert response.i == 5 - - with pytest.raises(Exception): - remote = client.foo(5, 10) - - with pytest.raises(Exception): - remote = client.foo(5, True, 100) - - with pytest.raises(Exception): - remote = client.foo(i="foo") - - with pytest.raises(AttributeError): - remote = client.foo2(i=5) - - with pytest.raises(Exception): - remote = client.foo(baz=5) - - -async def test_pipeline(): - client = capability.TestPipeline._new_client(PipelineServer()) - foo_client = capability.TestInterface._new_client(Server()) - - remote = client.getCap(n=5, inCap=foo_client) - - outCap = remote.outBox.cap - pipelinePromise = outCap.foo(i=10) - - response = await pipelinePromise - assert response.x == "150" - - response = await remote - assert response.s == "26_foo" - - -class BadServer(capability.TestInterface.Server): - def __init__(self, val=1): - self.val = val - - async def foo(self, i, j, **kwargs): - extra = 0 - if j: - extra = 1 - return str(i * 5 + extra + self.val), 10 # returning too many args - - -async def test_exception_client(): - client = capability.TestInterface._new_client(BadServer()) - - remote = client._send("foo", i=5) - with pytest.raises(capnp.KjException): - await remote - - -class BadPipelineServer(capability.TestPipeline.Server): - async def getCap(self, n, inCap, _context, **kwargs): - try: - await inCap.foo(i=n) - except capnp.KjException: - raise Exception("test was a success") - - -async def test_exception_chain(): - client = capability.TestPipeline._new_client(BadPipelineServer()) - foo_client = capability.TestInterface._new_client(BadServer()) - - remote = client.getCap(n=5, inCap=foo_client) - - try: - await remote - except Exception as e: - assert "test was a success" in str(e) - - -async def test_pipeline_exception(): - client = capability.TestPipeline._new_client(BadPipelineServer()) - foo_client = capability.TestInterface._new_client(BadServer()) - - remote = client.getCap(n=5, inCap=foo_client) - - outCap = remote.outBox.cap - pipelinePromise = outCap.foo(i=10) - - with pytest.raises(Exception): - await pipelinePromise - - with pytest.raises(Exception): - await remote - - -async def test_casting(): - client = capability.TestExtends._new_client(Server()) - client2 = client.upcast(capability.TestInterface) - _ = client2.cast_as(capability.TestInterface) - - with pytest.raises(Exception): - client.upcast(capability.TestPipeline) - - -class TailCallOrder(capability.TestCallOrder.Server): - def __init__(self): - self.count = -1 - - async def getCallSequence(self, expected, **kwargs): - self.count += 1 - return self.count - - -class TailCaller(capability.TestTailCaller.Server): - def __init__(self): - self.count = 0 - - async def foo(self, i, callee, _context, **kwargs): - self.count += 1 - - tail = callee.foo_request(i=i, t="from TailCaller") - return await _context.tail_call(tail) - - -class TailCallee(capability.TestTailCallee.Server): - def __init__(self): - self.count = 0 - - async def foo(self, i, t, _context, **kwargs): - self.count += 1 - - results = _context.results - results.i = i - results.t = t - results.c = TailCallOrder() - - -async def test_tail_call(): - callee_server = TailCallee() - caller_server = TailCaller() - - callee = capability.TestTailCallee._new_client(callee_server) - caller = capability.TestTailCaller._new_client(caller_server) - - promise = caller.foo(i=456, callee=callee) - dependent_call1 = promise.c.getCallSequence() - - response = await promise - - assert response.i == 456 - assert response.i == 456 - - dependent_call2 = response.c.getCallSequence() - dependent_call3 = response.c.getCallSequence() - - result = await dependent_call1 - assert result.n == 0 - result = await dependent_call2 - assert result.n == 1 - result = await dependent_call3 - assert result.n == 2 - - assert callee_server.count == 1 - assert caller_server.count == 1 - - -async def test_cancel(): - client = capability.TestInterface._new_client(Server()) - - req = client._request("foo") - req.i = 5 - - remote = req.send() - remote.cancel() - - with pytest.raises(Exception): - await remote - - req = client.foo(5) - await req - req.cancel() # Cancel a promise that was already consumed - - req = client.foo(5) - req.cancel() - with pytest.raises(Exception): - await req - - req = client.foo(5) - assert (await req).x == "26" - with pytest.raises(Exception): - await req - - -async def test_double_send(): - client = capability.TestInterface._new_client(Server()) - - req = client._request("foo") - req.i = 5 - - await req.send() - with pytest.raises(Exception): - await req.send() - - -class PromiseJoinServer(capability.TestPipeline.Server): - async def getCap(self, n, inCap, _context, **kwargs): - res = await inCap.foo(i=n) - response = await inCap.foo(i=int(res.x) + 1) - _results = _context.results - _results.s = response.x + "_bar" - _results.outBox.cap = inCap - - -async def test_promise_joining(): - client = capability.TestPipeline._new_client(PromiseJoinServer()) - foo_client = capability.TestInterface._new_client(Server()) - - remote = client.getCap(n=5, inCap=foo_client) - assert (await remote).s == "136_bar" - - -class ExtendsServer(Server): - async def qux(self, **kwargs): - pass - - -async def test_inheritance(): - client = capability.TestExtends._new_client(ExtendsServer()) - await client.qux() - - remote = client.foo(i=5) - response = await remote - - assert response.x == "26" - - -class PassedCapTest(capability.TestPassedCap.Server): - async def foo(self, cap, _context, **kwargs): - res = await cap.foo(5) - _context.results.x = res.x - - -async def test_null_cap(): - client = capability.TestPassedCap._new_client(PassedCapTest()) - assert (await client.foo(Server())).x == "26" - - with pytest.raises(capnp.KjException): - await client.foo() - - -class StructArgTest(capability.TestStructArg.Server): - async def bar(self, a, b, **kwargs): - return a + str(b) - - -async def test_struct_args(): - client = capability.TestStructArg._new_client(StructArgTest()) - assert (await client.bar(a="test", b=1)).c == "test1" - with pytest.raises(capnp.KjException): - assert (await client.bar("test", 1)).c == "test1" - - -class GenericTest(capability.TestGeneric.Server): - async def foo(self, a, **kwargs): - return a.as_text() + "test" - - -async def test_generic(): - client = capability.TestGeneric._new_client(GenericTest()) - - obj = capnp._MallocMessageBuilder().get_root_as_any() - obj.set_as_text("anypointer_") - assert (await client.foo(obj)).b == "anypointer_test" - - -class CancelServer(capability.TestInterface.Server): - def __init__(self, val=1): - self.val = val - - async def foo(self, i, j, **kwargs): - with pytest.raises(asyncio.CancelledError): - await asyncio.sleep(10) - - -async def test_cancel2(): - client = capability.TestInterface._new_client(CancelServer()) - - task = asyncio.ensure_future(client.foo(1, True)) - await asyncio.sleep(0) # Make sure that the task runs - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task diff --git a/test/test_capability_context.py b/test/test_capability_context.py deleted file mode 100644 index 5f05723..0000000 --- a/test/test_capability_context.py +++ /dev/null @@ -1,258 +0,0 @@ -import pytest - -import capnp -import test_capability_capnp as capability - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -class Server(capability.TestInterface.Server): - def __init__(self, val=1): - self.val = val - - async def foo_context(self, context): - extra = 0 - if context.params.j: - extra = 1 - context.results.x = str(context.params.i * 5 + extra + self.val) - - async def buz_context(self, context): - context.results.x = context.params.i.host + "_test" - - -class PipelineServer(capability.TestPipeline.Server): - async def getCap_context(self, context): - response = await context.params.inCap.foo(i=context.params.n) - context.results.s = response.x + "_foo" - context.results.outBox.cap = Server(100) - - -async def test_client_context(): - client = capability.TestInterface._new_client(Server()) - - req = client._request("foo") - req.i = 5 - - remote = req.send() - response = await remote - - assert response.x == "26" - - req = client.foo_request() - req.i = 5 - - remote = req.send() - response = await remote - - assert response.x == "26" - - with pytest.raises(AttributeError): - client.foo2_request() - - req = client.foo_request() - - with pytest.raises(Exception): - req.i = "foo" - - req = client.foo_request() - - with pytest.raises(AttributeError): - req.baz = 1 - - -async def test_simple_client_context(): - client = capability.TestInterface._new_client(Server()) - - remote = client._send("foo", i=5) - response = await remote - - assert response.x == "26" - - remote = client.foo(i=5) - response = await remote - - assert response.x == "26" - - remote = client.foo(i=5, j=True) - response = await remote - - assert response.x == "27" - - remote = client.foo(5) - response = await remote - - assert response.x == "26" - - remote = client.foo(5, True) - response = await remote - - assert response.x == "27" - - remote = client.foo(5, j=True) - response = await remote - - assert response.x == "27" - - remote = client.buz(capability.TestSturdyRefHostId.new_message(host="localhost")) - response = await remote - - assert response.x == "localhost_test" - - with pytest.raises(Exception): - remote = client.foo(5, 10) - - with pytest.raises(Exception): - remote = client.foo(5, True, 100) - - with pytest.raises(Exception): - remote = client.foo(i="foo") - - with pytest.raises(AttributeError): - remote = client.foo2(i=5) - - with pytest.raises(Exception): - remote = client.foo(baz=5) - - -async def test_pipeline_context(): - client = capability.TestPipeline._new_client(PipelineServer()) - foo_client = capability.TestInterface._new_client(Server()) - - remote = client.getCap(n=5, inCap=foo_client) - - outCap = remote.outBox.cap - pipelinePromise = outCap.foo(i=10) - - response = await pipelinePromise - assert response.x == "150" - - response = await remote - assert response.s == "26_foo" - - -class BadServer(capability.TestInterface.Server): - def __init__(self, val=1): - self.val = val - - async def foo_context(self, context): - context.results.x = str(context.params.i * 5 + self.val) - context.results.x2 = 5 # raises exception - - -async def test_exception_client_context(): - client = capability.TestInterface._new_client(BadServer()) - - remote = client._send("foo", i=5) - with pytest.raises(capnp.KjException): - await remote - - -class BadPipelineServer(capability.TestPipeline.Server): - async def getCap_context(self, context): - try: - await context.params.inCap.foo(i=context.params.n) - except capnp.KjException: - raise Exception("test was a success") - - -async def test_exception_chain_context(): - client = capability.TestPipeline._new_client(BadPipelineServer()) - foo_client = capability.TestInterface._new_client(BadServer()) - - remote = client.getCap(n=5, inCap=foo_client) - - try: - await remote - except Exception as e: - assert "test was a success" in str(e) - - -async def test_pipeline_exception_context(): - client = capability.TestPipeline._new_client(BadPipelineServer()) - foo_client = capability.TestInterface._new_client(BadServer()) - - remote = client.getCap(n=5, inCap=foo_client) - - outCap = remote.outBox.cap - pipelinePromise = outCap.foo(i=10) - - with pytest.raises(Exception): - await pipelinePromise - - with pytest.raises(Exception): - await remote - - -async def test_casting_context(): - client = capability.TestExtends._new_client(Server()) - client2 = client.upcast(capability.TestInterface) - _ = client2.cast_as(capability.TestInterface) - - with pytest.raises(Exception): - client.upcast(capability.TestPipeline) - - -class TailCallOrder(capability.TestCallOrder.Server): - def __init__(self): - self.count = -1 - - async def getCallSequence_context(self, context): - self.count += 1 - context.results.n = self.count - - -class TailCaller(capability.TestTailCaller.Server): - def __init__(self): - self.count = 0 - - async def foo_context(self, context): - self.count += 1 - - tail = context.params.callee.foo_request(i=context.params.i, t="from TailCaller") - await context.tail_call(tail) - - -class TailCallee(capability.TestTailCallee.Server): - def __init__(self): - self.count = 0 - - async def foo_context(self, context): - self.count += 1 - - results = context.results - results.i = context.params.i - results.t = context.params.t - results.c = TailCallOrder() - - -async def test_tail_call(): - callee_server = TailCallee() - caller_server = TailCaller() - - callee = capability.TestTailCallee._new_client(callee_server) - caller = capability.TestTailCaller._new_client(caller_server) - - promise = caller.foo(i=456, callee=callee) - dependent_call1 = promise.c.getCallSequence() - - response = await promise - - assert response.i == 456 - assert response.i == 456 - - dependent_call2 = response.c.getCallSequence() - dependent_call3 = response.c.getCallSequence() - - result = await dependent_call1 - assert result.n == 0 - result = await dependent_call2 - assert result.n == 1 - result = await dependent_call3 - assert result.n == 2 - - assert callee_server.count == 1 - assert caller_server.count == 1 diff --git a/test/test_context_manager.py b/test/test_context_manager.py deleted file mode 100644 index 596f6cb..0000000 --- a/test/test_context_manager.py +++ /dev/null @@ -1,241 +0,0 @@ -import pytest -import asyncio -import socket - -import capnp -import test_capability -import test_capability_capnp as capability - - -async def test_two_kj_one_asyncio(): - async with capnp.kj_loop(): - pass - async with capnp.kj_loop(): - pass - - -def test_two_kj_two_asyncio(): - async def do(): - async with capnp.kj_loop(): - pass - - asyncio.run(do()) - asyncio.run(do()) - - -async def test_nested_kj(): - with pytest.raises(RuntimeError) as exninfo: - async with capnp.kj_loop(): - async with capnp.kj_loop(): - pass - assert "The KJ event-loop is already running" in str(exninfo) - - -async def test_kj_loop_leak_new_client(): - async with capnp.kj_loop(): - client = capability.TestInterface._new_client(test_capability.Server()) - with pytest.raises(RuntimeError) as exninfo: - await client.foo(5, True) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_leak_client(): - read, write = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - _ = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - client = capnp.TwoPartyClient(read) - cap = client.bootstrap().cast_as(capability.TestInterface) - with pytest.raises(RuntimeError) as exninfo: - await cap.foo(5, True) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_leak_client2(): - read, write = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - _ = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - client = capnp.TwoPartyClient(read) - with pytest.raises(RuntimeError) as exninfo: - client.bootstrap().cast_as(capability.TestInterface) - assert "This client is closed" in str(exninfo) - - -async def test_kj_loop_leak_client3(): - read, write = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - _ = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - client = capnp.TwoPartyClient(read).bootstrap() - with pytest.raises(RuntimeError) as exninfo: - cap = client.cast_as(capability.TestInterface) - await cap.foo(5, True) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_no_kj_loop(): - read, write = socket.socketpair() - with pytest.raises(RuntimeError) as exninfo: - await capnp.AsyncIoStream.create_connection(sock=read) - assert "The KJ event-loop is not running" in str(exninfo) - with pytest.raises(RuntimeError) as exninfo: - await capnp.AsyncIoStream.create_connection(sock=write) - assert "The KJ event-loop is not running" in str(exninfo) - with pytest.raises(RuntimeError) as exninfo: - capability.TestPipeline._new_client(test_capability.PipelineServer()) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_promise_leaking1(): - async with capnp.kj_loop(): - client = capability.TestInterface._new_client(test_capability.Server()) - remote = client.foo(5, True) - task = asyncio.ensure_future(remote) - await asyncio.sleep(0) - with pytest.raises(capnp.KjException): - await task - - -async def test_promise_leaking2(): - async with capnp.kj_loop(): - client = capability.TestInterface._new_client(test_capability.Server()) - remote = client.foo(5, True) - task = asyncio.ensure_future(remote) - with pytest.raises(RuntimeError) as exninfo: - await task - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_promise_leaking3(): - async with capnp.kj_loop(): - client = capability.TestInterface._new_client(test_capability.Server()) - remote = client.foo(5, True) - with pytest.raises(RuntimeError) as exninfo: - await remote - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_promise_leaking4(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - connection = await capnp.AsyncIoStream.create_connection(sock=read) - client = capnp.TwoPartyClient(connection) - cap = client.bootstrap().cast_as(capability.TestInterface) - res = asyncio.ensure_future(cap.foo(5, True)) - await asyncio.sleep(0) - with pytest.raises(capnp.KjException): - await res - - -async def test_promise_leaking5(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - connection = await capnp.AsyncIoStream.create_connection(sock=read) - client = capnp.TwoPartyClient(connection) - cap = client.bootstrap().cast_as(capability.TestInterface) - res = asyncio.ensure_future(cap.foo(5, True)) - with pytest.raises(RuntimeError) as exninfo: - await res - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_promise_leaking6(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - connection = await capnp.AsyncIoStream.create_connection(sock=read) - client = capnp.TwoPartyClient(connection) - cap = client.bootstrap().cast_as(capability.TestInterface) - res = cap.foo(5, True) - with pytest.raises(RuntimeError) as exninfo: - await res - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_read_message_after_close(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - with pytest.raises(RuntimeError) as exninfo: - await capability.TestSturdyRefHostId.read_async(read) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_partial_read_message_after_close(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - message = capability.TestSturdyRefHostId.read_async(read) - with pytest.raises(RuntimeError) as exninfo: - await message - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_write_message_after_close(): - _, write = socket.socketpair() - async with capnp.kj_loop(): - write = await capnp.AsyncIoStream.create_connection(sock=write) - message = capability.TestSturdyRefHostId.new_message() - with pytest.raises(RuntimeError) as exninfo: - await message.write_async(write) - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_kj_loop_partial_write_message_after_close(): - _, write = socket.socketpair() - async with capnp.kj_loop(): - write = await capnp.AsyncIoStream.create_connection(sock=write) - message = capability.TestSturdyRefHostId.new_message() - send = message.write_async(write) - with pytest.raises(RuntimeError) as exninfo: - await send - assert "The KJ event-loop is not running" in str(exninfo) - - -async def test_client_on_disconnect_memory(): - read, _ = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - client = capnp.TwoPartyClient(read) - with pytest.raises(RuntimeError) as exninfo: - await client.on_disconnect() - assert "This client is closed" in str(exninfo) - - -async def test_server_on_disconnect_memory(): - _, write = socket.socketpair() - async with capnp.kj_loop(): - write = await capnp.AsyncIoStream.create_connection(sock=write) - server = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - with pytest.raises(RuntimeError) as exninfo: - await server.on_disconnect() - assert "This server is closed" in str(exninfo) - - -@pytest.mark.xfail( - strict=True, - reason="Fails because the promisefulfiller got destroyed. Possibly a bug in the C++ library.", -) -async def test_client_on_disconnect_memory2(): - """ - E capnp.lib.capnp.KjException: kj/async.c++:2813: failed: - PromiseFulfiller was destroyed without fulfilling the promise. - """ - read, _ = socket.socketpair() - async with capnp.kj_loop(): - read = await capnp.AsyncIoStream.create_connection(sock=read) - client = capnp.TwoPartyClient(read) - disc = client.on_disconnect() - await disc - - -async def test_server_on_disconnect_memory2(): - _, write = socket.socketpair() - async with capnp.kj_loop(): - write = await capnp.AsyncIoStream.create_connection(sock=write) - server = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - disc = server.on_disconnect() - await disc diff --git a/test/test_examples.py b/test/test_examples.py deleted file mode 100644 index 6eb373f..0000000 --- a/test/test_examples.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -import pytest -import socket -import subprocess -import sys -import time - -examples_dir = os.path.join(os.path.dirname(__file__), "..", "examples") -hostname = "localhost" - - -processes = [] - - -@pytest.fixture -def cleanup(): - yield - for p in processes: - p.kill() - - -def run_subprocesses(address, server, client, wildcard_server=False, ipv4_force=True): # noqa - server_attempt = 0 - server_attempts = 2 - done = False - addr, port = address.split(":") - c_address = address - s_address = address - while not done: - assert server_attempt < server_attempts, "Failed {} server attempts".format(server_attempts) - server_attempt += 1 - - # Force ipv4 for tests (known issues on GitHub Actions with IPv6 for some targets) - if "unix" not in addr and ipv4_force: - addr = socket.gethostbyname(addr) - c_address = "{}:{}".format(addr, port) - s_address = c_address - if wildcard_server: - s_address = "*:{}".format(port) # Use wildcard address for server - print("Forcing ipv4 -> {} => {} {}".format(address, c_address, s_address)) - - # Start server - cmd = [sys.executable, os.path.join(examples_dir, server), s_address] - serverp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) - print("Server started (Attempt #{})".format(server_attempt)) - processes.append(serverp) - retries = 300 - # Loop until we have a socket connection to the server (with timeout) - while True: - try: - if "unix" in address: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - result = sock.connect_ex(port) - if result == 0: - break - else: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - result = sock.connect_ex((addr, int(port))) - if result == 0: - break - except socket.gaierror as err: - print("gaierror: {}".format(err)) - # Give the server some small amount of time to start listening - time.sleep(0.1) - retries -= 1 - if retries == 0: - serverp.kill() - print("Timed out waiting for server to start") - break - - if serverp.poll() is not None: - print("Server exited prematurely: {}".format(serverp.returncode)) - break - - # 3 tries per server try - client_attempt = 0 - client_attempts = 3 - while not done: - if client_attempt >= client_attempts: - print("Failed {} client attempts".format(client_attempts)) - break - client_attempt += 1 - - # Start client - cmd = [sys.executable, os.path.join(examples_dir, client), c_address] - clientp = subprocess.Popen(cmd, stdout=sys.stdout, stderr=sys.stderr) - print("Client started (Attempt #{})".format(client_attempt)) - processes.append(clientp) - - retries = 30 * 10 - # Loop until the client is finished (with timeout) - while True: - if clientp.poll() == 0: - done = True - break - - if clientp.poll() is not None: - print("Client exited prematurely: {}".format(clientp.returncode)) - break - time.sleep(0.1) - retries -= 1 - if retries == 0: - print("Timed out waiting for client to finish") - clientp.kill() - break - - serverp.kill() - - serverp.kill() - - -def test_async_calculator_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_calculator_server.py" - client = "async_calculator_client.py" - run_subprocesses(address, server, client) - - -def test_addressbook_example(cleanup): - proc = subprocess.Popen([sys.executable, os.path.join(examples_dir, "addressbook.py")]) - ret = proc.wait() - assert ret == 0 - - -def test_async_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_server.py" - client = "async_client.py" - run_subprocesses(address, server, client) - - -def test_ssl_async_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_ssl_server.py" - client = "async_ssl_client.py" - run_subprocesses(address, server, client, ipv4_force=False) - - -def test_ssl_reconnecting_async_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_ssl_server.py" - client = "async_reconnecting_ssl_client.py" - run_subprocesses(address, server, client, ipv4_force=False) - - -def test_async_ssl_calculator_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_ssl_calculator_server.py" - client = "async_ssl_calculator_client.py" - run_subprocesses(address, server, client, ipv4_force=False) - - -def test_async_socket_message_example(unused_tcp_port, cleanup): - address = "{}:{}".format(hostname, unused_tcp_port) - server = "async_socket_message_server.py" - client = "async_socket_message_client.py" - run_subprocesses(address, server, client) diff --git a/test/test_get_data_view.py b/test/test_get_data_view.py deleted file mode 100644 index 776366f..0000000 --- a/test/test_get_data_view.py +++ /dev/null @@ -1,292 +0,0 @@ -import os -import tempfile -import weakref -from pathlib import Path - -import pytest -import capnp -import sys -import gc - - -@pytest.fixture(scope="module") -def all_types(): - """Load the standard all_types.capnp schema.""" - directory = os.path.dirname(__file__) - return capnp.load(os.path.join(directory, "all_types.capnp")) - - -def test_set_bytes_get_bytes(all_types): - """ - Scenario 1: Set Byte -> Get Byte - Verify standard behavior: writing bytes results in reading bytes. - """ - msg = all_types.TestAllTypes.new_message() - input_data = b"hello_world" - - # Set - msg.dataField = input_data - - # Get - output_data = msg.dataField - - # Verify - assert isinstance(output_data, bytes) - assert output_data == input_data - - -def test_set_view_get_bytes(all_types): - """ - Scenario 2: Set View -> Get Byte - Verify compatibility: Passing a memoryview sets the data, - but standard attribute access returns a bytes copy. - """ - msg = all_types.TestAllTypes.new_message() - - # Create a memoryview source - raw_source = bytearray(b"view_source") - view = memoryview(raw_source) - - # Set via memoryview - msg.dataField = view - - # Get via standard attribute - output_data = msg.dataField - - # Verify - assert isinstance(output_data, bytes) - assert output_data == b"view_source" - - -def test_set_bytes_get_view_and_modify(all_types): - """ - Scenario 3: Set Byte -> Get View - Verify the high-performance API get_data_as_view. - The view must be writable and modifications must reflect in the message. - """ - msg = all_types.TestAllTypes.new_message() - - # Initial write - msg.dataField = b"ABCDE" - - # Get view via new API - view = msg.get_data_as_view("dataField") - - # Verify view properties - assert isinstance(view, memoryview) - assert view.readonly is False - assert view.tobytes() == b"ABCDE" - - # Verify in-place modification - view[0] = ord("Z") # Change 'A' to 'Z' - - # Verify modification is reflected in standard access - assert msg.dataField == b"ZBCDE" - - -def test_reader_vs_builder_view(all_types): - """ - Verify that Builder views are writable, but Reader views are read-only. - """ - # 1. Builder phase - builder = all_types.TestAllTypes.new_message() - builder.dataField = b"test_rw" - - builder_view = builder.get_data_as_view("dataField") - assert builder_view.readonly is False - builder_view[0] = ord("T") # Modification allowed - - # 2. Reader phase - reader = builder.as_reader() - - # Standard Get - assert reader.dataField == b"Test_rw" - - # Reader get_data_as_view - reader_view = reader.get_data_as_view("dataField") - assert isinstance(reader_view, memoryview) - assert reader_view.readonly is True - - # Attempting to modify Reader view should raise TypeError - with pytest.raises(TypeError): - reader_view[0] = ord("X") - - -def test_nested_struct_data(all_types): - """ - Verify that get_data_as_view works correctly on nested structs. - """ - msg = all_types.TestAllTypes.new_message() - - # Initialize nested struct - inner = msg.init("structField") - inner.int32Field = 100 - inner.dataField = b"nested_data" - - # 1. Verify standard access - assert msg.structField.dataField == b"nested_data" - - # 2. Verify nested get_data_as_view - view = msg.structField.get_data_as_view("dataField") - - assert isinstance(view, memoryview) - assert view.tobytes() == b"nested_data" - - # Modify nested data - view[0] = ord("N") - assert msg.structField.dataField == b"Nested_data" - - -def test_corner_cases_values(all_types): - """ - Test edge cases: Empty bytes and binary data with nulls. - """ - msg = all_types.TestAllTypes.new_message() - - # Case A: Empty Bytes - msg.dataField = b"" - assert msg.dataField == b"" - view = msg.get_data_as_view("dataField") - assert len(view) == 0 - - # Case B: Binary data containing null bytes - binary_data = b"\x00\xff\x00\x01" - msg.dataField = binary_data - assert msg.dataField == binary_data - assert msg.get_data_as_view("dataField").tobytes() == binary_data - - -def test_uninitialized_data_get_view(all_types): - """ - Default DATA fields should expose an empty memoryview instead of failing on a NULL buffer pointer. - """ - builder = all_types.TestAllTypes.new_message() - builder_view = builder.get_data_as_view("dataField") - - assert isinstance(builder_view, memoryview) - assert builder_view.readonly is False - assert len(builder_view) == 0 - assert builder_view.tobytes() == b"" - - reader = all_types.TestAllTypes.new_message().as_reader() - reader_view = reader.get_data_as_view("dataField") - - assert isinstance(reader_view, memoryview) - assert reader_view.readonly is True - assert len(reader_view) == 0 - assert reader_view.tobytes() == b"" - - with pytest.raises(IndexError): - builder_view[0] = 0xFF - - with pytest.raises(ValueError): - builder_view[0:1] = b"\xff" - - -def test_error_wrong_type(all_types): - """ - Test error handling: Calling get_data_as_view on non-Data fields. - """ - msg = all_types.TestAllTypes.new_message() - msg.int32Field = 123 - msg.textField = "I am text" - - # Attempt on Int field - with pytest.raises(TypeError) as excinfo: - msg.get_data_as_view("int32Field") - assert "not a DATA field" in str(excinfo.value) - - # Attempt on Text field - with pytest.raises(TypeError) as excinfo: - msg.get_data_as_view("textField") - assert "not a DATA field" in str(excinfo.value) - - -def test_error_missing_field(all_types): - """ - Test error handling: Accessing a non-existent field name. - """ - msg = all_types.TestAllTypes.new_message() - - # Accessing a missing field should raise AttributeError (standard Python behavior) - with pytest.raises(AttributeError) as excinfo: - msg.get_data_as_view("non_existent_field") - - # Optional: Verify the error message contains the field name - assert "non_existent_field" in str(excinfo.value) - - -def test_view_keeps_message_alive(all_types): - """ - Verify that a View keeps messages alive. - """ - msg = all_types.TestAllTypes.new_message() - expected_data = b"persistence_check" - msg.dataField = expected_data - - initial_ref_count = sys.getrefcount(msg) - view = msg.get_data_as_view("dataField") - new_ref_count = sys.getrefcount(msg) - - assert new_ref_count > initial_ref_count, ( - f"View failed to hold reference to Message! (Old: {initial_ref_count}, New: {new_ref_count})" - ) - print(f"\n[Ref Check] Success: Ref count increased from {initial_ref_count} to {new_ref_count}") - - del msg - gc.collect() - - assert view.tobytes() == expected_data - - -def test_data_view_exports_through_buffer_exporter(all_types): - """Returned memoryviews should pin an internal exporter, not bare pointers.""" - msg = all_types.TestAllTypes.new_message() - msg.dataField = b"exporter_check" - view = msg.get_data_as_view("dataField") - - assert isinstance(view, memoryview) - assert view.obj is not None - assert len(view.obj) == len(view) - - -def test_data_view_survives_del_builder(all_types): - msg = all_types.TestAllTypes.new_message() - msg.dataField = b"persistence_check" - view = msg.get_data_as_view("dataField") - - del msg - gc.collect() - - assert view.tobytes() == b"persistence_check" - - -def test_data_view_releases_packed_payload(): - schema_text = """ - @0x9d7d4f087df9b6e1; - struct BlobMsg { - data @0 :Data; - } - """ - - class Payload(bytearray): - pass - - td = tempfile.TemporaryDirectory() - path = Path(td.name) / "blob.capnp" - path.write_text(schema_text) - schema = capnp.load(str(path)) - try: - payload = Payload(schema.BlobMsg.new_message(data=b"x" * 4096).to_bytes_packed()) - payload_ref = weakref.ref(payload) - - reader = schema.BlobMsg.from_bytes_packed(payload) - view = reader.get_data_as_view("data") - view.release() - - del view, reader, payload - gc.collect() - - assert payload_ref() is None - finally: - td.cleanup() diff --git a/test/test_large_read.py b/test/test_large_read.py index fc064cb..24c9cf1 100644 --- a/test/test_large_read.py +++ b/test/test_large_read.py @@ -24,27 +24,15 @@ def test_large_read(test_capnp): for i in range(len(values)): values[i] = i - array.write_packed(f) + f.write(array.to_bytes()) f.seek(0) - array = test_capnp.MultiArray.read_packed(f) + with test_capnp.MultiArray.from_bytes(f.read()) as reader: + array = reader del f assert array.rows[0].values[9000] == 9000 -def test_large_read_multiple(test_capnp): - f = tempfile.TemporaryFile() - msg1 = test_capnp.Msg.new_message() - msg1.data = [0x41] * 8192 - msg1.write(f) - msg2 = test_capnp.Msg.new_message() - msg2.write(f) - f.seek(0) - - for m in test_capnp.Msg.read_multiple(f): - pass - - def get_two_adjacent_messages(test_capnp): msg1 = test_capnp.Msg.new_message() msg1.data = [0x41] * 8192 diff --git a/test/test_lifetime.py b/test/test_lifetime.py new file mode 100644 index 0000000..5085a4f --- /dev/null +++ b/test/test_lifetime.py @@ -0,0 +1,60 @@ +"""Lifetimes used by realtime message producers and retained log readers.""" + +import gc +from pathlib import Path + +import capnp +import pytest + + +@pytest.fixture +def schema(): + return capnp.load(str(Path(__file__).with_name("addressbook.capnp"))) + + +def builder_count(): + return sum(type(obj) is capnp._MallocMessageBuilder for obj in gc.get_objects()) + + +def test_kwargs_release_without_gc(schema): + # Warm lazy schema state before counting. String fields must not create + # schema/field cycles that keep whole arenas alive until the next GC pass. + schema.Person.new_message(name="warmup") + gc.collect() + enabled = gc.isenabled() + gc.disable() + try: + before = builder_count() + for _ in range(1000): + msg = schema.Person.new_message(name="Alice", phones=[{"type": "mobile"}]) + assert msg.name == "Alice" + assert msg.phones[0].type == "mobile" + del msg + assert builder_count() == before + finally: + if enabled: + gc.enable() + + +def test_unknown_string_field_raises(schema): + with pytest.raises(capnp.KjException): + schema.Person.new_message(notAField="Alice") + + +def test_readers_outlive_input_and_iterator(schema): + data = b"".join(schema.Person.new_message(name=name).to_bytes() for name in ("Alice", "Bob")) + messages = schema.Person.read_multiple_bytes(data) + alice = next(messages) + bob = next(messages) + del data, messages + gc.collect() + assert (alice.name, bob.name) == ("Alice", "Bob") + + +def test_nested_reader_outlives_root(schema): + data = schema.AddressBook.new_message(people=[{"name": "Alice"}]).to_bytes() + with schema.AddressBook.from_bytes(data) as root: + person = root.people[0] + del data, root + gc.collect() + assert person.name == "Alice" diff --git a/test/test_load.py b/test/test_load.py index c4ccf7b..34dbd7b 100644 --- a/test/test_load.py +++ b/test/test_load.py @@ -1,7 +1,6 @@ import pytest import capnp import os -import sys this_dir = os.path.dirname(__file__) @@ -62,93 +61,3 @@ def test_failed_import(): with pytest.raises(Exception): bar.foo = foo - - -def test_defualt_import_hook(): - # Make sure any previous imports of addressbook_capnp are gone - capnp.cleanup_global_schema_parser() - - import addressbook_capnp # noqa: F401 - - -def test_dash_import(): - import addressbook_with_dashes_capnp # noqa: F401 - - -def test_spaces_import(): - import addressbook_with_spaces_capnp # noqa: F401 - - -def test_add_import_hook(): - capnp.add_import_hook() - - # Make sure any previous imports of addressbook_capnp are gone - capnp.cleanup_global_schema_parser() - - import addressbook_capnp - - addressbook_capnp.AddressBook.new_message() - - -def test_multiple_add_import_hook(): - capnp.add_import_hook() - capnp.add_import_hook() - - # Make sure any previous imports of addressbook_capnp are gone - capnp.cleanup_global_schema_parser() - - import addressbook_capnp - - addressbook_capnp.AddressBook.new_message() - - -def test_remove_import_hook(): - capnp.add_import_hook() - capnp.remove_import_hook() - - if "addressbook_capnp" in sys.modules: - # hack to deal with it being imported already - del sys.modules["addressbook_capnp"] - - with pytest.raises(ImportError): - import addressbook_capnp # noqa: F401 - - -def test_bundled_import_hook(): - # stream.capnp should be bundled, or provided by the system capnproto - capnp.add_import_hook() - from capnp import stream_capnp # noqa: F401 - - -def test_nested_import(): - import schemas.parent_capnp # noqa: F401 - import schemas.child_capnp # noqa: F401 - - -async def test_load_capnp(foo): - # test dynamically loading - loader = capnp.SchemaLoader() - loader.load(foo.Baz.schema.get_proto()) - loader.load_dynamic(foo.Qux.schema.get_proto().node) - - schema = loader.get(foo.Baz.schema.get_proto().node.id).as_struct() - assert "text" in schema.fieldnames - assert "qux" in schema.fieldnames - assert schema.fields["qux"].proto.slot.type.which == "struct" - - class Wrapper(foo.Wrapper.Server): - async def wrapped(self, object, **kwargs): - assert isinstance(object, capnp.lib.capnp._DynamicObjectReader) - baz_ = object.as_struct(schema) - assert baz_.text == "test" - assert baz_.qux.id == 2 - - # test calling into the wrapper with a Baz message. - baz_ = foo.Baz.new_message() - baz_.text = "test" - baz_.qux.id = 2 - - async with capnp.kj_loop(): - wrapper = foo.Wrapper._new_client(Wrapper()) - remote = wrapper.wrapped(baz_) - await remote diff --git a/test/test_memory_handling.py b/test/test_memory_handling.py deleted file mode 100644 index ea3b6d8..0000000 --- a/test/test_memory_handling.py +++ /dev/null @@ -1,32 +0,0 @@ -from types import coroutine -import pytest -import socket -import gc - -import capnp -import test_capability -import test_capability_capnp as capability - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -@coroutine -def wrap(p): - return (yield from p) - - -async def test_kj_loop_await_attach(): - read, write = socket.socketpair() - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - _ = capnp.TwoPartyServer(write, bootstrap=test_capability.Server()) - client = capnp.TwoPartyClient(read).bootstrap().cast_as(capability.TestInterface) - t = wrap(client.foo(5, True).__await__()) - del client - del read - gc.collect() - await t diff --git a/test/test_object.py b/test/test_object.py deleted file mode 100644 index b9dc42e..0000000 --- a/test/test_object.py +++ /dev/null @@ -1,51 +0,0 @@ -import pytest -import capnp -import os - -this_dir = os.path.dirname(__file__) - - -@pytest.fixture -def addressbook(): - return capnp.load(os.path.join(this_dir, "addressbook.capnp")) - - -def test_object_basic(addressbook): - obj = capnp._MallocMessageBuilder().get_root_as_any() - person = obj.as_struct(addressbook.Person) - person.name = "test" - person.id = 1000 - - same_person = obj.as_struct(addressbook.Person) - assert same_person.name == "test" - assert same_person.id == 1000 - - obj_r = obj.as_reader() - same_person = obj_r.as_struct(addressbook.Person) - assert same_person.name == "test" - assert same_person.id == 1000 - - -def test_object_list(addressbook): - obj = capnp._MallocMessageBuilder().get_root_as_any() - listSchema = capnp._ListSchema(addressbook.Person) - people = obj.init_as_list(listSchema, 2) - person = people[0] - person.name = "test" - person.id = 1000 - person = people[1] - person.name = "test2" - person.id = 1001 - - same_person = obj.as_list(listSchema) - assert same_person[0].name == "test" - assert same_person[0].id == 1000 - assert same_person[1].name == "test2" - assert same_person[1].id == 1001 - - obj_r = obj.as_reader() - same_person = obj_r.as_list(listSchema) - assert same_person[0].name == "test" - assert same_person[0].id == 1000 - assert same_person[1].name == "test2" - assert same_person[1].id == 1001 diff --git a/test/test_openpilot.py b/test/test_openpilot.py new file mode 100644 index 0000000..c9df6df --- /dev/null +++ b/test/test_openpilot.py @@ -0,0 +1,97 @@ +"""Optional integration checks against a local openpilot checkout. + +Set OPENPILOT_PATH and run with openpilot's dependencies available. These tests +use the checkout's real schemas; they never copy or modify them. +""" + +import os +import pickle +import sys +from pathlib import Path + +import capnp +import pytest + + +@pytest.fixture(scope="module") +def openpilot(): + path = os.environ.get("OPENPILOT_PATH") + if path is None: + pytest.skip("set OPENPILOT_PATH to run openpilot integration tests") + sys.path.insert(0, str(Path(path).resolve())) + from openpilot.cereal import log + + return log + + +def test_event_types(openpilot): + from openpilot.cereal import messaging + from openpilot.cereal.services import SERVICE_LIST + + for event in openpilot.Event.schema.union_fields: + if event not in SERVICE_LIST: + continue + try: + msg = messaging.new_message(event) + except capnp.KjException: + msg = messaging.new_message(event, 2) + with openpilot.Event.from_bytes(msg.to_bytes()) as reader: + assert reader.which() == event + assert reader.logMonoTime == msg.logMonoTime + assert reader.valid == msg.valid + + +def test_can_fields(openpilot): + from openpilot.selfdrive.pandad.pandad_api_impl import can_capnp_to_list, can_list_to_can_capnp + + frames = [(0x123, b"\x00\xff\x01", 0), (0x456, b"\x02\x03", 1)] + raw = can_list_to_can_capnp(frames) + # Exercise both the writer's and reader's cached schema field paths. + decoded = can_capnp_to_list([raw]) + assert decoded[0][1] == frames + with openpilot.Event.from_bytes(raw) as reader: + assert [(f.address, f.dat, f.src) for f in reader.can] == frames + + +def test_schema_reflection(openpilot): + from openpilot.system.webrtc.schema import generate_struct + + from opendbc.car.structs import car + + schema = generate_struct(car.CarState.schema) + assert schema["vEgo"] == "float32" + assert schema["gearShifter"] == "text" + assert isinstance(schema["wheelSpeeds"], dict) + assert ( + car.CarParams.schema.fields["safetyConfigs"].schema.elementType.node.id + == car.CarParams.SafetyConfig.schema.node.id + ) + + +def test_logreader_and_pickle(openpilot): + from openpilot.cereal import messaging + from openpilot.tools.lib.logreader import LogReader + + msgs = [] + for i in range(20): + msg = messaging.new_message("carState") + msg.carState.vEgo = float(i) + msgs.append(msg.to_bytes()) + readers = list(LogReader.from_bytes(b"".join(msgs))) + assert [msg.carState.vEgo for msg in readers] == list(range(20)) + restored = pickle.loads(pickle.dumps(readers[3])) + assert restored.which() == "carState" + assert restored.carState.vEgo == 3.0 + + +def test_fuzzy_messages_and_replay_comparison(openpilot): + from openpilot.common.fuzzy import Fuzzy, capnp_random_dict + from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs + + for event in ("carState", "carControl", "carParams", "modelV2", "can", "extrinsicsCalibration"): + data = capnp_random_dict(Fuzzy(42, 51), openpilot.Event.schema, event, real_floats=True) + builder = openpilot.Event.new_message(**data) + with openpilot.Event.from_bytes(builder.to_bytes()) as reader: + assert reader.which() == event + assert reader.to_dict() == builder.to_dict() + assert compare_logs([reader], [reader.as_builder().as_reader()]) == [] diff --git a/test/test_py_custom_message_builder.py b/test/test_py_custom_message_builder.py deleted file mode 100644 index 5011b5c..0000000 --- a/test/test_py_custom_message_builder.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -import pytest -import capnp # noqa: F401 -import os - -this_dir = os.path.dirname(__file__) - - -@pytest.fixture -def all_types(): - return capnp.load(os.path.join(this_dir, "all_types.capnp")) - - -def test_bytearray_allocator(all_types): - class Allocator: - def __init__(self): - self.cur_size = 0 - self.last_size = 0 - - def __call__(self, minimum_size: int) -> bytearray: - actual_size = max(minimum_size, self.cur_size) - self.last_size = actual_size - self.cur_size += actual_size - WORD_SIZE = 8 - byte_count = actual_size * WORD_SIZE - return bytearray(byte_count) - - allocator = Allocator() - assert allocator.cur_size == 0 - assert allocator.last_size == 0 - msg_builder = capnp._PyCustomMessageBuilder(allocator, 1024) - struct_builder = msg_builder.init_root(all_types.TestAllTypes) - assert allocator.cur_size == 1024 - assert allocator.last_size == 1024 - - struct_builder.init("dataField", 5) - assert bytes(struct_builder._get("dataField")) == b"\x00\x00\x00\x00\x00" - - struct_builder.dataField = b"hello" - assert bytes(struct_builder._get("dataField")) == b"hello" - - struct_reader = struct_builder.as_reader() - assert bytes(struct_reader._get("dataField")) == b"hello" - - -def test_memoryview_allocator(all_types): - class MemoryViewAllocator: - def __init__(self): - self.cur_size = 0 - self.last_size = 0 - self.buffers = [] - - def __call__(self, minimum_size: int) -> memoryview: - actual_size = max(minimum_size, self.cur_size) - self.last_size = actual_size - self.cur_size += actual_size - WORD_SIZE = 8 - byte_count = actual_size * WORD_SIZE - buffer = bytearray(byte_count) - self.buffers.append(buffer) - return memoryview(buffer) - - allocator = MemoryViewAllocator() - assert allocator.cur_size == 0 - assert allocator.last_size == 0 - msg_builder = capnp._PyCustomMessageBuilder(allocator, 1024) - struct_builder = msg_builder.init_root(all_types.TestAllTypes) - assert allocator.cur_size == 1024 - assert allocator.last_size == 1024 - - struct_builder.init("dataField", 5) - assert bytes(struct_builder._get("dataField")) == b"\x00\x00\x00\x00\x00" - - struct_builder.dataField = b"hello" - assert bytes(struct_builder._get("dataField")) == b"hello" - - struct_reader = struct_builder.as_reader() - assert bytes(struct_reader._get("dataField")) == b"hello" diff --git a/test/test_regression.py b/test/test_regression.py index bcf1bc5..5029bf0 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -20,7 +20,7 @@ def addressbook(): def test_addressbook_message_classes(addressbook): - def writeAddressBook(fd): + def writeAddressBook(file): message = capnp._MallocMessageBuilder() addressBook = message.init_root(addressbook.AddressBook) people = addressBook.init("people", 2) @@ -45,11 +45,11 @@ def test_addressbook_message_classes(addressbook): bobPhones[1].type = "work" bob.employment.unemployed = None - capnp._write_packed_message_to_fd(fd, message) + file.write(addressBook.to_bytes()) - def printAddressBook(fd): - message = capnp._PackedFdMessageReader(f) - addressBook = message.get_root(addressbook.AddressBook) + def printAddressBook(file): + with addressbook.AddressBook.from_bytes(file.read()) as reader: + addressBook = reader people = addressBook.people @@ -73,11 +73,11 @@ def test_addressbook_message_classes(addressbook): assert bobPhones[1].type == "work" assert bob.employment.unemployed is None - f = open("example", "w") - writeAddressBook(f.fileno()) + f = open("example", "wb") + writeAddressBook(f) - f = open("example", "r") - printAddressBook(f.fileno()) + f = open("example", "rb") + printAddressBook(f) def test_addressbook(addressbook): @@ -105,10 +105,11 @@ def test_addressbook(addressbook): bobPhones[1].type = "work" bob.employment.unemployed = None - addresses.write(file) + file.write(addresses.to_bytes()) def printAddressBook(file): - addresses = addressbook.AddressBook.read(file) + with addressbook.AddressBook.from_bytes(file.read()) as reader: + addresses = reader people = addresses.people @@ -132,71 +133,10 @@ def test_addressbook(addressbook): assert bobPhones[1].type == "work" assert bob.employment.unemployed is None - f = open("example", "w") + f = open("example", "wb") writeAddressBook(f) - f = open("example", "r") - printAddressBook(f) - - -def test_addressbook_resizable(addressbook): - def writeAddressBook(file): - addresses = addressbook.AddressBook.new_message() - people = addresses.init_resizable_list("people") - - alice = people.add() - alice.id = 123 - alice.name = "Alice" - alice.email = "alice@example.com" - alicePhones = alice.init("phones", 1) - alicePhones[0].number = "555-1212" - alicePhones[0].type = "mobile" - alice.employment.school = "MIT" - - bob = people.add() - bob.id = 456 - bob.name = "Bob" - bob.email = "bob@example.com" - bobPhones = bob.init("phones", 2) - bobPhones[0].number = "555-4567" - bobPhones[0].type = "home" - bobPhones[1].number = "555-7654" - bobPhones[1].type = "work" - bob.employment.unemployed = None - - people.finish() - - addresses.write(file) - - def printAddressBook(file): - addresses = addressbook.AddressBook.read(file) - - people = addresses.people - - alice = people[0] - assert alice.id == 123 - assert alice.name == "Alice" - assert alice.email == "alice@example.com" - alicePhones = alice.phones - assert alicePhones[0].number == "555-1212" - assert alicePhones[0].type == "mobile" - assert alice.employment.school == "MIT" - - bob = people[1] - assert bob.id == 456 - assert bob.name == "Bob" - assert bob.email == "bob@example.com" - bobPhones = bob.phones - assert bobPhones[0].number == "555-4567" - assert bobPhones[0].type == "home" - assert bobPhones[1].number == "555-7654" - assert bobPhones[1].type == "work" - assert bob.employment.unemployed is None - - f = open("example", "w") - writeAddressBook(f) - - f = open("example", "r") + f = open("example", "rb") printAddressBook(f) @@ -230,10 +170,11 @@ def test_addressbook_explicit_fields(addressbook): employment = bob._get_by_field(person_fields["employment"]) employment._set_by_field(addressbook.Person.Employment.schema.fields["unemployed"], None) - addresses.write(file) + file.write(addresses.to_bytes()) def printAddressBook(file): - addresses = addressbook.AddressBook.read(file) + with addressbook.AddressBook.from_bytes(file.read()) as reader: + addresses = reader address_fields = addressbook.AddressBook.schema.fields person_fields = addressbook.Person.schema.fields phone_fields = addressbook.Person.PhoneNumber.schema.fields @@ -262,10 +203,10 @@ def test_addressbook_explicit_fields(addressbook): employment = bob._get_by_field(person_fields["employment"]) employment._get_by_field(addressbook.Person.Employment.schema.fields["unemployed"]) is None - f = open("example", "w") + f = open("example", "wb") writeAddressBook(f) - f = open("example", "r") + f = open("example", "rb") printAddressBook(f) @@ -515,8 +456,9 @@ def test_build_first_segment_size(all_types): def test_binary_read(all_types): - f = open(os.path.join(this_dir, "all-types.binary"), "r", encoding="utf8") - root = all_types.TestAllTypes.read(f) + f = open(os.path.join(this_dir, "all-types.binary"), "rb") + with all_types.TestAllTypes.from_bytes(f.read()) as reader: + root = reader check_all_types(root) expectedText = open(os.path.join(this_dir, "all-types.txt"), "r", encoding="utf8").read() @@ -532,26 +474,10 @@ def test_binary_read(all_types): check_all_types(builder2.get_root(all_types.TestAllTypes)) -def test_packed_read(all_types): - f = open(os.path.join(this_dir, "all-types.packed"), "r", encoding="utf8") - root = all_types.TestAllTypes.read_packed(f) - check_all_types(root) - - expectedText = open(os.path.join(this_dir, "all-types.txt"), "r", encoding="utf8").read() - assert str(root) + "\n" == expectedText - - def test_binary_write(all_types): root = all_types.TestAllTypes.new_message() init_all_types(root) - root.write(open("example", "w")) + open("example", "wb").write(root.to_bytes()) - check_all_types(all_types.TestAllTypes.read(open("example", "r"))) - - -def test_packed_write(all_types): - root = all_types.TestAllTypes.new_message() - init_all_types(root) - root.write_packed(open("example", "w")) - - check_all_types(all_types.TestAllTypes.read_packed(open("example", "r"))) + with all_types.TestAllTypes.from_bytes(open("example", "rb").read()) as reader: + check_all_types(reader) diff --git a/test/test_response.capnp b/test/test_response.capnp deleted file mode 100644 index 268bc08..0000000 --- a/test/test_response.capnp +++ /dev/null @@ -1,13 +0,0 @@ -@0x84249be5c3bff005; - -interface Foo { - foo @0 () -> (val :UInt32); -} - -struct Bar { - foo @0 :Foo; -} - -interface Baz { - grault @0 () -> (bar: Bar); -} diff --git a/test/test_response.py b/test/test_response.py deleted file mode 100644 index 56b24f5..0000000 --- a/test/test_response.py +++ /dev/null @@ -1,48 +0,0 @@ -import pytest - -import capnp -import test_response_capnp - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -class FooServer(test_response_capnp.Foo.Server): - def __init__(self, val=1): - self.val = val - - async def foo(self, **kwargs): - return 1 - - -class BazServer(test_response_capnp.Baz.Server): - def __init__(self, val=1): - self.val = val - - async def grault(self, **kwargs): - return {"foo": FooServer()} - - -async def test_response_reference(): - baz = test_response_capnp.Baz._new_client(BazServer()) - - bar = (await baz.grault()).bar - - foo = bar.foo - # This used to cause an exception about invalid pointers because the response got garbage collected - assert (await foo.foo()).val == 1 - - -async def test_response_reference2(): - baz = test_response_capnp.Baz._new_client(BazServer()) - - bar = (await baz.grault()).bar - - # This always worked since it saved the intermediate response object - response = await baz.grault() - bar = response.bar - foo = bar.foo - assert (await foo.foo()).val == 1 diff --git a/test/test_rpc.py b/test/test_rpc.py deleted file mode 100644 index 2e918bd..0000000 --- a/test/test_rpc.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -rpc test -""" - -import pytest -import capnp -import socket - -import test_capability_capnp - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -class Server(test_capability_capnp.TestInterface.Server): - def __init__(self, val=100): - self.val = val - - async def foo(self, i, j, **kwargs): - return str(i * 5 + self.val) - - -async def test_simple_rpc_with_options(): - read, write = socket.socketpair() - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - - _ = capnp.TwoPartyServer(write, bootstrap=Server()) - # This traversal limit is too low to receive the response in, so we expect - # an exception during the call. - client = capnp.TwoPartyClient(read, traversal_limit_in_words=1) - - with pytest.raises(capnp.KjException): - cap = client.bootstrap().cast_as(test_capability_capnp.TestInterface) - - remote = cap.foo(i=5) - _ = remote.wait() - - -async def test_simple_rpc_bootstrap(): - read, write = socket.socketpair() - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - - _ = capnp.TwoPartyServer(write, bootstrap=Server(100)) - client = capnp.TwoPartyClient(read) - - cap = client.bootstrap() - cap = cap.cast_as(test_capability_capnp.TestInterface) - - # Check not only that the methods are there, but also that they are listed - # as expected. - assert "foo" in dir(cap) - assert "bar" in dir(cap) - assert "buz" in dir(cap) - assert "bam" in dir(cap) - - remote = cap.foo(i=5) - response = await remote - - assert response.x == "125" diff --git a/test/test_rpc_calculator.py b/test/test_rpc_calculator.py deleted file mode 100644 index f0b1c7f..0000000 --- a/test/test_rpc_calculator.py +++ /dev/null @@ -1,50 +0,0 @@ -import gc -import os -import socket -import sys # add examples dir to sys.path -import pytest - -import capnp - -examples_dir = os.path.join(os.path.dirname(__file__), "..", "examples") -sys.path.append(examples_dir) - -import async_calculator_client # noqa: E402 -import async_calculator_server # noqa: E402 - - -@pytest.fixture(autouse=True) -async def kj_loop(): - async with capnp.kj_loop(): - yield - - -async def test_calculator(): - read, write = socket.socketpair() - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - - _ = capnp.TwoPartyServer(write, bootstrap=async_calculator_server.CalculatorImpl()) - await async_calculator_client.main(read) - - -async def test_calculator_gc(): - def new_evaluate_impl(old_evaluate_impl): - def call(*args, **kwargs): - gc.collect() - return old_evaluate_impl(*args, **kwargs) - - return call - - read, write = socket.socketpair() - read = await capnp.AsyncIoStream.create_connection(sock=read) - write = await capnp.AsyncIoStream.create_connection(sock=write) - - # inject a gc.collect to the beginning of every evaluate_impl call - evaluate_impl_orig = async_calculator_server.evaluate_impl - async_calculator_server.evaluate_impl = new_evaluate_impl(evaluate_impl_orig) - - _ = capnp.TwoPartyServer(write, bootstrap=async_calculator_server.CalculatorImpl()) - await async_calculator_client.main(read) - - async_calculator_server.evaluate_impl = evaluate_impl_orig diff --git a/test/test_schema.py b/test/test_schema.py index 0af0942..566aecc 100644 --- a/test/test_schema.py +++ b/test/test_schema.py @@ -10,11 +10,6 @@ def addressbook(): return capnp.load(os.path.join(this_dir, "addressbook.capnp")) -@pytest.fixture -def annotations(): - return capnp.load(os.path.join(this_dir, "annotations.capnp")) - - def test_basic_schema(addressbook): assert addressbook.Person.schema.fieldnames[0] == "id" @@ -24,27 +19,3 @@ def test_list_schema(addressbook): personType = peopleField.schema.elementType assert personType.node.id == addressbook.Person.schema.node.id - - personListSchema = capnp._ListSchema(addressbook.Person) - - assert personListSchema.elementType.node.id == addressbook.Person.schema.node.id - - -def test_annotations(annotations): - assert annotations.schema.node.annotations[0].value.text == "TestFile" - - annotation = annotations.TestAnnotationOne.schema.node.annotations[0] - assert annotation.value.text == "Test" - - annotation = annotations.TestAnnotationTwo.schema.node.annotations[0] - assert annotation.value.struct.as_struct(annotations.AnnotationStruct).test == 100 - - annotation = annotations.TestAnnotationThree.schema.node.annotations[0] - annotation_list = annotation.value.list.as_list(capnp._ListSchema(annotations.AnnotationStruct)) - assert annotation_list[0].test == 100 - assert annotation_list[1].test == 101 - - annotation = annotations.TestAnnotationFour.schema.node.annotations[0] - annotation_list = annotation.value.list.as_list(capnp._ListSchema(capnp.types.UInt16)) - assert annotation_list[0] == 200 - assert annotation_list[1] == 201 diff --git a/test/test_serialization.py b/test/test_serialization.py index d21b086..855038b 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -1,11 +1,9 @@ import warnings from contextlib import contextmanager -import gc import pytest import capnp import os -import platform import test_regression import tempfile import pickle @@ -20,28 +18,6 @@ def all_types(): return capnp.load(os.path.join(this_dir, "all_types.capnp")) -def test_roundtrip_file(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write(f) - - f.seek(0) - msg = all_types.TestAllTypes.read(f) - test_regression.check_all_types(msg) - - -def test_roundtrip_file_packed(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write_packed(f) - - f.seek(0) - msg = all_types.TestAllTypes.read_packed(f) - test_regression.check_all_types(msg) - - def test_roundtrip_bytes(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -51,105 +27,6 @@ def test_roundtrip_bytes(all_types): test_regression.check_all_types(msg) -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="TODO: Investigate why this works on CPython but fails on PyPy.", -) -def test_roundtrip_segments(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - segments = msg.to_segments() - msg = all_types.TestAllTypes.from_segments(segments) - test_regression.check_all_types(msg) - - -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="TODO: Investigate segmented serialization support on PyPy.", -) -def test_segment_views_are_read_only_buffers(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - - segments = msg.to_segments() - segment_views = msg.to_segment_views() - - assert len(segment_views) == len(segments) - assert len(segment_views) >= 1 - - for segment_view, segment_bytes in zip(segment_views, segments): - assert not isinstance(segment_view, bytes) - view = memoryview(segment_view) - try: - assert view.readonly is True - assert view.tobytes() == segment_bytes - finally: - view.release() - - -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="TODO: Investigate segmented serialization support on PyPy.", -) -def test_roundtrip_segment_views(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - - segment_views = msg.to_segment_views() - msg = all_types.TestAllTypes.from_segments(segment_views) - test_regression.check_all_types(msg) - - -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="TODO: Investigate segmented serialization support on PyPy.", -) -def test_segment_views_are_not_writable(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - - segment_views = msg.to_segment_views() - view = memoryview(segment_views[0]) - try: - assert len(view) > 0 - with pytest.raises(TypeError): - view[0] = 0 - finally: - view.release() - - -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="TODO: Investigate segmented serialization support on PyPy.", -) -def test_segment_view_keeps_message_alive(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - - segment_views = msg.to_segment_views() - segment_view = segment_views[0] - view = memoryview(segment_view) - expected = view.tobytes() - - del msg - del segment_views - del segment_view - gc.collect() - - try: - assert view.tobytes() == expected - finally: - view.release() - - -def test_segment_views_require_root_struct(all_types): - msg = all_types.TestAllTypes.new_message() - nested = msg.init("structField") - - with pytest.raises(capnp.KjException): - nested.to_segment_views() - - @pytest.mark.skipif( sys.version_info[0] < 3, reason="mmap doesn't implement the buffer interface under python 2.", @@ -159,7 +36,7 @@ def test_roundtrip_bytes_mmap(all_types): test_regression.init_all_types(msg) with tempfile.TemporaryFile() as f: - msg.write(f) + f.write(msg.to_bytes()) length = f.tell() f.seek(0) @@ -188,19 +65,6 @@ def test_roundtrip_bytes_fail(all_types): pass -@pytest.mark.skipif( - platform.python_implementation() == "PyPy", - reason="This works in PyPy 4.0.1 but travisci's version of PyPy has some bug that fails this test.", -) -def test_roundtrip_bytes_packed(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - message_bytes = msg.to_bytes_packed() - - msg = all_types.TestAllTypes.from_bytes_packed(message_bytes) - test_regression.check_all_types(msg) - - @contextmanager def _warnings(expected_count=2, expected_text="This message has already been written once."): with warnings.catch_warnings(record=True) as w: @@ -211,23 +75,6 @@ def _warnings(expected_count=2, expected_text="This message has already been wri assert all(expected_text in str(x.message) for x in w), w -def test_roundtrip_file_multiple(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write(f) - with _warnings(2): - msg.write(f) - msg.write(f) - - f.seek(0) - i = 0 - for msg in all_types.TestAllTypes.read_multiple(f): - test_regression.check_all_types(msg) - i += 1 - assert i == 3 - - def test_roundtrip_bytes_multiple(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -244,63 +91,6 @@ def test_roundtrip_bytes_multiple(all_types): assert i == 3 -def test_roundtrip_file_multiple_packed(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write_packed(f) - with _warnings(2): - msg.write_packed(f) - msg.write_packed(f) - - f.seek(0) - i = 0 - for msg in all_types.TestAllTypes.read_multiple_packed(f): - test_regression.check_all_types(msg) - i += 1 - assert i == 3 - - -def test_roundtrip_bytes_multiple_packed(all_types): - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - - msgs = msg.to_bytes_packed() - with _warnings(2): - msgs += msg.to_bytes_packed() - msgs += msg.to_bytes_packed() - - i = 0 - for msg in all_types.TestAllTypes.read_multiple_bytes_packed(msgs): - test_regression.check_all_types(msg) - i += 1 - assert i == 3 - - -def test_file_and_bytes(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write(f) - - f.seek(0) - - with _warnings(1): - assert f.read() == msg.to_bytes() - - -def test_file_and_bytes_packed(all_types): - f = tempfile.TemporaryFile() - msg = all_types.TestAllTypes.new_message() - test_regression.init_all_types(msg) - msg.write_packed(f) - - f.seek(0) - - with _warnings(1): - assert f.read() == msg.to_bytes_packed() - - def test_pickle(all_types): msg = all_types.TestAllTypes.new_message() test_regression.init_all_types(msg) @@ -326,22 +116,6 @@ def test_from_bytes_traversal_limit(all_types): assert msg.structList[i].uInt8Field == 0 -def test_from_bytes_packed_traversal_limit(all_types): - size = 1024 - bld = all_types.TestAllTypes.new_message() - bld.init("structList", size) - data = bld.to_bytes_packed() - - msg = all_types.TestAllTypes.from_bytes_packed(data) - with pytest.raises(capnp.KjException): - for i in range(0, size): - msg.structList[i].uInt8Field == 0 - - msg = all_types.TestAllTypes.from_bytes_packed(data, traversal_limit_in_words=2**62) - for i in range(0, size): - assert msg.structList[i].uInt8Field == 0 - - def test_malformed_text_field_reraise(): SCHEMA = "@0xdbb9ad1f14bf0b36;\nstruct Person { name @0 :Text; age @1 :UInt32; }\n" with tempfile.NamedTemporaryFile(suffix=".capnp", mode="w", delete=False) as f: @@ -353,10 +127,7 @@ def test_malformed_text_field_reraise(): buf = bytearray(Person.new_message(name="alice", age=30).to_bytes()) buf[37] ^= 0xFF - # The process should raise an exception, not SIGSEGV - try: - with Person.from_bytes(bytes(buf), traversal_limit_in_words=2**20) as r: - _ = str(r.name) - except Exception: - # Success: We caught an exception cleanly - pass + # An invalid UTF-8 error description may itself raise UnicodeDecodeError. + with pytest.raises((capnp.KjException, UnicodeDecodeError)): + with Person.from_bytes(bytes(buf), traversal_limit_in_words=2**20) as reader: + _ = reader.name diff --git a/test/test_struct.py b/test/test_struct.py index 3290058..17db1da 100644 --- a/test/test_struct.py +++ b/test/test_struct.py @@ -50,7 +50,7 @@ def test_which_builder(addressbook): def test_which_reader(addressbook): - def writeAddressBook(fd): + def writeAddressBook(file): message = capnp._MallocMessageBuilder() addressBook = message.init_root(addressbook.AddressBook) people = addressBook.init("people", 2) @@ -61,13 +61,14 @@ def test_which_reader(addressbook): bob = people[1] bob.employment.unemployed = None - capnp._write_packed_message_to_fd(fd, message) + file.write(addressBook.to_bytes()) f = tempfile.TemporaryFile() - writeAddressBook(f.fileno()) + writeAddressBook(f) f.seek(0) - addresses = addressbook.AddressBook.read_packed(f) + with addressbook.AddressBook.from_bytes(f.read()) as reader: + addresses = reader people = addresses.people diff --git a/test/test_structs_sequence.capnp b/test/test_structs_sequence.capnp deleted file mode 100644 index 79cb74c..0000000 --- a/test/test_structs_sequence.capnp +++ /dev/null @@ -1,27 +0,0 @@ -@0x9dafb673e5609df6; - - -enum FruitId { - apple @0; - banana @1; - cherry @2; -} - -struct UnknownFruit { - fruitId @0: FruitId; -} - -struct Apple { - fruitId @0: FruitId; - color @1: Text; -} - -struct Banana { - fruitId @0: FruitId; - length @1: Float32; -} - -struct Cherry { - fruitId @0: FruitId; - sweetness @1: UInt8; -} diff --git a/test/test_structs_sequence.py b/test/test_structs_sequence.py deleted file mode 100644 index e2b5ed4..0000000 --- a/test/test_structs_sequence.py +++ /dev/null @@ -1,93 +0,0 @@ -import os - -import pytest - -import capnp - -this_dir = os.path.dirname(__file__) - - -@pytest.fixture -def message_schemas(): - return capnp.load(os.path.join(this_dir, "test_structs_sequence.capnp")) - - -@pytest.fixture -def make_apple(message_schemas): - def _make_apple(color: str): - apple = message_schemas.Apple.new_message() - apple.fruitId = message_schemas.FruitId.apple - apple.color = color - return apple - - return _make_apple - - -@pytest.fixture -def red_apple(make_apple): - return make_apple("Red") - - -@pytest.fixture -def green_apple(make_apple): - return make_apple("Green") - - -@pytest.fixture -def banana(message_schemas): - banana_ = message_schemas.Banana.new_message() - banana_.fruitId = message_schemas.FruitId.banana - banana_.length = 12.345 - return banana_ - - -@pytest.fixture -def cherry(message_schemas): - cherry_ = message_schemas.Cherry.new_message() - cherry_.fruitId = message_schemas.FruitId.cherry - cherry_.sweetness = 64 - return cherry_ - - -@pytest.fixture -def fruit_basket(cherry, red_apple, banana, green_apple): - return [cherry, red_apple, banana, green_apple] - - -@pytest.fixture -def fruit_basket_encoded(fruit_basket): - return b"".join(fruit.to_bytes_packed() for fruit in fruit_basket) - - -@pytest.fixture -def expected(fruit_basket): - return [fruit.to_dict() for fruit in fruit_basket] - - -def test_parse_structs_sequence(message_schemas, fruit_basket_encoded, expected): - # ARRANGE - reader = capnp.read_multiple_bytes_packed(fruit_basket_encoded) - - def _parse_fruit(any_): - unknown_fruit = any_.as_struct(message_schemas.UnknownFruit) - if unknown_fruit.fruitId == message_schemas.FruitId.apple: - return any_.as_struct(message_schemas.Apple) - - if unknown_fruit.fruitId == message_schemas.FruitId.banana: - return any_.as_struct(message_schemas.Banana) - - if unknown_fruit.fruitId == message_schemas.FruitId.cherry: - return any_.as_struct(message_schemas.Cherry) - - return unknown_fruit - - # ACT - parsed = [_parse_fruit(any_).to_dict() for any_ in reader] - - # ASSERT - assert parsed == expected - - -def test_empty_sequence(): - reader = capnp.read_multiple_bytes_packed(b"") - assert len(list(reader)) == 0 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 7a7e0d1..0000000 --- a/tox.ini +++ /dev/null @@ -1,19 +0,0 @@ -[tox] -envlist = py38,py39,py310,py311,py12 -skipsdist = True - -[testenv] -deps= - pkgconfig - Jinja2 - pytest - pytest-asyncio - cython>=3 - -commands = - pip install . - py.test {posargs} - -setenv = - CFLAGS='-stdlib=libc++' - CXXFLAGS='-stdlib=libc++'