add capnp and ffmpeg (#4)

.github/workflows/release.yml

@@ -21,6 +21,9 @@ jobs:
         with:
           python-version: "3.14"
 
+      - name: Install build dependencies
+        run: ./setup.sh
+
       - name: build and release wheels
         env:
           GH_TOKEN: ${{ github.token }}
@@ -73,9 +76,17 @@ jobs:
             version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
             tag="${pkg}/v${version}"
             module="${pkg//-/_}"
+            # detect the package-data directory (e.g. "toolchain" or "install")
+            datadir=$(python3 -c "
+          import tomllib
+          t = tomllib.load(open('$toml', 'rb'))
+          pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0]
+          print(pat.split('/')[0])
+          ")
+            description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])")
 
             echo "========================================="
-            echo "Creating shim: $pkg v$version"
+            echo "Creating shim: $pkg v$version (datadir=$datadir)"
             echo "========================================="
 
             pkg_dir="$tmp/$pkg"
@@ -85,7 +96,6 @@ jobs:
             # copy __init__.py from source
             cp "$pkg/$module/__init__.py" "$mod_dir/"
 
-            # pyproject.toml — same metadata, no package-data (toolchain comes from wheel)
             cat > "$pkg_dir/pyproject.toml" << TOML
           [build-system]
           requires = ["setuptools>=64", "wheel"]
@@ -94,7 +104,7 @@ jobs:
           [project]
           name = "$pkg"
           version = "$version"
-          description = "ARM GCC toolchain for bare-metal targets (pre-built)"
+          description = "$description (pre-built)"
           requires-python = ">=3.8"
 
           [project.scripts]
@@ -104,10 +114,10 @@ jobs:
           include = ["${module}*"]
 
           [tool.setuptools.package-data]
-          $module = ["toolchain/**/*"]
+          $module = ["${datadir}/**/*"]
           TOML
 
-            # setup.py — downloads pre-built wheel from GH releases and extracts toolchain
+            # setup.py — downloads pre-built wheel from GH releases and extracts data
             cat > "$pkg_dir/setup.py" << 'SETUP'
           import os
           import platform
@@ -121,6 +131,7 @@ jobs:
           TAG = "TAG_PLACEHOLDER"
           VERSION = "VERSION_PLACEHOLDER"
           MODULE = "MODULE_PLACEHOLDER"
+          DATADIR = "DATADIR_PLACEHOLDER"
 
           PLATFORM_MAP = {
             ("Linux", "x86_64"): "linux_x86_64",
@@ -130,13 +141,13 @@ jobs:
 
 
           class InstallPrebuilt(build_py):
-            """Download pre-built wheel from GitHub Releases and extract the toolchain."""
+            """Download pre-built wheel from GitHub Releases and extract binaries."""
 
             def run(self):
               pkg_dir = os.path.dirname(os.path.abspath(__file__))
-              toolchain_dir = os.path.join(pkg_dir, MODULE, "toolchain")
+              data_dir = os.path.join(pkg_dir, MODULE, DATADIR)
 
-              if not os.path.exists(os.path.join(toolchain_dir, "bin")):
+              if not os.path.exists(os.path.join(data_dir, "bin")):
                 key = (platform.system(), platform.machine())
                 plat = PLATFORM_MAP.get(key)
                 if plat is None:
@@ -148,25 +159,23 @@ jobs:
                 print(f"Downloading {url} ...")
                 data = urlopen(url).read()
 
-                print("Extracting toolchain ...")
+                print(f"Extracting {DATADIR} ...")
                 with zipfile.ZipFile(BytesIO(data)) as zf:
-                  prefix = f"{MODULE}/toolchain/"
+                  prefix = f"{MODULE}/{DATADIR}/"
-                  # also handle .data/purelib/ layout
+                  alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/"
-                  alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/toolchain/"
                   for info in zf.infolist():
                     for p in (prefix, alt_prefix):
                       if info.filename.startswith(p):
                         rel = info.filename[len(p):]
                         if not rel:
                           continue
-                        dest = os.path.join(toolchain_dir, rel)
+                        dest = os.path.join(data_dir, rel)
                         if info.is_dir():
                           os.makedirs(dest, exist_ok=True)
                         else:
                           os.makedirs(os.path.dirname(dest), exist_ok=True)
                           with open(dest, "wb") as f:
                             f.write(zf.read(info))
-                          # preserve executable bit
                           if info.external_attr >> 16 & 0o111:
                             os.chmod(dest, 0o755)
                         break
@@ -188,6 +197,7 @@ jobs:
             sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py"
             sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py"
             sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py"
+            sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py"
           done
 
           # push to releases branch

.github/workflows/test.yml

@@ -24,6 +24,8 @@ jobs:
       - uses: actions/setup-python@v6
         with:
           python-version: "3.14"
+      - name: Install build dependencies
+        run: ./setup.sh
       - name: Build wheels
         run: |
           for pkg in */pyproject.toml; do

.gitignore

@@ -1,6 +1,11 @@
 .git/
 repo/
 
+# agents
+.claude/
+.context/
+TASK.md
+
 # build artifacts (applies to all packages)
 build/
 dist/
@@ -8,3 +13,9 @@ dist/
 
 # downloaded toolchain binaries
 gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/
+
+# built capnproto
+capnproto/capnproto/install/
+
+# built ffmpeg
+ffmpeg/ffmpeg/install/

capnproto/build.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR"
+
+VERSION="1.0.1"
+INSTALL_DIR="$DIR/capnproto/install"
+
+# Idempotent: skip if already built
+if [ -x "$INSTALL_DIR/bin/capnp" ]; then
+  echo "capnproto already present, skipping build."
+  exit 0
+fi
+
+NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
+
+# Clone
+if [ ! -d "capnproto-src" ]; then
+  git clone --depth 1 --branch "v${VERSION}" https://github.com/capnproto/capnproto.git capnproto-src
+fi
+
+# Build
+PREFIX="$DIR/build/prefix"
+mkdir -p "$DIR/build"
+
+cmake -S capnproto-src -B "$DIR/build" \
+  -DCMAKE_BUILD_TYPE=MinSizeRel \
+  -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
+  -DCMAKE_INSTALL_PREFIX="$PREFIX" \
+  -DCMAKE_INSTALL_LIBDIR=lib \
+  -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
+  -DWITH_OPENSSL=OFF \
+  -DBUILD_TESTING=OFF \
+  -DBUILD_SHARED_LIBS=OFF
+
+cmake --build "$DIR/build" -j"$NJOBS"
+cmake --install "$DIR/build"
+
+# Copy to package install dir
+rm -rf "$INSTALL_DIR"
+mkdir -p "$INSTALL_DIR"/{bin,lib,include}
+
+# Binaries
+cp "$PREFIX/bin/capnp" "$INSTALL_DIR/bin/"
+cp "$PREFIX/bin/capnpc-c++" "$INSTALL_DIR/bin/"
+ln -sf capnp "$INSTALL_DIR/bin/capnpc"
+
+# Libraries (only the ones openpilot needs)
+cp "$PREFIX/lib/libcapnp.a" "$INSTALL_DIR/lib/"
+cp "$PREFIX/lib/libkj.a" "$INSTALL_DIR/lib/"
+
+# Headers
+cp -r "$PREFIX/include/capnp" "$INSTALL_DIR/include/"
+cp -r "$PREFIX/include/kj" "$INSTALL_DIR/include/"
+
+# Strip binaries and libs
+strip "$INSTALL_DIR/bin/capnp" "$INSTALL_DIR/bin/capnpc-c++" 2>/dev/null || true
+
+# Strip unused kj objects from libkj.a (not needed by openpilot)
+for obj in filesystem.c++.o main.c++.o test-helpers.c++.o; do
+  ar d "$INSTALL_DIR/lib/libkj.a" "$obj" 2>/dev/null || true
+done
+
+# Clean up
+rm -rf capnproto-src "$DIR/build"
+
+echo "Installed capnproto to $INSTALL_DIR"
+du -sh "$INSTALL_DIR"

capnproto/capnproto/__init__.py

@@ -0,0 +1,31 @@
+import os
+import sys
+
+DIR = os.path.join(os.path.dirname(__file__), "install")
+BIN_DIR = os.path.join(DIR, "bin")
+LIB_DIR = os.path.join(DIR, "lib")
+INCLUDE_DIR = os.path.join(DIR, "include")
+
+
+def _run(name):
+  binary = os.path.join(BIN_DIR, name)
+  env = os.environ.copy()
+  # ensure sibling binaries (e.g. capnpc-c++) are findable
+  env["PATH"] = BIN_DIR + ":" + env.get("PATH", "")
+  os.execvpe(binary, [binary] + sys.argv[1:], env)
+
+
+def _run_capnp():
+  _run("capnp")
+
+
+def _run_capnpc():
+  # capnpc is a symlink to capnp; capnp checks argv[0] to enter compile mode
+  binary = os.path.join(BIN_DIR, "capnp")
+  env = os.environ.copy()
+  env["PATH"] = BIN_DIR + ":" + env.get("PATH", "")
+  os.execvpe(binary, ["capnpc"] + sys.argv[1:], env)
+
+
+def _run_capnpc_cpp():
+  _run("capnpc-c++")

capnproto/pyproject.toml

@@ -0,0 +1,20 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "capnproto"
+version = "1.0.1"
+description = "Cap'n Proto serialization library and compiler"
+requires-python = ">=3.8"
+
+[project.scripts]
+capnp = "capnproto:_run_capnp"
+capnpc = "capnproto:_run_capnpc"
+"capnpc-c++" = "capnproto:_run_capnpc_cpp"
+
+[tool.setuptools.packages.find]
+include = ["capnproto*"]
+
+[tool.setuptools.package-data]
+capnproto = ["install/**/*"]

capnproto/setup.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import subprocess
+
+from setuptools.command.build_py import build_py
+
+try:
+  from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+  bdist_wheel = None
+
+
+class BuildCapnproto(build_py):
+  """Run build.sh to compile capnproto before collecting package data."""
+
+  def run(self):
+    pkg_dir = os.path.dirname(os.path.abspath(__file__))
+    marker = os.path.join(pkg_dir, "capnproto", "install", "bin", "capnp")
+
+    if not os.path.exists(marker):
+      build_script = os.path.join(pkg_dir, "build.sh")
+      subprocess.check_call(["bash", build_script], cwd=pkg_dir)
+
+    super().run()
+
+
+cmdclass = {"build_py": BuildCapnproto}
+
+if bdist_wheel is not None:
+
+  class PlatformWheel(bdist_wheel):
+    """Produce a platform-specific, Python-version-agnostic wheel."""
+
+    def finalize_options(self):
+      super().finalize_options()
+      self.root_is_pure = False
+
+    def get_tag(self):
+      system = platform.system()
+      machine = platform.machine()
+
+      if system == "Linux":
+        plat = f"linux_{machine}"
+      elif system == "Darwin":
+        plat = "macosx_11_0_arm64"
+      else:
+        plat = f"{system.lower()}_{machine}"
+
+      return "py3", "none", plat
+
+  cmdclass["bdist_wheel"] = PlatformWheel
+
+
+def setup():
+  from setuptools import setup as _setup
+
+  _setup(cmdclass=cmdclass)
+
+
+if __name__ == "__main__":
+  setup()

ffmpeg/build.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR"
+
+FFMPEG_VERSION="7.1"
+INSTALL_DIR="$DIR/ffmpeg/install"
+
+# Idempotent: skip if already built
+if [ -x "$INSTALL_DIR/bin/ffmpeg" ]; then
+  echo "ffmpeg already present, skipping build."
+  exit 0
+fi
+
+NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
+PREFIX="$DIR/build/prefix"
+mkdir -p "$DIR/build"
+
+# --- Build x264 (static) ---
+if [ ! -d "x264-src" ]; then
+  git clone --depth 1 --branch stable https://code.videolan.org/videolan/x264.git x264-src
+fi
+
+cd x264-src
+./configure \
+  --prefix="$PREFIX" \
+  --enable-static \
+  --disable-shared \
+  --disable-cli \
+  --disable-opencl \
+  --enable-pic
+make -j"$NJOBS"
+make install
+cd "$DIR"
+
+# --- Build FFmpeg ---
+if [ ! -d "ffmpeg-src" ]; then
+  git clone --depth 1 --branch "n${FFMPEG_VERSION}" https://github.com/FFmpeg/FFmpeg.git ffmpeg-src
+fi
+
+cd ffmpeg-src
+PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \
+./configure \
+  --prefix="$PREFIX" \
+  --enable-gpl \
+  --enable-static \
+  --disable-shared \
+  --enable-libx264 \
+  --enable-pic \
+  --disable-doc \
+  --disable-ffplay \
+  --disable-autodetect \
+  --disable-everything \
+  --enable-encoder=libx264,aac,ffvhuff,rawvideo,png \
+  --enable-decoder=h264,hevc,ffvhuff,aac,rawvideo,png,mjpeg,mp3,pcm_s16le \
+  --enable-muxer=mpegts,matroska,mp4,hevc,rawvideo,image2,null,mov \
+  --enable-demuxer=hevc,matroska,mpegts,mov,rawvideo,image2,aac,concat \
+  --enable-parser=h264,hevc,aac,mpegaudio \
+  --enable-protocol=file,pipe \
+  --enable-filter=blend,vflip,format,scale,aformat,anull,aresample,null \
+  --enable-bsf=extract_extradata,h264_mp4toannexb,hevc_mp4toannexb \
+  --extra-cflags="-I$PREFIX/include" \
+  --extra-ldflags="-L$PREFIX/lib"
+make -j"$NJOBS"
+make install
+cd "$DIR"
+
+# Copy to package install dir
+rm -rf "$INSTALL_DIR"
+mkdir -p "$INSTALL_DIR"/{bin,lib,include}
+
+# Binaries
+cp "$PREFIX/bin/ffmpeg" "$INSTALL_DIR/bin/"
+cp "$PREFIX/bin/ffprobe" "$INSTALL_DIR/bin/"
+
+# Libraries
+for lib in libavformat.a libavcodec.a libavutil.a libswresample.a libx264.a; do
+  cp "$PREFIX/lib/$lib" "$INSTALL_DIR/lib/"
+done
+
+# Headers
+for dir in libavformat libavcodec libavutil libswresample; do
+  cp -r "$PREFIX/include/$dir" "$INSTALL_DIR/include/"
+done
+
+# Strip binaries
+strip "$INSTALL_DIR/bin/ffmpeg" "$INSTALL_DIR/bin/ffprobe" 2>/dev/null || true
+
+# Clean up
+rm -rf x264-src ffmpeg-src "$DIR/build"
+
+echo "Installed ffmpeg to $INSTALL_DIR"
+du -sh "$INSTALL_DIR"

ffmpeg/ffmpeg/__init__.py

@@ -0,0 +1,20 @@
+import os
+import sys
+
+DIR = os.path.join(os.path.dirname(__file__), "install")
+BIN_DIR = os.path.join(DIR, "bin")
+LIB_DIR = os.path.join(DIR, "lib")
+INCLUDE_DIR = os.path.join(DIR, "include")
+
+
+def _run(name):
+  binary = os.path.join(BIN_DIR, name)
+  os.execvp(binary, [binary] + sys.argv[1:])
+
+
+def _run_ffmpeg():
+  _run("ffmpeg")
+
+
+def _run_ffprobe():
+  _run("ffprobe")

ffmpeg/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ffmpeg"
+version = "7.1.0"
+description = "FFmpeg media tools and libraries (minimal build for openpilot)"
+requires-python = ">=3.8"
+
+[project.scripts]
+ffmpeg = "ffmpeg:_run_ffmpeg"
+ffprobe = "ffmpeg:_run_ffprobe"
+
+[tool.setuptools.packages.find]
+include = ["ffmpeg*"]
+
+[tool.setuptools.package-data]
+ffmpeg = ["install/**/*"]

ffmpeg/setup.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import subprocess
+
+from setuptools.command.build_py import build_py
+
+try:
+  from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+  bdist_wheel = None
+
+
+class BuildFFmpeg(build_py):
+  """Run build.sh to compile ffmpeg before collecting package data."""
+
+  def run(self):
+    pkg_dir = os.path.dirname(os.path.abspath(__file__))
+    marker = os.path.join(pkg_dir, "ffmpeg", "install", "bin", "ffmpeg")
+
+    if not os.path.exists(marker):
+      build_script = os.path.join(pkg_dir, "build.sh")
+      subprocess.check_call(["bash", build_script], cwd=pkg_dir)
+
+    super().run()
+
+
+cmdclass = {"build_py": BuildFFmpeg}
+
+if bdist_wheel is not None:
+
+  class PlatformWheel(bdist_wheel):
+    """Produce a platform-specific, Python-version-agnostic wheel."""
+
+    def finalize_options(self):
+      super().finalize_options()
+      self.root_is_pure = False
+
+    def get_tag(self):
+      system = platform.system()
+      machine = platform.machine()
+
+      if system == "Linux":
+        plat = f"linux_{machine}"
+      elif system == "Darwin":
+        plat = "macosx_11_0_arm64"
+      else:
+        plat = f"{system.lower()}_{machine}"
+
+      return "py3", "none", plat
+
+  cmdclass["bdist_wheel"] = PlatformWheel
+
+
+def setup():
+  from setuptools import setup as _setup
+
+  _setup(cmdclass=cmdclass)
+
+
+if __name__ == "__main__":
+  setup()

setup.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "$(uname)" = "Darwin" ]; then
+  brew install nasm pkg-config
+else
+  sudo apt-get update && sudo apt-get install -y nasm cmake g++ pkg-config
+fi