diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb3856c..da534b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ jobs: with: python-version: "3.14" + - name: Install build dependencies + run: ./setup.sh + - name: build and release wheels env: GH_TOKEN: ${{ github.token }} @@ -73,9 +76,17 @@ jobs: version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])") tag="${pkg}/v${version}" module="${pkg//-/_}" + # detect the package-data directory (e.g. "toolchain" or "install") + datadir=$(python3 -c " + import tomllib + t = tomllib.load(open('$toml', 'rb')) + pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0] + print(pat.split('/')[0]) + ") + description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])") echo "=========================================" - echo "Creating shim: $pkg v$version" + echo "Creating shim: $pkg v$version (datadir=$datadir)" echo "=========================================" pkg_dir="$tmp/$pkg" @@ -85,7 +96,6 @@ jobs: # copy __init__.py from source cp "$pkg/$module/__init__.py" "$mod_dir/" - # pyproject.toml — same metadata, no package-data (toolchain comes from wheel) cat > "$pkg_dir/pyproject.toml" << TOML [build-system] requires = ["setuptools>=64", "wheel"] @@ -94,7 +104,7 @@ jobs: [project] name = "$pkg" version = "$version" - description = "ARM GCC toolchain for bare-metal targets (pre-built)" + description = "$description (pre-built)" requires-python = ">=3.8" [project.scripts] @@ -104,10 +114,10 @@ jobs: include = ["${module}*"] [tool.setuptools.package-data] - $module = ["toolchain/**/*"] + $module = ["${datadir}/**/*"] TOML - # setup.py — downloads pre-built wheel from GH releases and extracts toolchain + # setup.py — downloads pre-built wheel from GH releases and extracts data cat > "$pkg_dir/setup.py" << 'SETUP' import os import platform @@ -121,6 +131,7 @@ jobs: TAG = "TAG_PLACEHOLDER" VERSION = "VERSION_PLACEHOLDER" MODULE = "MODULE_PLACEHOLDER" + DATADIR = "DATADIR_PLACEHOLDER" PLATFORM_MAP = { ("Linux", "x86_64"): "linux_x86_64", @@ -130,13 +141,13 @@ jobs: class InstallPrebuilt(build_py): - """Download pre-built wheel from GitHub Releases and extract the toolchain.""" + """Download pre-built wheel from GitHub Releases and extract binaries.""" def run(self): pkg_dir = os.path.dirname(os.path.abspath(__file__)) - toolchain_dir = os.path.join(pkg_dir, MODULE, "toolchain") + data_dir = os.path.join(pkg_dir, MODULE, DATADIR) - if not os.path.exists(os.path.join(toolchain_dir, "bin")): + if not os.path.exists(os.path.join(data_dir, "bin")): key = (platform.system(), platform.machine()) plat = PLATFORM_MAP.get(key) if plat is None: @@ -148,25 +159,23 @@ jobs: print(f"Downloading {url} ...") data = urlopen(url).read() - print("Extracting toolchain ...") + print(f"Extracting {DATADIR} ...") with zipfile.ZipFile(BytesIO(data)) as zf: - prefix = f"{MODULE}/toolchain/" - # also handle .data/purelib/ layout - alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/toolchain/" + prefix = f"{MODULE}/{DATADIR}/" + alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/" for info in zf.infolist(): for p in (prefix, alt_prefix): if info.filename.startswith(p): rel = info.filename[len(p):] if not rel: continue - dest = os.path.join(toolchain_dir, rel) + dest = os.path.join(data_dir, rel) if info.is_dir(): os.makedirs(dest, exist_ok=True) else: os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, "wb") as f: f.write(zf.read(info)) - # preserve executable bit if info.external_attr >> 16 & 0o111: os.chmod(dest, 0o755) break @@ -188,6 +197,7 @@ jobs: sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py" sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py" sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py" + sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py" done # push to releases branch diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2581fe..402d7b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,8 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.14" + - name: Install build dependencies + run: ./setup.sh - name: Build wheels run: | for pkg in */pyproject.toml; do diff --git a/.gitignore b/.gitignore index 120de9a..89ff3da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ .git/ repo/ +# agents +.claude/ +.context/ +TASK.md + # build artifacts (applies to all packages) build/ dist/ @@ -8,3 +13,9 @@ dist/ # downloaded toolchain binaries gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/ + +# built capnproto +capnproto/capnproto/install/ + +# built ffmpeg +ffmpeg/ffmpeg/install/ diff --git a/capnproto/build.sh b/capnproto/build.sh new file mode 100644 index 0000000..d9ebc40 --- /dev/null +++ b/capnproto/build.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$DIR" + +VERSION="1.0.1" +INSTALL_DIR="$DIR/capnproto/install" + +# Idempotent: skip if already built +if [ -x "$INSTALL_DIR/bin/capnp" ]; then + echo "capnproto already present, skipping build." + exit 0 +fi + +NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" + +# Clone +if [ ! -d "capnproto-src" ]; then + git clone --depth 1 --branch "v${VERSION}" https://github.com/capnproto/capnproto.git capnproto-src +fi + +# Build +PREFIX="$DIR/build/prefix" +mkdir -p "$DIR/build" + +cmake -S capnproto-src -B "$DIR/build" \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \ + -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DWITH_OPENSSL=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=OFF + +cmake --build "$DIR/build" -j"$NJOBS" +cmake --install "$DIR/build" + +# Copy to package install dir +rm -rf "$INSTALL_DIR" +mkdir -p "$INSTALL_DIR"/{bin,lib,include} + +# Binaries +cp "$PREFIX/bin/capnp" "$INSTALL_DIR/bin/" +cp "$PREFIX/bin/capnpc-c++" "$INSTALL_DIR/bin/" +ln -sf capnp "$INSTALL_DIR/bin/capnpc" + +# Libraries (only the ones openpilot needs) +cp "$PREFIX/lib/libcapnp.a" "$INSTALL_DIR/lib/" +cp "$PREFIX/lib/libkj.a" "$INSTALL_DIR/lib/" + +# Headers +cp -r "$PREFIX/include/capnp" "$INSTALL_DIR/include/" +cp -r "$PREFIX/include/kj" "$INSTALL_DIR/include/" + +# Strip binaries and libs +strip "$INSTALL_DIR/bin/capnp" "$INSTALL_DIR/bin/capnpc-c++" 2>/dev/null || true + +# Strip unused kj objects from libkj.a (not needed by openpilot) +for obj in filesystem.c++.o main.c++.o test-helpers.c++.o; do + ar d "$INSTALL_DIR/lib/libkj.a" "$obj" 2>/dev/null || true +done + +# Clean up +rm -rf capnproto-src "$DIR/build" + +echo "Installed capnproto to $INSTALL_DIR" +du -sh "$INSTALL_DIR" diff --git a/capnproto/capnproto/__init__.py b/capnproto/capnproto/__init__.py new file mode 100644 index 0000000..197deb9 --- /dev/null +++ b/capnproto/capnproto/__init__.py @@ -0,0 +1,31 @@ +import os +import sys + +DIR = os.path.join(os.path.dirname(__file__), "install") +BIN_DIR = os.path.join(DIR, "bin") +LIB_DIR = os.path.join(DIR, "lib") +INCLUDE_DIR = os.path.join(DIR, "include") + + +def _run(name): + binary = os.path.join(BIN_DIR, name) + env = os.environ.copy() + # ensure sibling binaries (e.g. capnpc-c++) are findable + env["PATH"] = BIN_DIR + ":" + env.get("PATH", "") + os.execvpe(binary, [binary] + sys.argv[1:], env) + + +def _run_capnp(): + _run("capnp") + + +def _run_capnpc(): + # capnpc is a symlink to capnp; capnp checks argv[0] to enter compile mode + binary = os.path.join(BIN_DIR, "capnp") + env = os.environ.copy() + env["PATH"] = BIN_DIR + ":" + env.get("PATH", "") + os.execvpe(binary, ["capnpc"] + sys.argv[1:], env) + + +def _run_capnpc_cpp(): + _run("capnpc-c++") diff --git a/capnproto/pyproject.toml b/capnproto/pyproject.toml new file mode 100644 index 0000000..3bea931 --- /dev/null +++ b/capnproto/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "capnproto" +version = "1.0.1" +description = "Cap'n Proto serialization library and compiler" +requires-python = ">=3.8" + +[project.scripts] +capnp = "capnproto:_run_capnp" +capnpc = "capnproto:_run_capnpc" +"capnpc-c++" = "capnproto:_run_capnpc_cpp" + +[tool.setuptools.packages.find] +include = ["capnproto*"] + +[tool.setuptools.package-data] +capnproto = ["install/**/*"] diff --git a/capnproto/setup.py b/capnproto/setup.py new file mode 100644 index 0000000..a2354d0 --- /dev/null +++ b/capnproto/setup.py @@ -0,0 +1,61 @@ +import os +import platform +import subprocess + +from setuptools.command.build_py import build_py + +try: + from wheel.bdist_wheel import bdist_wheel +except ImportError: + bdist_wheel = None + + +class BuildCapnproto(build_py): + """Run build.sh to compile capnproto before collecting package data.""" + + def run(self): + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + marker = os.path.join(pkg_dir, "capnproto", "install", "bin", "capnp") + + if not os.path.exists(marker): + build_script = os.path.join(pkg_dir, "build.sh") + subprocess.check_call(["bash", build_script], cwd=pkg_dir) + + super().run() + + +cmdclass = {"build_py": BuildCapnproto} + +if bdist_wheel is not None: + + class PlatformWheel(bdist_wheel): + """Produce a platform-specific, Python-version-agnostic wheel.""" + + def finalize_options(self): + super().finalize_options() + self.root_is_pure = False + + def get_tag(self): + system = platform.system() + machine = platform.machine() + + if system == "Linux": + plat = f"linux_{machine}" + elif system == "Darwin": + plat = "macosx_11_0_arm64" + else: + plat = f"{system.lower()}_{machine}" + + return "py3", "none", plat + + cmdclass["bdist_wheel"] = PlatformWheel + + +def setup(): + from setuptools import setup as _setup + + _setup(cmdclass=cmdclass) + + +if __name__ == "__main__": + setup() diff --git a/ffmpeg/build.sh b/ffmpeg/build.sh new file mode 100644 index 0000000..9f6d0da --- /dev/null +++ b/ffmpeg/build.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$DIR" + +FFMPEG_VERSION="7.1" +INSTALL_DIR="$DIR/ffmpeg/install" + +# Idempotent: skip if already built +if [ -x "$INSTALL_DIR/bin/ffmpeg" ]; then + echo "ffmpeg already present, skipping build." + exit 0 +fi + +NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" +PREFIX="$DIR/build/prefix" +mkdir -p "$DIR/build" + +# --- Build x264 (static) --- +if [ ! -d "x264-src" ]; then + git clone --depth 1 --branch stable https://code.videolan.org/videolan/x264.git x264-src +fi + +cd x264-src +./configure \ + --prefix="$PREFIX" \ + --enable-static \ + --disable-shared \ + --disable-cli \ + --disable-opencl \ + --enable-pic +make -j"$NJOBS" +make install +cd "$DIR" + +# --- Build FFmpeg --- +if [ ! -d "ffmpeg-src" ]; then + git clone --depth 1 --branch "n${FFMPEG_VERSION}" https://github.com/FFmpeg/FFmpeg.git ffmpeg-src +fi + +cd ffmpeg-src +PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \ +./configure \ + --prefix="$PREFIX" \ + --enable-gpl \ + --enable-static \ + --disable-shared \ + --enable-libx264 \ + --enable-pic \ + --disable-doc \ + --disable-ffplay \ + --disable-autodetect \ + --disable-everything \ + --enable-encoder=libx264,aac,ffvhuff,rawvideo,png \ + --enable-decoder=h264,hevc,ffvhuff,aac,rawvideo,png,mjpeg,mp3,pcm_s16le \ + --enable-muxer=mpegts,matroska,mp4,hevc,rawvideo,image2,null,mov \ + --enable-demuxer=hevc,matroska,mpegts,mov,rawvideo,image2,aac,concat \ + --enable-parser=h264,hevc,aac,mpegaudio \ + --enable-protocol=file,pipe \ + --enable-filter=blend,vflip,format,scale,aformat,anull,aresample,null \ + --enable-bsf=extract_extradata,h264_mp4toannexb,hevc_mp4toannexb \ + --extra-cflags="-I$PREFIX/include" \ + --extra-ldflags="-L$PREFIX/lib" +make -j"$NJOBS" +make install +cd "$DIR" + +# Copy to package install dir +rm -rf "$INSTALL_DIR" +mkdir -p "$INSTALL_DIR"/{bin,lib,include} + +# Binaries +cp "$PREFIX/bin/ffmpeg" "$INSTALL_DIR/bin/" +cp "$PREFIX/bin/ffprobe" "$INSTALL_DIR/bin/" + +# Libraries +for lib in libavformat.a libavcodec.a libavutil.a libswresample.a libx264.a; do + cp "$PREFIX/lib/$lib" "$INSTALL_DIR/lib/" +done + +# Headers +for dir in libavformat libavcodec libavutil libswresample; do + cp -r "$PREFIX/include/$dir" "$INSTALL_DIR/include/" +done + +# Strip binaries +strip "$INSTALL_DIR/bin/ffmpeg" "$INSTALL_DIR/bin/ffprobe" 2>/dev/null || true + +# Clean up +rm -rf x264-src ffmpeg-src "$DIR/build" + +echo "Installed ffmpeg to $INSTALL_DIR" +du -sh "$INSTALL_DIR" diff --git a/ffmpeg/ffmpeg/__init__.py b/ffmpeg/ffmpeg/__init__.py new file mode 100644 index 0000000..dc1a82d --- /dev/null +++ b/ffmpeg/ffmpeg/__init__.py @@ -0,0 +1,20 @@ +import os +import sys + +DIR = os.path.join(os.path.dirname(__file__), "install") +BIN_DIR = os.path.join(DIR, "bin") +LIB_DIR = os.path.join(DIR, "lib") +INCLUDE_DIR = os.path.join(DIR, "include") + + +def _run(name): + binary = os.path.join(BIN_DIR, name) + os.execvp(binary, [binary] + sys.argv[1:]) + + +def _run_ffmpeg(): + _run("ffmpeg") + + +def _run_ffprobe(): + _run("ffprobe") diff --git a/ffmpeg/pyproject.toml b/ffmpeg/pyproject.toml new file mode 100644 index 0000000..5ac7fa2 --- /dev/null +++ b/ffmpeg/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "ffmpeg" +version = "7.1.0" +description = "FFmpeg media tools and libraries (minimal build for openpilot)" +requires-python = ">=3.8" + +[project.scripts] +ffmpeg = "ffmpeg:_run_ffmpeg" +ffprobe = "ffmpeg:_run_ffprobe" + +[tool.setuptools.packages.find] +include = ["ffmpeg*"] + +[tool.setuptools.package-data] +ffmpeg = ["install/**/*"] diff --git a/ffmpeg/setup.py b/ffmpeg/setup.py new file mode 100644 index 0000000..19b8e97 --- /dev/null +++ b/ffmpeg/setup.py @@ -0,0 +1,61 @@ +import os +import platform +import subprocess + +from setuptools.command.build_py import build_py + +try: + from wheel.bdist_wheel import bdist_wheel +except ImportError: + bdist_wheel = None + + +class BuildFFmpeg(build_py): + """Run build.sh to compile ffmpeg before collecting package data.""" + + def run(self): + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + marker = os.path.join(pkg_dir, "ffmpeg", "install", "bin", "ffmpeg") + + if not os.path.exists(marker): + build_script = os.path.join(pkg_dir, "build.sh") + subprocess.check_call(["bash", build_script], cwd=pkg_dir) + + super().run() + + +cmdclass = {"build_py": BuildFFmpeg} + +if bdist_wheel is not None: + + class PlatformWheel(bdist_wheel): + """Produce a platform-specific, Python-version-agnostic wheel.""" + + def finalize_options(self): + super().finalize_options() + self.root_is_pure = False + + def get_tag(self): + system = platform.system() + machine = platform.machine() + + if system == "Linux": + plat = f"linux_{machine}" + elif system == "Darwin": + plat = "macosx_11_0_arm64" + else: + plat = f"{system.lower()}_{machine}" + + return "py3", "none", plat + + cmdclass["bdist_wheel"] = PlatformWheel + + +def setup(): + from setuptools import setup as _setup + + _setup(cmdclass=cmdclass) + + +if __name__ == "__main__": + setup() diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..999cc07 --- /dev/null +++ b/setup.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$(uname)" = "Darwin" ]; then + brew install nasm pkg-config +else + sudo apt-get update && sudo apt-get install -y nasm cmake g++ pkg-config +fi