Compare commits

...

9 Commits

Author SHA1 Message Date
Adeeb Shihadeh
f117a0e60d add cppcheck package (#10)
Cppcheck 2.19.1 static analysis tool, built from source. Includes
the MISRA C addon, cfg files, and platform definitions for opendbc
and panda safety analysis.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-23 10:18:33 -08:00
Adeeb Shihadeh
bf379a6268 add eigen package (#9)
Header-only Eigen 3.4.0 C++ linear algebra library, downloaded from
GitLab release. Exposes INCLUDE_DIR for scons integration.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-22 21:39:17 -08:00
Adeeb Shihadeh
fcd4d49232 better README (#5) 2026-02-22 21:34:30 -08:00
Adeeb Shihadeh
9a30c4f717 update .gitignore 2026-02-22 21:29:04 -08:00
Adeeb Shihadeh
c57efe15bc add git-lfs package (#8) 2026-02-22 21:19:16 -08:00
Adeeb Shihadeh
f67ffb5c94 add zeromq package (#7)
Static build of libzmq 4.3.5 providing headers and library for
C/C++ projects that link against zmq.

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-22 19:45:10 -08:00
Adeeb Shihadeh
cd913d0f9f add capnp and ffmpeg (#4) 2026-02-22 19:06:33 -08:00
Adeeb Shihadeh
e66179c262 build & publish wheels on master pushes (#3) 2026-02-22 17:56:43 -08:00
Adeeb Shihadeh
453518f608 restructure CI into build + smoketest pipeline (#6) 2026-02-22 17:52:42 -08:00
31 changed files with 1278 additions and 39 deletions

212
.github/workflows/release.yml vendored Normal file
View File

@@ -0,0 +1,212 @@
name: release wheels
on:
push:
branches: [master]
jobs:
build:
name: build - ${{ matrix.runs-on }}
runs-on: ${{ matrix.runs-on }}
permissions:
contents: write
strategy:
fail-fast: false
matrix:
runs-on: [ubuntu-latest, ubuntu-24.04-arm, macos-latest]
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Install build dependencies
run: ./setup.sh
- name: build and release wheels
env:
GH_TOKEN: ${{ github.token }}
run: |
pip install build
for toml in */pyproject.toml; do
[ -f "$toml" ] || continue
pkg="$(dirname "$toml")"
version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
tag="${pkg}/v${version}"
echo "========================================="
echo "Building: $pkg v$version"
echo "========================================="
python -m build --wheel "$pkg"
whl=$(ls "$pkg"/dist/*.whl)
echo "Uploading: $whl"
if gh release view "$tag" --repo "$GITHUB_REPOSITORY" &>/dev/null; then
gh release upload "$tag" "$whl" --repo "$GITHUB_REPOSITORY" --clobber
else
gh release create "$tag" "$whl" \
--repo "$GITHUB_REPOSITORY" \
--title "$pkg v$version" \
--notes "Platform wheels for $pkg $version"
fi
done
publish-shims:
name: publish releases branch
needs: build
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v6
- name: build and push shim packages
run: |
REPO_URL="https://github.com/${GITHUB_REPOSITORY}"
# set up a temp dir for the releases branch
tmp="$(mktemp -d)"
trap 'rm -rf "$tmp"' EXIT
for toml in */pyproject.toml; do
[ -f "$toml" ] || continue
pkg="$(dirname "$toml")"
version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
tag="${pkg}/v${version}"
module="${pkg//-/_}"
# detect the package-data directory (e.g. "toolchain" or "install")
datadir=$(python3 -c "
import tomllib
t = tomllib.load(open('$toml', 'rb'))
pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0]
print(pat.split('/')[0])
")
description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])")
echo "========================================="
echo "Creating shim: $pkg v$version (datadir=$datadir)"
echo "========================================="
pkg_dir="$tmp/$pkg"
mod_dir="$pkg_dir/$module"
mkdir -p "$mod_dir"
# copy __init__.py from source
cp "$pkg/$module/__init__.py" "$mod_dir/"
cat > "$pkg_dir/pyproject.toml" << TOML
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "$pkg"
version = "$version"
description = "$description (pre-built)"
requires-python = ">=3.8"
[project.scripts]
$(grep -A100 '^\[project\.scripts\]' "$pkg/pyproject.toml" | tail -n +2 | sed '/^\[/q' | grep -v '^\[')
[tool.setuptools.packages.find]
include = ["${module}*"]
[tool.setuptools.package-data]
$module = ["${datadir}/**/*"]
TOML
# setup.py — downloads pre-built wheel from GH releases and extracts data
cat > "$pkg_dir/setup.py" << 'SETUP'
import os
import platform
import zipfile
from io import BytesIO
from urllib.request import urlopen
from setuptools.command.build_py import build_py
REPO_URL = "REPO_URL_PLACEHOLDER"
TAG = "TAG_PLACEHOLDER"
VERSION = "VERSION_PLACEHOLDER"
MODULE = "MODULE_PLACEHOLDER"
DATADIR = "DATADIR_PLACEHOLDER"
PLATFORM_MAP = {
("Linux", "x86_64"): "linux_x86_64",
("Linux", "aarch64"): "linux_aarch64",
("Darwin", "arm64"): "macosx_11_0_arm64",
}
class InstallPrebuilt(build_py):
"""Download pre-built wheel from GitHub Releases and extract binaries."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
data_dir = os.path.join(pkg_dir, MODULE, DATADIR)
if not os.path.exists(os.path.join(data_dir, "bin")):
key = (platform.system(), platform.machine())
plat = PLATFORM_MAP.get(key)
if plat is None:
raise RuntimeError(f"unsupported platform: {key}")
whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl"
url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}"
print(f"Downloading {url} ...")
data = urlopen(url).read()
print(f"Extracting {DATADIR} ...")
with zipfile.ZipFile(BytesIO(data)) as zf:
prefix = f"{MODULE}/{DATADIR}/"
alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/"
for info in zf.infolist():
for p in (prefix, alt_prefix):
if info.filename.startswith(p):
rel = info.filename[len(p):]
if not rel:
continue
dest = os.path.join(data_dir, rel)
if info.is_dir():
os.makedirs(dest, exist_ok=True)
else:
os.makedirs(os.path.dirname(dest), exist_ok=True)
with open(dest, "wb") as f:
f.write(zf.read(info))
if info.external_attr >> 16 & 0o111:
os.chmod(dest, 0o755)
break
super().run()
def setup():
from setuptools import setup as _setup
_setup(cmdclass={"build_py": InstallPrebuilt})
if __name__ == "__main__":
setup()
SETUP
# template in the actual values
sed -i "s|REPO_URL_PLACEHOLDER|${REPO_URL}|" "$pkg_dir/setup.py"
sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py"
sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py"
sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py"
sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py"
done
# push to releases branch
cd "$tmp"
git init
git checkout -b releases
git add .
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git commit -m "update shim packages"
git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git"
git push -f origin releases

View File

@@ -1,4 +1,4 @@
name: ./test.sh
name: ci
on:
push:
@@ -6,20 +6,47 @@ on:
pull_request:
jobs:
test:
name: ${{ matrix.image || matrix.os }}
build:
name: build (${{ matrix.platform }})
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include:
# native (no docker)
- os: macos-latest
- platform: macos
runs-on: macos-latest
- platform: linux-x86_64
runs-on: ubuntu-latest
- platform: linux-arm64
runs-on: ubuntu-24.04-arm
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.14"
- name: Install build dependencies
run: ./setup.sh
- name: Build wheels
run: |
for pkg in */pyproject.toml; do
pip wheel "./$(dirname "$pkg")" --no-deps --wheel-dir dist/
done
- name: Smoketest locally
run: bash smoketest.sh dist/
- uses: actions/upload-artifact@v4
with:
name: wheels-${{ matrix.platform }}
path: dist/*.whl
# the provided github actions runner images
# have tons of pre-installed packages. the
# goal here is to test across many vanilla setups
test:
name: test (${{ matrix.image }})
needs: build
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false
matrix:
include:
# test on vanilla linux images
- image: debian:bookworm-slim
runs-on: ubuntu-latest
- image: ubuntu:24.04
@@ -28,8 +55,9 @@ jobs:
runs-on: ubuntu-latest
- image: archlinux:latest
runs-on: ubuntu-latest
- image: alpine:3.21 # musl libc
runs-on: ubuntu-latest
# TODO: make musl work
#- image: alpine:3.21
# runs-on: ubuntu-latest
- image: opensuse/tumbleweed:latest
runs-on: ubuntu-latest
- image: ghcr.io/void-linux/void-glibc:latest
@@ -38,36 +66,27 @@ jobs:
runs-on: ubuntu-24.04-arm
steps:
- uses: actions/checkout@v6
# native (macOS)
- if: ${{ !matrix.image }}
uses: actions/setup-python@v6
- uses: actions/download-artifact@v4
with:
python-version: "3.14"
- if: ${{ !matrix.image }}
name: ./test.sh
run: ./test.sh
# docker (vanilla Linux)
- if: ${{ matrix.image }}
name: install python3
name: wheels-${{ contains(matrix.runs-on, 'arm') && 'linux-arm64' || 'linux-x86_64' }}
path: wheels
- name: build docker
run: |
docker build -t test -f - . <<'DOCKERFILE'
docker build -t smoketest -f - . <<'DOCKERFILE'
FROM ${{ matrix.image }}
RUN if command -v apk >/dev/null; then \
apk add --no-cache python3 py3-pip bash curl; \
apk add --no-cache python3 py3-pip bash; \
elif command -v apt-get >/dev/null; then \
apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv curl ca-certificates; \
apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \
elif command -v dnf >/dev/null; then \
dnf install -y python3 python3-pip curl; \
dnf install -y python3 python3-pip; \
elif command -v pacman >/dev/null; then \
pacman -Sy --noconfirm python python-pip curl; \
pacman -Sy --noconfirm python python-pip; \
elif command -v zypper >/dev/null; then \
zypper install -y python3 python3-pip curl; \
zypper install -y python3 python3-pip; \
elif command -v xbps-install >/dev/null; then \
xbps-install -Sy python3 python3-pip bash curl; \
xbps-install -Sy python3 python3-pip bash; \
fi
DOCKERFILE
- if: ${{ matrix.image }}
name: ./test.sh
run: docker run --rm -v "$PWD:/work" -w /work test bash ./test.sh
- name: ./smoketest.sh
run: docker run --rm -v "$PWD:/work" -w /work smoketest bash smoketest.sh wheels/

12
.gitignore vendored
View File

@@ -1,10 +1,22 @@
.git/
repo/
.venv/
__pycache__/
# agents
.claude/
.context/
TASK.md
# build artifacts (applies to all packages)
build/
dist/
install/
*.egg-info/
# downloaded toolchain binaries
gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/
# downloaded git-lfs
git-lfs/git_lfs/bin/

View File

@@ -1,14 +1,24 @@
# dependencies
a central repo for managing and vendoring third party dependencies for all comma projects.
a central repo for managing and [vendoring](https://htmx.org/essays/vendoring/) third party dependencies for all comma projects.
all of our projects are python projects, so each of these gets packaged as a pip project.
since all our projects are Python, we wrap each vendored dependency as a pip package. `git clone` and `uv sync` is all you need.
this vendoring serves a few goals:
- all dependencies are centrally managed here
- we can slim them down to only what we need
- all platforms get the same versions installed
- tighter control of distribution for fast installs (e.g. Ubuntu's `apt-get` is slow)
motivations for this approach
- `apt-get` is slow
- `apt-get` updates its packages on a schedule we don't control
- `apt-get` package versions don't match `brew` versions
- `apt-get` doesn't come with Arch Linux
- `apt-get` doesn't always have the exact package we need
- `apt-get` packages are often bloated
<!--
this critically adds friction to adding dependencies to our project
- `apt-get` installing a package is easy. is it 1MB, 10MB, or 100MB? no idea.
- `apt-get` installing a package is much easier than adding it here. how much do you want it?
-->
`uv`, as opposed to `apt-get`, `brew`, and friends, is fast and already used in our projects.
we target the following platforms:
- Linux x86_64
@@ -16,3 +26,30 @@ we target the following platforms:
- Darwin aarch64 (Apple Silicon)
contributions welcome for other platforms!
## packages
| package | description |
|-------------------|--------------------------------------------------------------------------|
| gcc-arm-none-eabi | builds [panda](https://github.com/commaai/panda) firmware for STM32 MCUs |
| capnproto | message serialization for openpilot |
| ffmpeg | video encode and decode for openpilot |
| git-lfs | for tracking large files in openpilot |
| zeromq | bridging the openpilot IPC between different hosts |
## usage
```python
dependencies = [
"capnproto @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=capnproto",
"ffmpeg @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=ffmpeg",
]
```
## workflow
to add a new package:
* start a new top-level directory as a new package
* `./test.sh` tests the building of all packages
* on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release
* the `releases` branch contains shim packages that allow pointing to a git branch and always getting the appropriate wheel for your platform

69
capnproto/build.sh Normal file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
VERSION="1.0.1"
INSTALL_DIR="$DIR/capnproto/install"
# Idempotent: skip if already built
if [ -x "$INSTALL_DIR/bin/capnp" ]; then
echo "capnproto already present, skipping build."
exit 0
fi
NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
# Clone
if [ ! -d "capnproto-src" ]; then
git clone --depth 1 --branch "v${VERSION}" https://github.com/capnproto/capnproto.git capnproto-src
fi
# Build
PREFIX="$DIR/build/prefix"
mkdir -p "$DIR/build"
cmake -S capnproto-src -B "$DIR/build" \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DWITH_OPENSSL=OFF \
-DBUILD_TESTING=OFF \
-DBUILD_SHARED_LIBS=OFF
cmake --build "$DIR/build" -j"$NJOBS"
cmake --install "$DIR/build"
# Copy to package install dir
rm -rf "$INSTALL_DIR"
mkdir -p "$INSTALL_DIR"/{bin,lib,include}
# Binaries
cp "$PREFIX/bin/capnp" "$INSTALL_DIR/bin/"
cp "$PREFIX/bin/capnpc-c++" "$INSTALL_DIR/bin/"
ln -sf capnp "$INSTALL_DIR/bin/capnpc"
# Libraries (only the ones openpilot needs)
cp "$PREFIX/lib/libcapnp.a" "$INSTALL_DIR/lib/"
cp "$PREFIX/lib/libkj.a" "$INSTALL_DIR/lib/"
# Headers
cp -r "$PREFIX/include/capnp" "$INSTALL_DIR/include/"
cp -r "$PREFIX/include/kj" "$INSTALL_DIR/include/"
# Strip binaries and libs
strip "$INSTALL_DIR/bin/capnp" "$INSTALL_DIR/bin/capnpc-c++" 2>/dev/null || true
# Strip unused kj objects from libkj.a (not needed by openpilot)
for obj in filesystem.c++.o main.c++.o test-helpers.c++.o; do
ar d "$INSTALL_DIR/lib/libkj.a" "$obj" 2>/dev/null || true
done
# Clean up
rm -rf capnproto-src "$DIR/build"
echo "Installed capnproto to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

View File

@@ -0,0 +1,31 @@
import os
import sys
DIR = os.path.join(os.path.dirname(__file__), "install")
BIN_DIR = os.path.join(DIR, "bin")
LIB_DIR = os.path.join(DIR, "lib")
INCLUDE_DIR = os.path.join(DIR, "include")
def _run(name):
binary = os.path.join(BIN_DIR, name)
env = os.environ.copy()
# ensure sibling binaries (e.g. capnpc-c++) are findable
env["PATH"] = BIN_DIR + ":" + env.get("PATH", "")
os.execvpe(binary, [binary] + sys.argv[1:], env)
def _run_capnp():
_run("capnp")
def _run_capnpc():
# capnpc is a symlink to capnp; capnp checks argv[0] to enter compile mode
binary = os.path.join(BIN_DIR, "capnp")
env = os.environ.copy()
env["PATH"] = BIN_DIR + ":" + env.get("PATH", "")
os.execvpe(binary, ["capnpc"] + sys.argv[1:], env)
def _run_capnpc_cpp():
_run("capnpc-c++")

20
capnproto/pyproject.toml Normal file
View File

@@ -0,0 +1,20 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "capnproto"
version = "1.0.1"
description = "Cap'n Proto serialization library and compiler"
requires-python = ">=3.8"
[project.scripts]
capnp = "capnproto:_run_capnp"
capnpc = "capnproto:_run_capnpc"
"capnpc-c++" = "capnproto:_run_capnpc_cpp"
[tool.setuptools.packages.find]
include = ["capnproto*"]
[tool.setuptools.package-data]
capnproto = ["install/**/*"]

61
capnproto/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildCapnproto(build_py):
"""Run build.sh to compile capnproto before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "capnproto", "install", "bin", "capnp")
if not os.path.exists(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildCapnproto}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

42
cppcheck/build.sh Executable file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
VERSION="2.19.1"
INSTALL_DIR="$DIR/cppcheck/install"
# Idempotent: skip if already built
if [ -x "$INSTALL_DIR/cppcheck" ]; then
echo "cppcheck already present, skipping build."
exit 0
fi
NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
# Clone
if [ ! -d "cppcheck-src" ]; then
git clone --depth 1 --branch "$VERSION" https://github.com/danmar/cppcheck.git cppcheck-src
fi
# Build
cd cppcheck-src
make MATCHCOMPILER=yes CXXFLAGS="-O2" -j"$NJOBS"
cd "$DIR"
# Install
rm -rf "$INSTALL_DIR"
mkdir -p "$INSTALL_DIR"
cp cppcheck-src/cppcheck "$INSTALL_DIR/"
cp -r cppcheck-src/addons "$INSTALL_DIR/"
cp -r cppcheck-src/cfg "$INSTALL_DIR/"
cp -r cppcheck-src/platforms "$INSTALL_DIR/"
strip "$INSTALL_DIR/cppcheck" 2>/dev/null || true
# Clean up
rm -rf cppcheck-src
echo "Installed cppcheck to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

View File

@@ -0,0 +1,16 @@
import os
import sys
DIR = os.path.join(os.path.dirname(__file__), "install")
def _run():
binary = os.path.join(DIR, "cppcheck")
os.execvp(binary, ["cppcheck"] + sys.argv[1:])
def smoketest():
import subprocess
binary = os.path.join(DIR, "cppcheck")
result = subprocess.run([binary, "--version"], capture_output=True, text=True, check=True)
print(result.stdout.strip())

18
cppcheck/pyproject.toml Normal file
View File

@@ -0,0 +1,18 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "cppcheck"
version = "2.19.1"
description = "Cppcheck static analysis tool"
requires-python = ">=3.8"
[project.scripts]
cppcheck = "cppcheck:_run"
[tool.setuptools.packages.find]
include = ["cppcheck*"]
[tool.setuptools.package-data]
cppcheck = ["install/**/*"]

61
cppcheck/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildCppcheck(build_py):
"""Run build.sh to compile cppcheck before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "cppcheck", "install", "cppcheck")
if not os.path.exists(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildCppcheck}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

34
eigen/build.sh Executable file
View File

@@ -0,0 +1,34 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
VERSION="3.4.0"
INSTALL_DIR="$DIR/eigen/install"
# Idempotent: skip if already present
if [ -d "$INSTALL_DIR/eigen3/Eigen" ]; then
echo "eigen already present, skipping download."
exit 0
fi
TARBALL="eigen-${VERSION}.tar.gz"
URL="https://gitlab.com/libeigen/eigen/-/archive/${VERSION}/${TARBALL}"
echo "Downloading Eigen ${VERSION} ..."
curl -fSL -o "$TARBALL" "$URL"
echo "Extracting headers ..."
rm -rf "$INSTALL_DIR"
mkdir -p "$INSTALL_DIR/eigen3"
# Extract only the Eigen/ and unsupported/Eigen/ header directories
tar --strip-components=1 -xzf "$TARBALL" -C "$INSTALL_DIR/eigen3" \
"eigen-${VERSION}/Eigen" \
"eigen-${VERSION}/unsupported"
rm -f "$TARBALL"
echo "Installed eigen to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

9
eigen/eigen/__init__.py Normal file
View File

@@ -0,0 +1,9 @@
import os
DIR = os.path.join(os.path.dirname(__file__), "install")
INCLUDE_DIR = DIR
LIB_DIR = DIR # header-only; no libraries
def smoketest():
assert os.path.isdir(os.path.join(DIR, "eigen3", "Eigen")), "Eigen headers not found"

15
eigen/pyproject.toml Normal file
View File

@@ -0,0 +1,15 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "eigen"
version = "3.4.0"
description = "Eigen C++ linear algebra headers"
requires-python = ">=3.8"
[tool.setuptools.packages.find]
include = ["eigen*"]
[tool.setuptools.package-data]
eigen = ["install/**/*"]

61
eigen/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildEigen(build_py):
"""Run build.sh to download Eigen headers before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "eigen", "install", "eigen3", "Eigen")
if not os.path.isdir(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildEigen}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

94
ffmpeg/build.sh Normal file
View File

@@ -0,0 +1,94 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
FFMPEG_VERSION="7.1"
INSTALL_DIR="$DIR/ffmpeg/install"
# Idempotent: skip if already built
if [ -x "$INSTALL_DIR/bin/ffmpeg" ]; then
echo "ffmpeg already present, skipping build."
exit 0
fi
NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
PREFIX="$DIR/build/prefix"
mkdir -p "$DIR/build"
# --- Build x264 (static) ---
if [ ! -d "x264-src" ]; then
git clone --depth 1 --branch stable https://code.videolan.org/videolan/x264.git x264-src
fi
cd x264-src
./configure \
--prefix="$PREFIX" \
--enable-static \
--disable-shared \
--disable-cli \
--disable-opencl \
--enable-pic
make -j"$NJOBS"
make install
cd "$DIR"
# --- Build FFmpeg ---
if [ ! -d "ffmpeg-src" ]; then
git clone --depth 1 --branch "n${FFMPEG_VERSION}" https://github.com/FFmpeg/FFmpeg.git ffmpeg-src
fi
cd ffmpeg-src
PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}" \
./configure \
--prefix="$PREFIX" \
--enable-gpl \
--enable-static \
--disable-shared \
--enable-libx264 \
--enable-pic \
--disable-doc \
--disable-ffplay \
--disable-autodetect \
--disable-everything \
--enable-encoder=libx264,aac,ffvhuff,rawvideo,png \
--enable-decoder=h264,hevc,ffvhuff,aac,rawvideo,png,mjpeg,mp3,pcm_s16le \
--enable-muxer=mpegts,matroska,mp4,hevc,rawvideo,image2,null,mov \
--enable-demuxer=hevc,matroska,mpegts,mov,rawvideo,image2,aac,concat \
--enable-parser=h264,hevc,aac,mpegaudio \
--enable-protocol=file,pipe \
--enable-filter=blend,vflip,format,scale,aformat,anull,aresample,null \
--enable-bsf=extract_extradata,h264_mp4toannexb,hevc_mp4toannexb \
--extra-cflags="-I$PREFIX/include" \
--extra-ldflags="-L$PREFIX/lib"
make -j"$NJOBS"
make install
cd "$DIR"
# Copy to package install dir
rm -rf "$INSTALL_DIR"
mkdir -p "$INSTALL_DIR"/{bin,lib,include}
# Binaries
cp "$PREFIX/bin/ffmpeg" "$INSTALL_DIR/bin/"
cp "$PREFIX/bin/ffprobe" "$INSTALL_DIR/bin/"
# Libraries
for lib in libavformat.a libavcodec.a libavutil.a libswresample.a libx264.a; do
cp "$PREFIX/lib/$lib" "$INSTALL_DIR/lib/"
done
# Headers
for dir in libavformat libavcodec libavutil libswresample; do
cp -r "$PREFIX/include/$dir" "$INSTALL_DIR/include/"
done
# Strip binaries
strip "$INSTALL_DIR/bin/ffmpeg" "$INSTALL_DIR/bin/ffprobe" 2>/dev/null || true
# Clean up
rm -rf x264-src ffmpeg-src "$DIR/build"
echo "Installed ffmpeg to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

20
ffmpeg/ffmpeg/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
import os
import sys
DIR = os.path.join(os.path.dirname(__file__), "install")
BIN_DIR = os.path.join(DIR, "bin")
LIB_DIR = os.path.join(DIR, "lib")
INCLUDE_DIR = os.path.join(DIR, "include")
def _run(name):
binary = os.path.join(BIN_DIR, name)
os.execvp(binary, [binary] + sys.argv[1:])
def _run_ffmpeg():
_run("ffmpeg")
def _run_ffprobe():
_run("ffprobe")

19
ffmpeg/pyproject.toml Normal file
View File

@@ -0,0 +1,19 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "ffmpeg"
version = "7.1.0"
description = "FFmpeg media tools and libraries (minimal build for openpilot)"
requires-python = ">=3.8"
[project.scripts]
ffmpeg = "ffmpeg:_run_ffmpeg"
ffprobe = "ffmpeg:_run_ffprobe"
[tool.setuptools.packages.find]
include = ["ffmpeg*"]
[tool.setuptools.package-data]
ffmpeg = ["install/**/*"]

61
ffmpeg/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildFFmpeg(build_py):
"""Run build.sh to compile ffmpeg before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "ffmpeg", "install", "bin", "ffmpeg")
if not os.path.exists(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildFFmpeg}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

View File

@@ -19,3 +19,9 @@ def _run_objcopy():
def _run_size():
_run("arm-none-eabi-size")
def smoketest():
import subprocess
gcc = os.path.join(TOOLCHAIN_DIR, "bin", "arm-none-eabi-gcc")
subprocess.run([gcc, "--version"], check=True)

57
git-lfs/build.sh Executable file
View File

@@ -0,0 +1,57 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
VERSION="3.6.1"
INSTALL_DIR="$DIR/git_lfs/bin"
# Idempotent: skip if already present
if [ -x "$INSTALL_DIR/git-lfs" ]; then
echo "git-lfs already present, skipping download."
exit 0
fi
OS="$(uname -s)"
ARCH="$(uname -m)"
case "${OS}-${ARCH}" in
Linux-x86_64) PLATFORM="linux-amd64" ; EXT="tar.gz" ;;
Linux-aarch64) PLATFORM="linux-arm64" ; EXT="tar.gz" ;;
Darwin-arm64) PLATFORM="darwin-arm64" ; EXT="zip" ;;
*)
echo "Unsupported platform: ${OS}-${ARCH}" >&2
exit 1
;;
esac
FILENAME="git-lfs-${PLATFORM}-v${VERSION}.${EXT}"
URL="https://github.com/git-lfs/git-lfs/releases/download/v${VERSION}/${FILENAME}"
echo "Downloading $FILENAME ..."
curl -fSL -o "$FILENAME" "$URL"
echo "Extracting ..."
mkdir -p "$INSTALL_DIR"
if [ "$EXT" = "zip" ]; then
python3 -c "
import zipfile, sys
with zipfile.ZipFile('$FILENAME') as zf:
for info in zf.infolist():
if info.filename.endswith('/git-lfs'):
with open('$INSTALL_DIR/git-lfs', 'wb') as f:
f.write(zf.read(info))
break
"
else
tar --strip-components=1 -xzf "$FILENAME" -C "$INSTALL_DIR" --wildcards '*/git-lfs'
fi
chmod +x "$INSTALL_DIR/git-lfs"
strip "$INSTALL_DIR/git-lfs" 2>/dev/null || true
rm -f "$FILENAME"
echo "Installed git-lfs to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

View File

@@ -0,0 +1,15 @@
import os
import sys
BIN_DIR = os.path.join(os.path.dirname(__file__), "bin")
def _run():
binary = os.path.join(BIN_DIR, "git-lfs")
os.execvp(binary, [binary] + sys.argv[1:])
def smoketest():
import subprocess
binary = os.path.join(BIN_DIR, "git-lfs")
subprocess.run([binary, "--version"], check=True)

18
git-lfs/pyproject.toml Normal file
View File

@@ -0,0 +1,18 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "git-lfs"
version = "3.6.1"
description = "Git Large File Storage extension"
requires-python = ">=3.8"
[project.scripts]
git-lfs = "git_lfs:_run"
[tool.setuptools.packages.find]
include = ["git_lfs*"]
[tool.setuptools.package-data]
git_lfs = ["bin/**/*"]

61
git-lfs/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildGitLfs(build_py):
"""Run build.sh to download git-lfs before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "git_lfs", "bin", "git-lfs")
if not os.path.exists(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildGitLfs}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

8
setup.sh Executable file
View File

@@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "$(uname)" = "Darwin" ]; then
brew install nasm pkg-config
else
sudo apt-get update && sudo apt-get install -y nasm cmake g++ pkg-config
fi

18
smoketest.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
WHEEL_DIR="${1:?Usage: smoketest.sh <wheel-directory>}"
WHEEL_DIR="$(cd "$WHEEL_DIR" && pwd)"
VENV_DIR="$(mktemp -d)"
trap 'rm -rf "$VENV_DIR"' EXIT
python3 -m venv "$VENV_DIR"
source "$VENV_DIR/bin/activate"
pip install --upgrade pip >/dev/null
pip install "$WHEEL_DIR"/*.whl
for toml in "$REPO_DIR"/*/pyproject.toml; do
module="$(basename "$(dirname "$toml")" | tr '-' '_')"
python3 -c "import $module; $module.smoketest()" && echo "$module: OK"
done

59
zeromq/build.sh Executable file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
cd "$DIR"
VERSION="4.3.5"
INSTALL_DIR="$DIR/zeromq/install"
# Idempotent: skip if already built
if [ -f "$INSTALL_DIR/lib/libzmq.a" ]; then
echo "zeromq already present, skipping build."
exit 0
fi
NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
# Clone
if [ ! -d "libzmq-src" ]; then
git clone --depth 1 --branch "v${VERSION}" https://github.com/zeromq/libzmq.git libzmq-src
fi
# Build
PREFIX="$DIR/build/prefix"
mkdir -p "$DIR/build"
cmake -S libzmq-src -B "$DIR/build" \
-DCMAKE_BUILD_TYPE=MinSizeRel \
-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DWITH_LIBSODIUM=OFF \
-DWITH_TLS=OFF \
-DWITH_DOCS=OFF \
-DBUILD_TESTS=OFF \
-DBUILD_SHARED=OFF \
-DBUILD_STATIC=ON \
-DENABLE_DRAFTS=OFF
cmake --build "$DIR/build" -j"$NJOBS"
cmake --install "$DIR/build"
# Copy to package install dir
rm -rf "$INSTALL_DIR"
mkdir -p "$INSTALL_DIR"/{lib,include}
# Library
cp "$PREFIX/lib/libzmq.a" "$INSTALL_DIR/lib/"
# Headers
cp "$PREFIX/include/zmq.h" "$INSTALL_DIR/include/"
cp "$PREFIX/include/zmq_utils.h" "$INSTALL_DIR/include/"
# Clean up
rm -rf libzmq-src "$DIR/build"
echo "Installed zeromq to $INSTALL_DIR"
du -sh "$INSTALL_DIR"

15
zeromq/pyproject.toml Normal file
View File

@@ -0,0 +1,15 @@
[build-system]
requires = ["setuptools>=64", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "zeromq"
version = "4.3.5"
description = "ZeroMQ messaging library (static build)"
requires-python = ">=3.8"
[tool.setuptools.packages.find]
include = ["zeromq*"]
[tool.setuptools.package-data]
zeromq = ["install/**/*"]

61
zeromq/setup.py Normal file
View File

@@ -0,0 +1,61 @@
import os
import platform
import subprocess
from setuptools.command.build_py import build_py
try:
from wheel.bdist_wheel import bdist_wheel
except ImportError:
bdist_wheel = None
class BuildZeromq(build_py):
"""Run build.sh to compile zeromq before collecting package data."""
def run(self):
pkg_dir = os.path.dirname(os.path.abspath(__file__))
marker = os.path.join(pkg_dir, "zeromq", "install", "lib", "libzmq.a")
if not os.path.exists(marker):
build_script = os.path.join(pkg_dir, "build.sh")
subprocess.check_call(["bash", build_script], cwd=pkg_dir)
super().run()
cmdclass = {"build_py": BuildZeromq}
if bdist_wheel is not None:
class PlatformWheel(bdist_wheel):
"""Produce a platform-specific, Python-version-agnostic wheel."""
def finalize_options(self):
super().finalize_options()
self.root_is_pure = False
def get_tag(self):
system = platform.system()
machine = platform.machine()
if system == "Linux":
plat = f"linux_{machine}"
elif system == "Darwin":
plat = "macosx_11_0_arm64"
else:
plat = f"{system.lower()}_{machine}"
return "py3", "none", plat
cmdclass["bdist_wheel"] = PlatformWheel
def setup():
from setuptools import setup as _setup
_setup(cmdclass=cmdclass)
if __name__ == "__main__":
setup()

10
zeromq/zeromq/__init__.py Normal file
View File

@@ -0,0 +1,10 @@
import os
DIR = os.path.join(os.path.dirname(__file__), "install")
LIB_DIR = os.path.join(DIR, "lib")
INCLUDE_DIR = os.path.join(DIR, "include")
def smoketest():
assert os.path.isfile(os.path.join(LIB_DIR, "libzmq.a")), "libzmq.a not found"
assert os.path.isfile(os.path.join(INCLUDE_DIR, "zmq.h")), "zmq.h not found"