add python3-dev package (#13)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

.gitignore

@@ -1,5 +1,7 @@
 .git/
 repo/
+.venv/
+__pycache__/
 
 # agents
 .claude/
@@ -9,16 +11,12 @@ TASK.md
 # build artifacts (applies to all packages)
 build/
 dist/
+install/
 *.egg-info/
 
 # downloaded toolchain binaries
 gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/
 
-# built capnproto
-capnproto/capnproto/install/
-
-# built ffmpeg
-ffmpeg/ffmpeg/install/
 
 # downloaded git-lfs
 git-lfs/git_lfs/bin/

README.md

@@ -1,14 +1,24 @@
 # dependencies
 
-a central repo for managing and vendoring third party dependencies for all comma projects.
+a central repo for managing and [vendoring](https://htmx.org/essays/vendoring/) third party dependencies for all comma projects.
 
-all of our projects are python projects, so each of these gets packaged as a pip project.
+since all our projects are Python, we wrap each vendored dependency as a pip package. `git clone` and `uv sync` is all you need.
 
-this vendoring serves a few goals:
-- all dependencies are centrally managed here
-- we can slim them down to only what we need
-- all platforms get the same versions installed
-- tighter control of distribution for fast installs (e.g. Ubuntu's `apt-get` is slow)
+motivations for this approach
+- `apt-get` is slow
+- `apt-get` updates its packages on a schedule we don't control
+- `apt-get` package versions don't match `brew` versions
+- `apt-get` doesn't come with Arch Linux
+- `apt-get` doesn't always have the exact package we need
+- `apt-get` packages are often bloated
+
+<!--
+this critically adds friction to adding dependencies to our project
+- `apt-get` installing a package is easy. is it 1MB, 10MB, or 100MB? no idea.
+- `apt-get` installing a package is much easier than adding it here. how much do you want it?
+-->
+
+`uv`, as opposed to `apt-get`, `brew`, and friends, is fast and already used in our projects.
 
 we target the following platforms:
 - Linux x86_64
@@ -16,3 +26,30 @@ we target the following platforms:
 - Darwin aarch64 (Apple Silicon)
 
 contributions welcome for other platforms!
+
+## packages
+
+| package           | description                                                              |
+|-------------------|--------------------------------------------------------------------------|
+| gcc-arm-none-eabi | builds [panda](https://github.com/commaai/panda) firmware for STM32 MCUs |
+| capnproto         | message serialization for openpilot                                      |
+| ffmpeg            | video encode and decode for openpilot                                    |
+| git-lfs           | for tracking large files in openpilot                                    |
+| zeromq            | bridging the openpilot IPC between different hosts                       |
+
+## usage
+
+```python
+dependencies = [
+  "capnproto @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=capnproto",
+  "ffmpeg @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=ffmpeg",
+]
+```
+
+## workflow
+
+to add a new package:
+* start a new top-level directory as a new package
+* `./test.sh` tests the building of all packages
+*  on pushes to `master`, wheels are built for our target platforms and pushed to a GitHub release
+*  the `releases` branch contains shim packages that allow pointing to a git branch and always getting the appropriate wheel for your platform

cppcheck/build.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR"
+
+VERSION="2.16.0"
+INSTALL_DIR="$DIR/cppcheck/install"
+
+# Idempotent: skip if already built
+if [ -x "$INSTALL_DIR/cppcheck" ]; then
+  echo "cppcheck already present, skipping build."
+  exit 0
+fi
+
+NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
+
+# Clone
+if [ ! -d "cppcheck-src" ]; then
+  git clone --depth 1 --branch "$VERSION" https://github.com/danmar/cppcheck.git cppcheck-src
+fi
+
+# Build
+cd cppcheck-src
+make MATCHCOMPILER=yes CXXFLAGS="-O2" -j"$NJOBS"
+cd "$DIR"
+
+# Install
+rm -rf "$INSTALL_DIR"
+mkdir -p "$INSTALL_DIR"
+
+cp cppcheck-src/cppcheck "$INSTALL_DIR/"
+cp -r cppcheck-src/addons "$INSTALL_DIR/"
+cp -r cppcheck-src/cfg "$INSTALL_DIR/"
+cp -r cppcheck-src/platforms "$INSTALL_DIR/"
+strip "$INSTALL_DIR/cppcheck" 2>/dev/null || true
+
+# Clean up
+rm -rf cppcheck-src
+
+echo "Installed cppcheck to $INSTALL_DIR"
+du -sh "$INSTALL_DIR"

cppcheck/cppcheck/__init__.py

@@ -0,0 +1,16 @@
+import os
+import sys
+
+DIR = os.path.join(os.path.dirname(__file__), "install")
+
+
+def _run():
+  binary = os.path.join(DIR, "cppcheck")
+  os.execvp(binary, ["cppcheck"] + sys.argv[1:])
+
+
+def smoketest():
+  import subprocess
+  binary = os.path.join(DIR, "cppcheck")
+  result = subprocess.run([binary, "--version"], capture_output=True, text=True, check=True)
+  print(result.stdout.strip())

cppcheck/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "cppcheck"
+version = "2.16.0"
+description = "Cppcheck static analysis tool"
+requires-python = ">=3.8"
+
+[project.scripts]
+cppcheck = "cppcheck:_run"
+
+[tool.setuptools.packages.find]
+include = ["cppcheck*"]
+
+[tool.setuptools.package-data]
+cppcheck = ["install/**/*"]

cppcheck/setup.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import subprocess
+
+from setuptools.command.build_py import build_py
+
+try:
+  from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+  bdist_wheel = None
+
+
+class BuildCppcheck(build_py):
+  """Run build.sh to compile cppcheck before collecting package data."""
+
+  def run(self):
+    pkg_dir = os.path.dirname(os.path.abspath(__file__))
+    marker = os.path.join(pkg_dir, "cppcheck", "install", "cppcheck")
+
+    if not os.path.exists(marker):
+      build_script = os.path.join(pkg_dir, "build.sh")
+      subprocess.check_call(["bash", build_script], cwd=pkg_dir)
+
+    super().run()
+
+
+cmdclass = {"build_py": BuildCppcheck}
+
+if bdist_wheel is not None:
+
+  class PlatformWheel(bdist_wheel):
+    """Produce a platform-specific, Python-version-agnostic wheel."""
+
+    def finalize_options(self):
+      super().finalize_options()
+      self.root_is_pure = False
+
+    def get_tag(self):
+      system = platform.system()
+      machine = platform.machine()
+
+      if system == "Linux":
+        plat = f"linux_{machine}"
+      elif system == "Darwin":
+        plat = "macosx_11_0_arm64"
+      else:
+        plat = f"{system.lower()}_{machine}"
+
+      return "py3", "none", plat
+
+  cmdclass["bdist_wheel"] = PlatformWheel
+
+
+def setup():
+  from setuptools import setup as _setup
+
+  _setup(cmdclass=cmdclass)
+
+
+if __name__ == "__main__":
+  setup()

eigen/build.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR"
+
+VERSION="3.4.0"
+INSTALL_DIR="$DIR/eigen/install"
+
+# Idempotent: skip if already present
+if [ -d "$INSTALL_DIR/eigen3/Eigen" ]; then
+  echo "eigen already present, skipping download."
+  exit 0
+fi
+
+TARBALL="eigen-${VERSION}.tar.gz"
+URL="https://gitlab.com/libeigen/eigen/-/archive/${VERSION}/${TARBALL}"
+
+echo "Downloading Eigen ${VERSION} ..."
+curl -fSL -o "$TARBALL" "$URL"
+
+echo "Extracting headers ..."
+rm -rf "$INSTALL_DIR"
+mkdir -p "$INSTALL_DIR/eigen3"
+
+# Extract only the Eigen/ and unsupported/Eigen/ header directories
+tar --strip-components=1 -xzf "$TARBALL" -C "$INSTALL_DIR/eigen3" \
+  "eigen-${VERSION}/Eigen" \
+  "eigen-${VERSION}/unsupported"
+
+rm -f "$TARBALL"
+
+echo "Installed eigen to $INSTALL_DIR"
+du -sh "$INSTALL_DIR"

eigen/eigen/__init__.py

@@ -0,0 +1,9 @@
+import os
+
+DIR = os.path.join(os.path.dirname(__file__), "install")
+INCLUDE_DIR = DIR
+LIB_DIR = DIR  # header-only; no libraries
+
+
+def smoketest():
+  assert os.path.isdir(os.path.join(DIR, "eigen3", "Eigen")), "Eigen headers not found"

eigen/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "eigen"
+version = "3.4.0"
+description = "Eigen C++ linear algebra headers"
+requires-python = ">=3.8"
+
+[tool.setuptools.packages.find]
+include = ["eigen*"]
+
+[tool.setuptools.package-data]
+eigen = ["install/**/*"]

eigen/setup.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import subprocess
+
+from setuptools.command.build_py import build_py
+
+try:
+  from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+  bdist_wheel = None
+
+
+class BuildEigen(build_py):
+  """Run build.sh to download Eigen headers before collecting package data."""
+
+  def run(self):
+    pkg_dir = os.path.dirname(os.path.abspath(__file__))
+    marker = os.path.join(pkg_dir, "eigen", "install", "eigen3", "Eigen")
+
+    if not os.path.isdir(marker):
+      build_script = os.path.join(pkg_dir, "build.sh")
+      subprocess.check_call(["bash", build_script], cwd=pkg_dir)
+
+    super().run()
+
+
+cmdclass = {"build_py": BuildEigen}
+
+if bdist_wheel is not None:
+
+  class PlatformWheel(bdist_wheel):
+    """Produce a platform-specific, Python-version-agnostic wheel."""
+
+    def finalize_options(self):
+      super().finalize_options()
+      self.root_is_pure = False
+
+    def get_tag(self):
+      system = platform.system()
+      machine = platform.machine()
+
+      if system == "Linux":
+        plat = f"linux_{machine}"
+      elif system == "Darwin":
+        plat = "macosx_11_0_arm64"
+      else:
+        plat = f"{system.lower()}_{machine}"
+
+      return "py3", "none", plat
+
+  cmdclass["bdist_wheel"] = PlatformWheel
+
+
+def setup():
+  from setuptools import setup as _setup
+
+  _setup(cmdclass=cmdclass)
+
+
+if __name__ == "__main__":
+  setup()

python3-dev/build.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$DIR"
+
+VERSION="3.12.8"
+INSTALL_DIR="$DIR/python3_dev/install"
+
+# Idempotent: skip if already present
+if [ -f "$INSTALL_DIR/include/Python.h" ]; then
+  echo "python3-dev headers already present, skipping."
+  exit 0
+fi
+
+TARBALL="Python-${VERSION}.tgz"
+URL="https://www.python.org/ftp/python/${VERSION}/${TARBALL}"
+
+echo "Downloading CPython ${VERSION} source ..."
+curl -fSL -o "$TARBALL" "$URL"
+
+echo "Extracting ..."
+tar xzf "$TARBALL"
+
+cd "Python-${VERSION}"
+
+# Run configure to generate pyconfig.h for this platform
+echo "Running configure to generate pyconfig.h ..."
+./configure --disable-shared --without-ensurepip > /dev/null 2>&1
+
+cd "$DIR"
+
+# Copy headers
+rm -rf "$INSTALL_DIR"
+mkdir -p "$INSTALL_DIR/include"
+
+cp -r "Python-${VERSION}/Include/"* "$INSTALL_DIR/include/"
+cp "Python-${VERSION}/pyconfig.h" "$INSTALL_DIR/include/"
+
+# Clean up
+rm -rf "Python-${VERSION}" "$TARBALL"
+
+echo "Installed python3-dev headers to $INSTALL_DIR"
+du -sh "$INSTALL_DIR"

python3-dev/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "python3-dev"
+version = "3.12.8"
+description = "CPython development headers"
+requires-python = ">=3.8"
+
+[tool.setuptools.packages.find]
+include = ["python3_dev*"]
+
+[tool.setuptools.package-data]
+python3_dev = ["install/**/*"]

python3-dev/python3_dev/__init__.py

@@ -0,0 +1,9 @@
+import os
+
+DIR = os.path.join(os.path.dirname(__file__), "install")
+INCLUDE_DIR = os.path.join(DIR, "include")
+
+
+def smoketest():
+  assert os.path.isfile(os.path.join(INCLUDE_DIR, "Python.h")), "Python.h not found"
+  assert os.path.isfile(os.path.join(INCLUDE_DIR, "pyconfig.h")), "pyconfig.h not found"

python3-dev/setup.py

@@ -0,0 +1,61 @@
+import os
+import platform
+import subprocess
+
+from setuptools.command.build_py import build_py
+
+try:
+  from wheel.bdist_wheel import bdist_wheel
+except ImportError:
+  bdist_wheel = None
+
+
+class BuildPython3Dev(build_py):
+  """Run build.sh to download CPython headers before collecting package data."""
+
+  def run(self):
+    pkg_dir = os.path.dirname(os.path.abspath(__file__))
+    marker = os.path.join(pkg_dir, "python3_dev", "install", "include", "Python.h")
+
+    if not os.path.exists(marker):
+      build_script = os.path.join(pkg_dir, "build.sh")
+      subprocess.check_call(["bash", build_script], cwd=pkg_dir)
+
+    super().run()
+
+
+cmdclass = {"build_py": BuildPython3Dev}
+
+if bdist_wheel is not None:
+
+  class PlatformWheel(bdist_wheel):
+    """Produce a platform-specific, Python-version-agnostic wheel."""
+
+    def finalize_options(self):
+      super().finalize_options()
+      self.root_is_pure = False
+
+    def get_tag(self):
+      system = platform.system()
+      machine = platform.machine()
+
+      if system == "Linux":
+        plat = f"linux_{machine}"
+      elif system == "Darwin":
+        plat = "macosx_11_0_arm64"
+      else:
+        plat = f"{system.lower()}_{machine}"
+
+      return "py3", "none", plat
+
+  cmdclass["bdist_wheel"] = PlatformWheel
+
+
+def setup():
+  from setuptools import setup as _setup
+
+  _setup(cmdclass=cmdclass)
+
+
+if __name__ == "__main__":
+  setup()