simplify building (#26)

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  build:
+    name: build (${{ matrix.platform }})
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux-x86_64
+            runs-on: ubuntu-latest
+          - platform: linux-arm64
+            runs-on: ubuntu-24.04-arm
+          - platform: macos
+            runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/cache@v4
+        with:
+          key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }}
+          restore-keys: build-cache-v1-${{ matrix.platform }}-
+          path: |
+            .uv-cache
+            */*/install
+            */*/toolchain
+            */*/bin
+      - name: Build wheels
+        env:
+          MANYLINUX: ${{ runner.os == 'Linux' && '1' || '0' }}
+          UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
+          BUILD_SH_REUSE_MANYLINUX_ARTIFACTS: "1"
+        run: ./build.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.platform }}
+          path: dist/*.whl
+
+  test:
+    name: test distro (${{ matrix.image }})
+    needs: build
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: debian:bookworm-slim
+            runs-on: ubuntu-latest
+          - image: ubuntu:24.04
+            runs-on: ubuntu-latest
+          - image: ubuntu:20.04
+            runs-on: ubuntu-latest
+          - image: fedora:41
+            runs-on: ubuntu-latest
+          - image: archlinux:latest
+            runs-on: ubuntu-latest
+          - image: opensuse/tumbleweed:latest
+            runs-on: ubuntu-latest
+          - image: ghcr.io/void-linux/void-glibc:latest
+            runs-on: ubuntu-latest
+          - image: debian:bookworm-slim
+            runs-on: ubuntu-24.04-arm
+          # TODO: add musl test
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ contains(matrix.runs-on, 'arm') && 'wheels-linux-arm64' || 'wheels-linux-x86_64' }}
+          path: dist/
+      - run: ./test_wheels_in_image.sh "${{ matrix.image }}"
+
+  publish:
+    name: publish wheels
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [build, test]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          merge-multiple: true
+          path: dist
+      - uses: actions/cache/restore@v4
+        with:
+          key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }}
+          restore-keys: build-cache-v1-${{ matrix.platform }}-
+          path: |
+            .uv-cache
+            */*/install
+            */*/toolchain
+            */*/bin
+      - name: Publish wheels and shims
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./release.sh --publish-only

.github/workflows/release.yml

@@ -1,217 +0,0 @@
-name: release wheels
-
-on:
-  push:
-    branches: [master]
-
-jobs:
-  build:
-    name: build - ${{ matrix.platform }}
-    runs-on: ${{ matrix.runs-on }}
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux-x86_64
-            runs-on: ubuntu-latest
-          - platform: linux-arm64
-            runs-on: ubuntu-24.04-arm
-          - platform: macos
-            runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v6
-
-      - uses: actions/setup-python@v6
-        if: runner.os != 'Linux'
-        with:
-          python-version: "3.14"
-
-      - name: Build wheels
-        run: ./build_wheels.sh dist/
-
-      - name: Upload releases
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          for toml in */pyproject.toml; do
-            [ -f "$toml" ] || continue
-            pkg="$(dirname "$toml")"
-            module="${pkg//-/_}"
-            version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
-            tag="${pkg}/v${version}"
-
-            echo "========================================="
-            echo "Uploading: $pkg v$version"
-            echo "========================================="
-
-            whl=$(ls dist/${module}-${version}-*.whl 2>/dev/null) || continue
-            echo "Wheel: $whl"
-            if gh release view "$tag" --repo "$GITHUB_REPOSITORY" &>/dev/null; then
-              gh release upload "$tag" "$whl" --repo "$GITHUB_REPOSITORY" --clobber
-            else
-              gh release create "$tag" "$whl" \
-                --repo "$GITHUB_REPOSITORY" \
-                --title "$pkg v$version" \
-                --notes "Platform wheels for $pkg $version"
-            fi
-          done
-
-  publish-shims:
-    name: publish releases branch
-    needs: build
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: build and push shim packages
-        run: |
-          REPO_URL="https://github.com/${GITHUB_REPOSITORY}"
-
-          # set up a temp dir for the releases branch
-          tmp="$(mktemp -d)"
-          trap 'rm -rf "$tmp"' EXIT
-
-          for toml in */pyproject.toml; do
-            [ -f "$toml" ] || continue
-            pkg="$(dirname "$toml")"
-            version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
-            tag="${pkg}/v${version}"
-            module="${pkg//-/_}"
-            # detect the package-data directory (e.g. "toolchain" or "install")
-            datadir=$(python3 -c "
-          import tomllib
-          t = tomllib.load(open('$toml', 'rb'))
-          pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0]
-          print(pat.split('/')[0])
-          ")
-            description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])")
-
-            echo "========================================="
-            echo "Creating shim: $pkg v$version (datadir=$datadir)"
-            echo "========================================="
-
-            pkg_dir="$tmp/$pkg"
-            mod_dir="$pkg_dir/$module"
-            mkdir -p "$mod_dir"
-
-            # copy __init__.py from source
-            cp "$pkg/$module/__init__.py" "$mod_dir/"
-
-            cat > "$pkg_dir/pyproject.toml" << TOML
-          [build-system]
-          requires = ["setuptools>=64", "wheel"]
-          build-backend = "setuptools.build_meta"
-
-          [project]
-          name = "$pkg"
-          version = "$version"
-          description = "$description (pre-built)"
-          requires-python = ">=3.8"
-
-          [project.scripts]
-          $(grep -A100 '^\[project\.scripts\]' "$pkg/pyproject.toml" | tail -n +2 | sed '/^\[/q' | grep -v '^\[')
-
-          [tool.setuptools.packages.find]
-          include = ["${module}*"]
-
-          [tool.setuptools.package-data]
-          $module = ["${datadir}/**/*"]
-          TOML
-
-            # setup.py — downloads pre-built wheel from GH releases and extracts data
-            cat > "$pkg_dir/setup.py" << 'SETUP'
-          import os
-          import platform
-          import zipfile
-          from io import BytesIO
-          from urllib.request import urlopen
-
-          from setuptools.command.build_py import build_py
-
-          REPO_URL = "REPO_URL_PLACEHOLDER"
-          TAG = "TAG_PLACEHOLDER"
-          VERSION = "VERSION_PLACEHOLDER"
-          MODULE = "MODULE_PLACEHOLDER"
-          DATADIR = "DATADIR_PLACEHOLDER"
-
-          PLATFORM_MAP = {
-            ("Linux", "x86_64"): "linux_x86_64",
-            ("Linux", "aarch64"): "linux_aarch64",
-            ("Darwin", "arm64"): "macosx_11_0_arm64",
-          }
-
-
-          class InstallPrebuilt(build_py):
-            """Download pre-built wheel from GitHub Releases and extract binaries."""
-
-            def run(self):
-              pkg_dir = os.path.dirname(os.path.abspath(__file__))
-              data_dir = os.path.join(pkg_dir, MODULE, DATADIR)
-
-              if not os.path.exists(os.path.join(data_dir, "bin")):
-                key = (platform.system(), platform.machine())
-                plat = PLATFORM_MAP.get(key)
-                if plat is None:
-                  raise RuntimeError(f"unsupported platform: {key}")
-
-                whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl"
-                url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}"
-
-                print(f"Downloading {url} ...")
-                data = urlopen(url).read()
-
-                print(f"Extracting {DATADIR} ...")
-                with zipfile.ZipFile(BytesIO(data)) as zf:
-                  prefix = f"{MODULE}/{DATADIR}/"
-                  alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/"
-                  for info in zf.infolist():
-                    for p in (prefix, alt_prefix):
-                      if info.filename.startswith(p):
-                        rel = info.filename[len(p):]
-                        if not rel:
-                          continue
-                        dest = os.path.join(data_dir, rel)
-                        if info.is_dir():
-                          os.makedirs(dest, exist_ok=True)
-                        else:
-                          os.makedirs(os.path.dirname(dest), exist_ok=True)
-                          with open(dest, "wb") as f:
-                            f.write(zf.read(info))
-                          if info.external_attr >> 16 & 0o111:
-                            os.chmod(dest, 0o755)
-                        break
-
-              super().run()
-
-
-          def setup():
-            from setuptools import setup as _setup
-            _setup(cmdclass={"build_py": InstallPrebuilt})
-
-
-          if __name__ == "__main__":
-            setup()
-          SETUP
-
-            # template in the actual values
-            sed -i "s|REPO_URL_PLACEHOLDER|${REPO_URL}|" "$pkg_dir/setup.py"
-            sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py"
-            sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py"
-            sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py"
-            sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py"
-          done
-
-          # push to releases branch
-          cd "$tmp"
-          git init
-          git checkout -b releases
-          git add .
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git commit -m "update shim packages"
-          git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git"
-          git push -f origin releases

.github/workflows/test.yml

@@ -1,90 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-
-jobs:
-  build:
-    name: build (${{ matrix.platform }})
-    runs-on: ${{ matrix.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: macos
-            runs-on: macos-latest
-          - platform: linux-x86_64
-            runs-on: ubuntu-latest
-          - platform: linux-arm64
-            runs-on: ubuntu-24.04-arm
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        if: runner.os != 'Linux'
-        with:
-          python-version: "3.14"
-      - name: Build wheels
-        run: ./build_wheels.sh dist/
-      - name: Smoketest
-        run: bash smoketest.sh dist/
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.platform }}
-          path: dist/*.whl
-
-  test:
-    name: test (${{ matrix.image }})
-    needs: build
-    runs-on: ${{ matrix.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # test on vanilla linux images
-          - image: debian:bookworm-slim
-            runs-on: ubuntu-latest
-          - image: ubuntu:24.04
-            runs-on: ubuntu-latest
-          - image: ubuntu:20.04
-            runs-on: ubuntu-latest
-          - image: fedora:41
-            runs-on: ubuntu-latest
-          - image: archlinux:latest
-            runs-on: ubuntu-latest
-          # TODO: make musl work
-          #- image: alpine:3.21
-          #  runs-on: ubuntu-latest
-          - image: opensuse/tumbleweed:latest
-            runs-on: ubuntu-latest
-          - image: ghcr.io/void-linux/void-glibc:latest
-            runs-on: ubuntu-latest
-          - image: debian:bookworm-slim
-            runs-on: ubuntu-24.04-arm
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/download-artifact@v4
-        with:
-          name: wheels-${{ contains(matrix.runs-on, 'arm') && 'linux-arm64' || 'linux-x86_64' }}
-          path: wheels
-      - name: build docker
-        run: |
-          docker build -t smoketest -f - . <<'DOCKERFILE'
-          FROM ${{ matrix.image }}
-          RUN if command -v apk >/dev/null; then \
-                apk add --no-cache python3 py3-pip bash; \
-              elif command -v apt-get >/dev/null; then \
-                apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \
-              elif command -v dnf >/dev/null; then \
-                dnf install -y python3 python3-pip; \
-              elif command -v pacman >/dev/null; then \
-                pacman -Sy --noconfirm python python-pip; \
-              elif command -v zypper >/dev/null; then \
-                zypper install -y python3 python3-pip; \
-              elif command -v xbps-install >/dev/null; then \
-                xbps-install -Sy python3 python3-pip bash; \
-              fi
-          DOCKERFILE
-      - name: ./smoketest.sh
-        run: docker run --rm -v "$PWD:/work" -w /work smoketest bash smoketest.sh wheels/

.gitignore

@@ -13,10 +13,16 @@ build/
 dist/
 install/
 *.egg-info/
+*-src/
+*.tar.gz
+*.tar.xz
+*.tgz
+Python-*/
+ncurses-*/
+arm-gnu-toolchain-*/
 
 # downloaded toolchain binaries
 gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/
 
-
 # downloaded git-lfs
 git-lfs/git_lfs/bin/

_shim_setup.py

@@ -0,0 +1,73 @@
+"""Shim setup.py: downloads pre-built wheels from GitHub Releases at install time."""
+import os
+import platform
+import zipfile
+from io import BytesIO
+from urllib.request import urlopen
+
+try:
+  import tomllib
+except ImportError:
+  import tomli as tomllib
+
+from setuptools import setup
+from setuptools.command.build_py import build_py
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+with open(os.path.join(_HERE, "pyproject.toml"), "rb") as _f:
+  _cfg = tomllib.load(_f)
+
+REPO_URL = _cfg["tool"]["shim"]["repo_url"]
+TAG = _cfg["tool"]["shim"]["tag"]
+DATADIR = _cfg["tool"]["shim"]["datadir"]
+VERSION = _cfg["project"]["version"]
+MODULE = _cfg["project"]["name"].replace("-", "_")
+
+PLATFORM_MAP = {
+  ("Linux", "x86_64"): "linux_x86_64",
+  ("Linux", "aarch64"): "linux_aarch64",
+  ("Darwin", "arm64"): "macosx_11_0_arm64",
+}
+
+
+class InstallPrebuilt(build_py):
+  def run(self):
+    data_dir = os.path.join(_HERE, MODULE, DATADIR)
+
+    if not os.path.exists(os.path.join(data_dir, "bin")):
+      key = (platform.system(), platform.machine())
+      plat = PLATFORM_MAP.get(key)
+      if plat is None:
+        raise RuntimeError(f"unsupported platform: {key}")
+
+      whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl"
+      url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}"
+
+      print(f"Downloading {url} ...")
+      raw = urlopen(url).read()
+
+      print(f"Extracting {DATADIR} ...")
+      with zipfile.ZipFile(BytesIO(raw)) as zf:
+        prefix = f"{MODULE}/{DATADIR}/"
+        alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/"
+        for info in zf.infolist():
+          for p in (prefix, alt_prefix):
+            if info.filename.startswith(p):
+              rel = info.filename[len(p):]
+              if not rel:
+                continue
+              dest = os.path.join(data_dir, rel)
+              if info.is_dir():
+                os.makedirs(dest, exist_ok=True)
+              else:
+                os.makedirs(os.path.dirname(dest), exist_ok=True)
+                with open(dest, "wb") as f:
+                  f.write(zf.read(info))
+                if info.external_attr >> 16 & 0o111:
+                  os.chmod(dest, 0o755)
+              break
+
+    super().run()
+
+
+setup(cmdclass={"build_py": InstallPrebuilt})

build.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$ROOT_DIR"
+
+if [[ $# -gt 0 ]]; then
+  echo "usage: MANYLINUX=1 ./build.sh" >&2
+  exit 2
+fi
+
+USE_MANYLINUX="${MANYLINUX:-0}"
+
+if [[ -z "${BUILD_SH_IN_MANYLINUX:-}" ]] && ! command -v uv >/dev/null 2>&1; then
+  ./setup.sh
+fi
+
+if [[ "$USE_MANYLINUX" == "1" && -z "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
+  UV_BIN="$(command -v uv)"
+  docker run --rm \
+    -e BUILD_SH_IN_MANYLINUX=1 \
+    -e BUILD_SH_REUSE_MANYLINUX_ARTIFACTS="${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" \
+    -e HOME=/tmp \
+    -e UV_CACHE_DIR=/work/.uv-cache \
+    -e UV_PYTHON=/opt/python/cp312-cp312/bin/python3 \
+    -v "$ROOT_DIR:/work" \
+    -v "$UV_BIN:/usr/local/bin/uv:ro" \
+    -w /work \
+    "quay.io/pypa/manylinux_2_28_$(uname -m)" \
+    bash build.sh
+  exit 0
+fi
+
+if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
+  export PATH="/opt/python/cp312-cp312/bin:$PATH"
+
+  ./setup.sh
+
+  if [[ -z "${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" ]]; then
+    for toml in */pyproject.toml; do
+      pkg="${toml%/pyproject.toml}"
+      module="${pkg//-/_}"
+      rm -rf "$pkg/$module/install" "$pkg/$module/toolchain" "$pkg/$module/bin"
+    done
+  fi
+fi
+
+echo "Building workspace packages into dist"
+START_SECS=$SECONDS
+
+uv build --all-packages --wheel --out-dir dist --no-create-gitignore --no-build-logs
+
+if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then
+  VENV_DIR="$ROOT_DIR/.venv-manylinux"
+else
+  VENV_DIR="$ROOT_DIR/.venv"
+fi
+
+echo
+echo "Running smoketests"
+
+uv venv --allow-existing --quiet "$VENV_DIR"
+uv pip install --python "$VENV_DIR/bin/python" --reinstall --no-deps --quiet dist/*.whl >/dev/null
+
+for toml in */pyproject.toml; do
+  module="$(basename "$(dirname "$toml")" | tr '-' '_')"
+  "$VENV_DIR/bin/python" -c "import $module; $module.smoketest()" >/dev/null
+done
+
+du -hs dist/* | sort -hr
+
+echo
+echo "Done in $((SECONDS - START_SECS))s"

build_wheels.sh

@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$DIR"
-
-WHEEL_DIR="${1:-dist}"
-
-# on Linux, build inside manylinux_2_28 container for glibc 2.28 compatibility.
-# for reference, Ubuntu 20.04 is glibc 2.31, so this gives us wide compatibility.
-if [ "$(uname)" = "Linux" ] && [ -z "${MANYLINUX:-}" ]; then
-  ARCH="$(uname -m)"
-  IMAGE="quay.io/pypa/manylinux_2_28_${ARCH}"
-  exec docker run --rm \
-    -e MANYLINUX=1 \
-    -v "$DIR:/work" \
-    -w /work \
-    "$IMAGE" \
-    bash build_wheels.sh "$WHEEL_DIR"
-fi
-
-# Set up Python inside manylinux container
-if [ -n "${MANYLINUX:-}" ]; then
-  export PATH="/opt/python/cp312-cp312/bin:$PATH"
-fi
-
-./setup.sh
-pip install setuptools wheel
-
-mkdir -p "$WHEEL_DIR"
-for pkg in */pyproject.toml; do
-  pip wheel "./$(dirname "$pkg")" --no-deps --wheel-dir "$WHEEL_DIR"/
-done

capnproto/capnproto/__init__.py

@@ -29,3 +29,16 @@ def _run_capnpc():
 
 def _run_capnpc_cpp():
   _run("capnpc-c++")
+
+
+def smoketest():
+  import subprocess
+
+  capnp = os.path.join(BIN_DIR, "capnp")
+  capnpc = os.path.join(BIN_DIR, "capnpc")
+  capnpc_cpp = os.path.join(BIN_DIR, "capnpc-c++")
+  env = os.environ.copy()
+  env["PATH"] = BIN_DIR + ":" + env.get("PATH", "")
+  subprocess.run([capnp, "--version"], check=True, env=env)
+  subprocess.run([capnpc, "--version"], check=True, env=env)
+  subprocess.run([capnpc_cpp, "--version"], check=True, env=env)

ffmpeg/ffmpeg/__init__.py

@@ -18,3 +18,12 @@ def _run_ffmpeg():
 
 def _run_ffprobe():
   _run("ffprobe")
+
+
+def smoketest():
+  import subprocess
+
+  ffmpeg = os.path.join(BIN_DIR, "ffmpeg")
+  ffprobe = os.path.join(BIN_DIR, "ffprobe")
+  subprocess.run([ffmpeg, "-version"], check=True)
+  subprocess.run([ffprobe, "-version"], check=True)

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.uv.workspace]
+members = [
+  "capnproto",
+  "cppcheck",
+  "eigen",
+  "ffmpeg",
+  "gcc-arm-none-eabi",
+  "git-lfs",
+  "libjpeg",
+  "ncurses",
+  "openssl3",
+  "python3-dev",
+  "zeromq",
+  "zstd",
+]

release.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$ROOT_DIR"
+
+REPO=commaai/dependencies
+
+echo
+echo "Publishing wheels to GitHub Releases ($REPO)"
+
+shopt -s nullglob
+for toml in */pyproject.toml; do
+  pkg="$(dirname "$toml")"
+  module="${pkg//-/_}"
+  version="$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")"
+  tag="${pkg}/v${version}"
+
+  wheels=("dist/${module}-${version}-"*.whl)
+  if [[ ${#wheels[@]} -eq 0 ]]; then
+    echo "missing wheel for $pkg ($module-$version) in dist" >&2
+    exit 1
+  fi
+
+  echo "[$pkg] Uploading ${#wheels[@]} wheel(s) to $tag"
+
+  gh release create "$tag" "${wheels[@]}" --repo "$REPO" --title "$pkg v$version" --notes "Platform wheels for $pkg $version" 2>/dev/null ||
+    gh release upload "$tag" "${wheels[@]}" --repo "$REPO" --clobber
+done
+shopt -u nullglob
+
+TOKEN="$(gh auth token 2>/dev/null)" || { echo "set GH_TOKEN to publish shim branch" >&2; exit 1; }
+
+TMP_DIR="$(mktemp -d)"
+python3 - "$TMP_DIR" "$REPO" <<'PY'
+import json
+import pathlib
+import shutil
+import tomllib
+import sys
+
+tmp_dir = pathlib.Path(sys.argv[1])
+repo = sys.argv[2]
+repo_url = f"https://github.com/{repo}"
+shim_setup = pathlib.Path("_shim_setup.py")
+
+for toml in sorted(pathlib.Path(".").glob("*/pyproject.toml")):
+  pkg = toml.parent.name
+  module = pkg.replace("-", "_")
+  data = tomllib.load(toml.open("rb"))
+  version = str(data["project"]["version"])
+  tag = f"{pkg}/v{version}"
+  description = data["project"]["description"]
+  patterns = data.get("tool", {}).get("setuptools", {}).get("package-data", {}).get(module, [""])
+  datadir = patterns[0].split("/", 1)[0] if patterns and patterns[0] else ""
+  scripts = data.get("project", {}).get("scripts", {}) or {}
+
+  pkg_dir = tmp_dir / pkg
+  mod_dir = pkg_dir / module
+  mod_dir.mkdir(parents=True, exist_ok=True)
+  shutil.copy2(pathlib.Path(pkg) / module / "__init__.py", mod_dir / "__init__.py")
+  shutil.copy2(shim_setup, pkg_dir / "setup.py")
+
+  lines = [
+    "[build-system]",
+    'requires = ["setuptools>=64", "wheel", \'tomli; python_version < \"3.11\"\']',
+    'build-backend = "setuptools.build_meta"',
+    "",
+    "[project]",
+    f'name = "{pkg}"',
+    f'version = "{version}"',
+    f"description = {json.dumps(description + ' (pre-built)')}",
+    'requires-python = ">=3.8"',
+  ]
+
+  if scripts:
+    lines += ["", "[project.scripts]"]
+    for name, target in scripts.items():
+      lines.append(f'"{name}" = {json.dumps(target)}')
+
+  lines += [
+    "",
+    "[tool.setuptools.packages.find]",
+    f'include = ["{module}*"]',
+    "",
+    "[tool.setuptools.package-data]",
+    f'{module} = ["{datadir}/**/*"]',
+    "",
+    "[tool.shim]",
+    f'repo_url = "{repo_url}"',
+    f'tag = "{tag}"',
+    f'datadir = "{datadir}"',
+  ]
+
+  (pkg_dir / "pyproject.toml").write_text("\n".join(lines) + "\n")
+PY
+
+(
+  cd "$TMP_DIR"
+  git init
+  git checkout -b releases
+  git add .
+  git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" commit -m "update shim packages"
+  git remote add origin "https://x-access-token:${TOKEN}@github.com/${REPO}.git"
+  git push -f origin releases
+)
+
+rm -rf "$TMP_DIR"

setup.sh

@@ -1,10 +1,39 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# sets up build-time dependencies (NOT runtime dependencies)
+
+run_as_root() {
+  if [ "$(id -u)" -eq 0 ]; then
+    "$@"
+  elif command -v sudo &>/dev/null; then
+    sudo "$@"
+  else
+    echo "error: root privileges required for: $*" >&2
+    exit 1
+  fi
+}
+
 if [ "$(uname)" = "Darwin" ]; then
   brew install nasm pkg-config
 elif command -v dnf &>/dev/null; then
   dnf install -y nasm cmake gcc-c++ pkgconfig git perl-IPC-Cmd
 elif command -v apt-get &>/dev/null; then
-  sudo apt-get update && sudo apt-get install -y nasm cmake g++ pkg-config
+  run_as_root apt-get update
+  run_as_root apt-get install -y nasm cmake g++ pkg-config curl
+fi
+
+if ! command -v uv &>/dev/null; then
+  command -v curl &>/dev/null || {
+    echo "error: curl is required to install uv" >&2
+    exit 1
+  }
+  UV_BIN_DIR="$HOME/.local/bin"
+  mkdir -p "$UV_BIN_DIR"
+  curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$UV_BIN_DIR" sh
+  export PATH="$UV_BIN_DIR:$PATH"
+  command -v uv &>/dev/null || {
+    echo "error: failed to install uv" >&2
+    exit 1
+  }
 fi

smoketest.sh

@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-WHEEL_DIR="${1:?Usage: smoketest.sh <wheel-directory>}"
-WHEEL_DIR="$(cd "$WHEEL_DIR" && pwd)"
-
-VENV_DIR="$(mktemp -d)"
-trap 'rm -rf "$VENV_DIR"' EXIT
-python3 -m venv "$VENV_DIR"
-source "$VENV_DIR/bin/activate"
-pip install --upgrade pip >/dev/null
-pip install "$WHEEL_DIR"/*.whl
-
-for toml in "$REPO_DIR"/*/pyproject.toml; do
-  module="$(basename "$(dirname "$toml")" | tr '-' '_')"
-  python3 -c "import $module; $module.smoketest()" && echo "$module: OK"
-done

test.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
-
-# Create a temporary virtualenv
-VENV_DIR="$(mktemp -d)"
-trap 'rm -rf "$VENV_DIR"' EXIT
-python3 -m venv "$VENV_DIR"
-source "$VENV_DIR/bin/activate"
-pip install --upgrade pip >/dev/null
-
-# Auto-discover packages: any subdirectory containing a pyproject.toml
-PACKAGES=()
-for toml in "$REPO_DIR"/*/pyproject.toml; do
-  [ -f "$toml" ] || continue
-  PACKAGES+=("$(dirname "$toml")")
-done
-
-if [ ${#PACKAGES[@]} -eq 0 ]; then
-  echo "No packages found."
-  exit 1
-fi
-
-FAILED=()
-
-for pkg in "${PACKAGES[@]}"; do
-  name="$(basename "$pkg")"
-  echo "========================================="
-  echo "Testing: $name"
-  echo "========================================="
-
-  # Install from git URL with subdirectory (simulates real-world usage)
-  echo "[$name] pip install ..."
-  pip install "$pkg" --verbose
-
-  # Verify import works
-  module="${name//-/_}"
-  echo "[$name] Verifying import of $module ..."
-  python -c "import $module; print(f'{$module.__name__} OK')"
-
-  echo "[$name] PASSED"
-  echo
-done
-
-if [ ${#FAILED[@]} -ne 0 ]; then
-  echo "FAILED packages: ${FAILED[*]}"
-  exit 1
-fi
-
-echo "All ${#PACKAGES[@]} package(s) passed."
-echo
-echo "Installed sizes:"
-for pkg in "${PACKAGES[@]}"; do
-  name="$(basename "$pkg")"
-  module="${name//-/_}"
-  mod_dir="$(python -c "import $module, os; print(os.path.dirname($module.__file__))")"
-  size="$(du -sh "$mod_dir" | cut -f1)"
-  echo "  $name: $size"
-done

test_wheels_in_image.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# a small script for testing our built wheels in a variety of linux distros
+
+IMAGE="${1:?usage: ./test_wheels_in_image.sh <image>}"
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd "$ROOT_DIR"
+
+# setup our deps - should only need a python3 build
+docker build -t wheeltest -f - . <<DOCKERFILE
+FROM $IMAGE
+RUN if command -v apk >/dev/null; then \
+      apk add --no-cache python3 py3-pip bash; \
+    elif command -v apt-get >/dev/null; then \
+      apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \
+    elif command -v dnf >/dev/null; then \
+      dnf install -y python3 python3-pip; \
+    elif command -v pacman >/dev/null; then \
+      pacman -Sy --noconfirm python python-pip; \
+    elif command -v zypper >/dev/null; then \
+      zypper install -y python3 python3-pip; \
+    elif command -v xbps-install >/dev/null; then \
+      xbps-install -Sy python3 python3-pip bash; \
+    fi
+DOCKERFILE
+
+# install + smoketest
+docker run --rm -v "$PWD:/work" -w /work wheeltest bash -lc '
+  set -euo pipefail
+  python3 -m venv /tmp/venv
+  source /tmp/venv/bin/activate
+  pip install dist/*.whl
+  for toml in */pyproject.toml; do
+    module="$(basename "$(dirname "$toml")" | tr "-" "_")"
+    python -c "import $module; $module.smoketest()" && echo "$module: OK"
+  done
+'
+
+echo
+echo "All good!"