diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b2e5c3c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: ci + +on: + pull_request: + push: + branches: [master] + +jobs: + build: + name: build (${{ matrix.platform }}) + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x86_64 + runs-on: ubuntu-latest + - platform: linux-arm64 + runs-on: ubuntu-24.04-arm + - platform: macos + runs-on: macos-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/cache@v4 + with: + key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }} + restore-keys: build-cache-v1-${{ matrix.platform }}- + path: | + .uv-cache + */*/install + */*/toolchain + */*/bin + - name: Build wheels + env: + MANYLINUX: ${{ runner.os == 'Linux' && '1' || '0' }} + UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache + BUILD_SH_REUSE_MANYLINUX_ARTIFACTS: "1" + run: ./build.sh + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform }} + path: dist/*.whl + + test: + name: test distro (${{ matrix.image }}) + needs: build + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + include: + - image: debian:bookworm-slim + runs-on: ubuntu-latest + - image: ubuntu:24.04 + runs-on: ubuntu-latest + - image: ubuntu:20.04 + runs-on: ubuntu-latest + - image: fedora:41 + runs-on: ubuntu-latest + - image: archlinux:latest + runs-on: ubuntu-latest + - image: opensuse/tumbleweed:latest + runs-on: ubuntu-latest + - image: ghcr.io/void-linux/void-glibc:latest + runs-on: ubuntu-latest + - image: debian:bookworm-slim + runs-on: ubuntu-24.04-arm + # TODO: add musl test + steps: + - uses: actions/checkout@v6 + - uses: actions/download-artifact@v4 + with: + name: ${{ contains(matrix.runs-on, 'arm') && 'wheels-linux-arm64' || 'wheels-linux-x86_64' }} + path: dist/ + - run: ./test_wheels_in_image.sh "${{ matrix.image }}" + + publish: + name: publish wheels + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: [build, test] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist + - uses: actions/cache/restore@v4 + with: + key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }} + restore-keys: build-cache-v1-${{ matrix.platform }}- + path: | + .uv-cache + */*/install + */*/toolchain + */*/bin + - name: Publish wheels and shims + env: + GH_TOKEN: ${{ github.token }} + run: ./release.sh --publish-only diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 1ad11fe..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,217 +0,0 @@ -name: release wheels - -on: - push: - branches: [master] - -jobs: - build: - name: build - ${{ matrix.platform }} - runs-on: ${{ matrix.runs-on }} - permissions: - contents: write - strategy: - fail-fast: false - matrix: - include: - - platform: linux-x86_64 - runs-on: ubuntu-latest - - platform: linux-arm64 - runs-on: ubuntu-24.04-arm - - platform: macos - runs-on: macos-latest - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-python@v6 - if: runner.os != 'Linux' - with: - python-version: "3.14" - - - name: Build wheels - run: ./build_wheels.sh dist/ - - - name: Upload releases - env: - GH_TOKEN: ${{ github.token }} - run: | - for toml in */pyproject.toml; do - [ -f "$toml" ] || continue - pkg="$(dirname "$toml")" - module="${pkg//-/_}" - version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])") - tag="${pkg}/v${version}" - - echo "=========================================" - echo "Uploading: $pkg v$version" - echo "=========================================" - - whl=$(ls dist/${module}-${version}-*.whl 2>/dev/null) || continue - echo "Wheel: $whl" - if gh release view "$tag" --repo "$GITHUB_REPOSITORY" &>/dev/null; then - gh release upload "$tag" "$whl" --repo "$GITHUB_REPOSITORY" --clobber - else - gh release create "$tag" "$whl" \ - --repo "$GITHUB_REPOSITORY" \ - --title "$pkg v$version" \ - --notes "Platform wheels for $pkg $version" - fi - done - - publish-shims: - name: publish releases branch - needs: build - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v6 - - - name: build and push shim packages - run: | - REPO_URL="https://github.com/${GITHUB_REPOSITORY}" - - # set up a temp dir for the releases branch - tmp="$(mktemp -d)" - trap 'rm -rf "$tmp"' EXIT - - for toml in */pyproject.toml; do - [ -f "$toml" ] || continue - pkg="$(dirname "$toml")" - version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])") - tag="${pkg}/v${version}" - module="${pkg//-/_}" - # detect the package-data directory (e.g. "toolchain" or "install") - datadir=$(python3 -c " - import tomllib - t = tomllib.load(open('$toml', 'rb')) - pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0] - print(pat.split('/')[0]) - ") - description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])") - - echo "=========================================" - echo "Creating shim: $pkg v$version (datadir=$datadir)" - echo "=========================================" - - pkg_dir="$tmp/$pkg" - mod_dir="$pkg_dir/$module" - mkdir -p "$mod_dir" - - # copy __init__.py from source - cp "$pkg/$module/__init__.py" "$mod_dir/" - - cat > "$pkg_dir/pyproject.toml" << TOML - [build-system] - requires = ["setuptools>=64", "wheel"] - build-backend = "setuptools.build_meta" - - [project] - name = "$pkg" - version = "$version" - description = "$description (pre-built)" - requires-python = ">=3.8" - - [project.scripts] - $(grep -A100 '^\[project\.scripts\]' "$pkg/pyproject.toml" | tail -n +2 | sed '/^\[/q' | grep -v '^\[') - - [tool.setuptools.packages.find] - include = ["${module}*"] - - [tool.setuptools.package-data] - $module = ["${datadir}/**/*"] - TOML - - # setup.py — downloads pre-built wheel from GH releases and extracts data - cat > "$pkg_dir/setup.py" << 'SETUP' - import os - import platform - import zipfile - from io import BytesIO - from urllib.request import urlopen - - from setuptools.command.build_py import build_py - - REPO_URL = "REPO_URL_PLACEHOLDER" - TAG = "TAG_PLACEHOLDER" - VERSION = "VERSION_PLACEHOLDER" - MODULE = "MODULE_PLACEHOLDER" - DATADIR = "DATADIR_PLACEHOLDER" - - PLATFORM_MAP = { - ("Linux", "x86_64"): "linux_x86_64", - ("Linux", "aarch64"): "linux_aarch64", - ("Darwin", "arm64"): "macosx_11_0_arm64", - } - - - class InstallPrebuilt(build_py): - """Download pre-built wheel from GitHub Releases and extract binaries.""" - - def run(self): - pkg_dir = os.path.dirname(os.path.abspath(__file__)) - data_dir = os.path.join(pkg_dir, MODULE, DATADIR) - - if not os.path.exists(os.path.join(data_dir, "bin")): - key = (platform.system(), platform.machine()) - plat = PLATFORM_MAP.get(key) - if plat is None: - raise RuntimeError(f"unsupported platform: {key}") - - whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl" - url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}" - - print(f"Downloading {url} ...") - data = urlopen(url).read() - - print(f"Extracting {DATADIR} ...") - with zipfile.ZipFile(BytesIO(data)) as zf: - prefix = f"{MODULE}/{DATADIR}/" - alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/" - for info in zf.infolist(): - for p in (prefix, alt_prefix): - if info.filename.startswith(p): - rel = info.filename[len(p):] - if not rel: - continue - dest = os.path.join(data_dir, rel) - if info.is_dir(): - os.makedirs(dest, exist_ok=True) - else: - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, "wb") as f: - f.write(zf.read(info)) - if info.external_attr >> 16 & 0o111: - os.chmod(dest, 0o755) - break - - super().run() - - - def setup(): - from setuptools import setup as _setup - _setup(cmdclass={"build_py": InstallPrebuilt}) - - - if __name__ == "__main__": - setup() - SETUP - - # template in the actual values - sed -i "s|REPO_URL_PLACEHOLDER|${REPO_URL}|" "$pkg_dir/setup.py" - sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py" - sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py" - sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py" - sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py" - done - - # push to releases branch - cd "$tmp" - git init - git checkout -b releases - git add . - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git commit -m "update shim packages" - git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git" - git push -f origin releases diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index e997200..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: ci - -on: - push: - branches: [master] - pull_request: - -jobs: - build: - name: build (${{ matrix.platform }}) - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - matrix: - include: - - platform: macos - runs-on: macos-latest - - platform: linux-x86_64 - runs-on: ubuntu-latest - - platform: linux-arm64 - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - if: runner.os != 'Linux' - with: - python-version: "3.14" - - name: Build wheels - run: ./build_wheels.sh dist/ - - name: Smoketest - run: bash smoketest.sh dist/ - - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.platform }} - path: dist/*.whl - - test: - name: test (${{ matrix.image }}) - needs: build - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - matrix: - include: - # test on vanilla linux images - - image: debian:bookworm-slim - runs-on: ubuntu-latest - - image: ubuntu:24.04 - runs-on: ubuntu-latest - - image: ubuntu:20.04 - runs-on: ubuntu-latest - - image: fedora:41 - runs-on: ubuntu-latest - - image: archlinux:latest - runs-on: ubuntu-latest - # TODO: make musl work - #- image: alpine:3.21 - # runs-on: ubuntu-latest - - image: opensuse/tumbleweed:latest - runs-on: ubuntu-latest - - image: ghcr.io/void-linux/void-glibc:latest - runs-on: ubuntu-latest - - image: debian:bookworm-slim - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@v6 - - uses: actions/download-artifact@v4 - with: - name: wheels-${{ contains(matrix.runs-on, 'arm') && 'linux-arm64' || 'linux-x86_64' }} - path: wheels - - name: build docker - run: | - docker build -t smoketest -f - . <<'DOCKERFILE' - FROM ${{ matrix.image }} - RUN if command -v apk >/dev/null; then \ - apk add --no-cache python3 py3-pip bash; \ - elif command -v apt-get >/dev/null; then \ - apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \ - elif command -v dnf >/dev/null; then \ - dnf install -y python3 python3-pip; \ - elif command -v pacman >/dev/null; then \ - pacman -Sy --noconfirm python python-pip; \ - elif command -v zypper >/dev/null; then \ - zypper install -y python3 python3-pip; \ - elif command -v xbps-install >/dev/null; then \ - xbps-install -Sy python3 python3-pip bash; \ - fi - DOCKERFILE - - name: ./smoketest.sh - run: docker run --rm -v "$PWD:/work" -w /work smoketest bash smoketest.sh wheels/ diff --git a/.gitignore b/.gitignore index 95ea652..b9dc743 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,16 @@ build/ dist/ install/ *.egg-info/ +*-src/ +*.tar.gz +*.tar.xz +*.tgz +Python-*/ +ncurses-*/ +arm-gnu-toolchain-*/ # downloaded toolchain binaries gcc-arm-none-eabi/gcc_arm_none_eabi/toolchain/ - # downloaded git-lfs git-lfs/git_lfs/bin/ diff --git a/_shim_setup.py b/_shim_setup.py new file mode 100644 index 0000000..80e6dde --- /dev/null +++ b/_shim_setup.py @@ -0,0 +1,73 @@ +"""Shim setup.py: downloads pre-built wheels from GitHub Releases at install time.""" +import os +import platform +import zipfile +from io import BytesIO +from urllib.request import urlopen + +try: + import tomllib +except ImportError: + import tomli as tomllib + +from setuptools import setup +from setuptools.command.build_py import build_py + +_HERE = os.path.dirname(os.path.abspath(__file__)) +with open(os.path.join(_HERE, "pyproject.toml"), "rb") as _f: + _cfg = tomllib.load(_f) + +REPO_URL = _cfg["tool"]["shim"]["repo_url"] +TAG = _cfg["tool"]["shim"]["tag"] +DATADIR = _cfg["tool"]["shim"]["datadir"] +VERSION = _cfg["project"]["version"] +MODULE = _cfg["project"]["name"].replace("-", "_") + +PLATFORM_MAP = { + ("Linux", "x86_64"): "linux_x86_64", + ("Linux", "aarch64"): "linux_aarch64", + ("Darwin", "arm64"): "macosx_11_0_arm64", +} + + +class InstallPrebuilt(build_py): + def run(self): + data_dir = os.path.join(_HERE, MODULE, DATADIR) + + if not os.path.exists(os.path.join(data_dir, "bin")): + key = (platform.system(), platform.machine()) + plat = PLATFORM_MAP.get(key) + if plat is None: + raise RuntimeError(f"unsupported platform: {key}") + + whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl" + url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}" + + print(f"Downloading {url} ...") + raw = urlopen(url).read() + + print(f"Extracting {DATADIR} ...") + with zipfile.ZipFile(BytesIO(raw)) as zf: + prefix = f"{MODULE}/{DATADIR}/" + alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/" + for info in zf.infolist(): + for p in (prefix, alt_prefix): + if info.filename.startswith(p): + rel = info.filename[len(p):] + if not rel: + continue + dest = os.path.join(data_dir, rel) + if info.is_dir(): + os.makedirs(dest, exist_ok=True) + else: + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as f: + f.write(zf.read(info)) + if info.external_attr >> 16 & 0o111: + os.chmod(dest, 0o755) + break + + super().run() + + +setup(cmdclass={"build_py": InstallPrebuilt}) diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..9ca69c2 --- /dev/null +++ b/build.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$ROOT_DIR" + +if [[ $# -gt 0 ]]; then + echo "usage: MANYLINUX=1 ./build.sh" >&2 + exit 2 +fi + +USE_MANYLINUX="${MANYLINUX:-0}" + +if [[ -z "${BUILD_SH_IN_MANYLINUX:-}" ]] && ! command -v uv >/dev/null 2>&1; then + ./setup.sh +fi + +if [[ "$USE_MANYLINUX" == "1" && -z "${BUILD_SH_IN_MANYLINUX:-}" ]]; then + UV_BIN="$(command -v uv)" + docker run --rm \ + -e BUILD_SH_IN_MANYLINUX=1 \ + -e BUILD_SH_REUSE_MANYLINUX_ARTIFACTS="${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" \ + -e HOME=/tmp \ + -e UV_CACHE_DIR=/work/.uv-cache \ + -e UV_PYTHON=/opt/python/cp312-cp312/bin/python3 \ + -v "$ROOT_DIR:/work" \ + -v "$UV_BIN:/usr/local/bin/uv:ro" \ + -w /work \ + "quay.io/pypa/manylinux_2_28_$(uname -m)" \ + bash build.sh + exit 0 +fi + +if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then + export PATH="/opt/python/cp312-cp312/bin:$PATH" + + ./setup.sh + + if [[ -z "${BUILD_SH_REUSE_MANYLINUX_ARTIFACTS:-}" ]]; then + for toml in */pyproject.toml; do + pkg="${toml%/pyproject.toml}" + module="${pkg//-/_}" + rm -rf "$pkg/$module/install" "$pkg/$module/toolchain" "$pkg/$module/bin" + done + fi +fi + +echo "Building workspace packages into dist" +START_SECS=$SECONDS + +uv build --all-packages --wheel --out-dir dist --no-create-gitignore --no-build-logs + +if [[ -n "${BUILD_SH_IN_MANYLINUX:-}" ]]; then + VENV_DIR="$ROOT_DIR/.venv-manylinux" +else + VENV_DIR="$ROOT_DIR/.venv" +fi + +echo +echo "Running smoketests" + +uv venv --allow-existing --quiet "$VENV_DIR" +uv pip install --python "$VENV_DIR/bin/python" --reinstall --no-deps --quiet dist/*.whl >/dev/null + +for toml in */pyproject.toml; do + module="$(basename "$(dirname "$toml")" | tr '-' '_')" + "$VENV_DIR/bin/python" -c "import $module; $module.smoketest()" >/dev/null +done + +du -hs dist/* | sort -hr + +echo +echo "Done in $((SECONDS - START_SECS))s" diff --git a/build_wheels.sh b/build_wheels.sh deleted file mode 100755 index 4950475..0000000 --- a/build_wheels.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$DIR" - -WHEEL_DIR="${1:-dist}" - -# on Linux, build inside manylinux_2_28 container for glibc 2.28 compatibility. -# for reference, Ubuntu 20.04 is glibc 2.31, so this gives us wide compatibility. -if [ "$(uname)" = "Linux" ] && [ -z "${MANYLINUX:-}" ]; then - ARCH="$(uname -m)" - IMAGE="quay.io/pypa/manylinux_2_28_${ARCH}" - exec docker run --rm \ - -e MANYLINUX=1 \ - -v "$DIR:/work" \ - -w /work \ - "$IMAGE" \ - bash build_wheels.sh "$WHEEL_DIR" -fi - -# Set up Python inside manylinux container -if [ -n "${MANYLINUX:-}" ]; then - export PATH="/opt/python/cp312-cp312/bin:$PATH" -fi - -./setup.sh -pip install setuptools wheel - -mkdir -p "$WHEEL_DIR" -for pkg in */pyproject.toml; do - pip wheel "./$(dirname "$pkg")" --no-deps --wheel-dir "$WHEEL_DIR"/ -done diff --git a/capnproto/capnproto/__init__.py b/capnproto/capnproto/__init__.py index 197deb9..b654748 100644 --- a/capnproto/capnproto/__init__.py +++ b/capnproto/capnproto/__init__.py @@ -29,3 +29,16 @@ def _run_capnpc(): def _run_capnpc_cpp(): _run("capnpc-c++") + + +def smoketest(): + import subprocess + + capnp = os.path.join(BIN_DIR, "capnp") + capnpc = os.path.join(BIN_DIR, "capnpc") + capnpc_cpp = os.path.join(BIN_DIR, "capnpc-c++") + env = os.environ.copy() + env["PATH"] = BIN_DIR + ":" + env.get("PATH", "") + subprocess.run([capnp, "--version"], check=True, env=env) + subprocess.run([capnpc, "--version"], check=True, env=env) + subprocess.run([capnpc_cpp, "--version"], check=True, env=env) diff --git a/ffmpeg/ffmpeg/__init__.py b/ffmpeg/ffmpeg/__init__.py index dc1a82d..8411bbc 100644 --- a/ffmpeg/ffmpeg/__init__.py +++ b/ffmpeg/ffmpeg/__init__.py @@ -18,3 +18,12 @@ def _run_ffmpeg(): def _run_ffprobe(): _run("ffprobe") + + +def smoketest(): + import subprocess + + ffmpeg = os.path.join(BIN_DIR, "ffmpeg") + ffprobe = os.path.join(BIN_DIR, "ffprobe") + subprocess.run([ffmpeg, "-version"], check=True) + subprocess.run([ffprobe, "-version"], check=True) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eaea3f3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.uv.workspace] +members = [ + "capnproto", + "cppcheck", + "eigen", + "ffmpeg", + "gcc-arm-none-eabi", + "git-lfs", + "libjpeg", + "ncurses", + "openssl3", + "python3-dev", + "zeromq", + "zstd", +] diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..1efb19c --- /dev/null +++ b/release.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$ROOT_DIR" + +REPO=commaai/dependencies + +echo +echo "Publishing wheels to GitHub Releases ($REPO)" + +shopt -s nullglob +for toml in */pyproject.toml; do + pkg="$(dirname "$toml")" + module="${pkg//-/_}" + version="$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")" + tag="${pkg}/v${version}" + + wheels=("dist/${module}-${version}-"*.whl) + if [[ ${#wheels[@]} -eq 0 ]]; then + echo "missing wheel for $pkg ($module-$version) in dist" >&2 + exit 1 + fi + + echo "[$pkg] Uploading ${#wheels[@]} wheel(s) to $tag" + + gh release create "$tag" "${wheels[@]}" --repo "$REPO" --title "$pkg v$version" --notes "Platform wheels for $pkg $version" 2>/dev/null || + gh release upload "$tag" "${wheels[@]}" --repo "$REPO" --clobber +done +shopt -u nullglob + +TOKEN="$(gh auth token 2>/dev/null)" || { echo "set GH_TOKEN to publish shim branch" >&2; exit 1; } + +TMP_DIR="$(mktemp -d)" +python3 - "$TMP_DIR" "$REPO" <<'PY' +import json +import pathlib +import shutil +import tomllib +import sys + +tmp_dir = pathlib.Path(sys.argv[1]) +repo = sys.argv[2] +repo_url = f"https://github.com/{repo}" +shim_setup = pathlib.Path("_shim_setup.py") + +for toml in sorted(pathlib.Path(".").glob("*/pyproject.toml")): + pkg = toml.parent.name + module = pkg.replace("-", "_") + data = tomllib.load(toml.open("rb")) + version = str(data["project"]["version"]) + tag = f"{pkg}/v{version}" + description = data["project"]["description"] + patterns = data.get("tool", {}).get("setuptools", {}).get("package-data", {}).get(module, [""]) + datadir = patterns[0].split("/", 1)[0] if patterns and patterns[0] else "" + scripts = data.get("project", {}).get("scripts", {}) or {} + + pkg_dir = tmp_dir / pkg + mod_dir = pkg_dir / module + mod_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(pathlib.Path(pkg) / module / "__init__.py", mod_dir / "__init__.py") + shutil.copy2(shim_setup, pkg_dir / "setup.py") + + lines = [ + "[build-system]", + 'requires = ["setuptools>=64", "wheel", \'tomli; python_version < \"3.11\"\']', + 'build-backend = "setuptools.build_meta"', + "", + "[project]", + f'name = "{pkg}"', + f'version = "{version}"', + f"description = {json.dumps(description + ' (pre-built)')}", + 'requires-python = ">=3.8"', + ] + + if scripts: + lines += ["", "[project.scripts]"] + for name, target in scripts.items(): + lines.append(f'"{name}" = {json.dumps(target)}') + + lines += [ + "", + "[tool.setuptools.packages.find]", + f'include = ["{module}*"]', + "", + "[tool.setuptools.package-data]", + f'{module} = ["{datadir}/**/*"]', + "", + "[tool.shim]", + f'repo_url = "{repo_url}"', + f'tag = "{tag}"', + f'datadir = "{datadir}"', + ] + + (pkg_dir / "pyproject.toml").write_text("\n".join(lines) + "\n") +PY + +( + cd "$TMP_DIR" + git init + git checkout -b releases + git add . + git -c user.name="github-actions[bot]" -c user.email="github-actions[bot]@users.noreply.github.com" commit -m "update shim packages" + git remote add origin "https://x-access-token:${TOKEN}@github.com/${REPO}.git" + git push -f origin releases +) + +rm -rf "$TMP_DIR" diff --git a/setup.sh b/setup.sh index 0cc73ef..b9ad031 100755 --- a/setup.sh +++ b/setup.sh @@ -1,10 +1,39 @@ #!/usr/bin/env bash set -euo pipefail +# sets up build-time dependencies (NOT runtime dependencies) + +run_as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif command -v sudo &>/dev/null; then + sudo "$@" + else + echo "error: root privileges required for: $*" >&2 + exit 1 + fi +} + if [ "$(uname)" = "Darwin" ]; then brew install nasm pkg-config elif command -v dnf &>/dev/null; then dnf install -y nasm cmake gcc-c++ pkgconfig git perl-IPC-Cmd elif command -v apt-get &>/dev/null; then - sudo apt-get update && sudo apt-get install -y nasm cmake g++ pkg-config + run_as_root apt-get update + run_as_root apt-get install -y nasm cmake g++ pkg-config curl +fi + +if ! command -v uv &>/dev/null; then + command -v curl &>/dev/null || { + echo "error: curl is required to install uv" >&2 + exit 1 + } + UV_BIN_DIR="$HOME/.local/bin" + mkdir -p "$UV_BIN_DIR" + curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$UV_BIN_DIR" sh + export PATH="$UV_BIN_DIR:$PATH" + command -v uv &>/dev/null || { + echo "error: failed to install uv" >&2 + exit 1 + } fi diff --git a/smoketest.sh b/smoketest.sh deleted file mode 100755 index f209927..0000000 --- a/smoketest.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -WHEEL_DIR="${1:?Usage: smoketest.sh }" -WHEEL_DIR="$(cd "$WHEEL_DIR" && pwd)" - -VENV_DIR="$(mktemp -d)" -trap 'rm -rf "$VENV_DIR"' EXIT -python3 -m venv "$VENV_DIR" -source "$VENV_DIR/bin/activate" -pip install --upgrade pip >/dev/null -pip install "$WHEEL_DIR"/*.whl - -for toml in "$REPO_DIR"/*/pyproject.toml; do - module="$(basename "$(dirname "$toml")" | tr '-' '_')" - python3 -c "import $module; $module.smoketest()" && echo "$module: OK" -done diff --git a/test.sh b/test.sh deleted file mode 100755 index 7eea2c8..0000000 --- a/test.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -set -e - -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - -# Create a temporary virtualenv -VENV_DIR="$(mktemp -d)" -trap 'rm -rf "$VENV_DIR"' EXIT -python3 -m venv "$VENV_DIR" -source "$VENV_DIR/bin/activate" -pip install --upgrade pip >/dev/null - -# Auto-discover packages: any subdirectory containing a pyproject.toml -PACKAGES=() -for toml in "$REPO_DIR"/*/pyproject.toml; do - [ -f "$toml" ] || continue - PACKAGES+=("$(dirname "$toml")") -done - -if [ ${#PACKAGES[@]} -eq 0 ]; then - echo "No packages found." - exit 1 -fi - -FAILED=() - -for pkg in "${PACKAGES[@]}"; do - name="$(basename "$pkg")" - echo "=========================================" - echo "Testing: $name" - echo "=========================================" - - # Install from git URL with subdirectory (simulates real-world usage) - echo "[$name] pip install ..." - pip install "$pkg" --verbose - - # Verify import works - module="${name//-/_}" - echo "[$name] Verifying import of $module ..." - python -c "import $module; print(f'{$module.__name__} OK')" - - echo "[$name] PASSED" - echo -done - -if [ ${#FAILED[@]} -ne 0 ]; then - echo "FAILED packages: ${FAILED[*]}" - exit 1 -fi - -echo "All ${#PACKAGES[@]} package(s) passed." -echo -echo "Installed sizes:" -for pkg in "${PACKAGES[@]}"; do - name="$(basename "$pkg")" - module="${name//-/_}" - mod_dir="$(python -c "import $module, os; print(os.path.dirname($module.__file__))")" - size="$(du -sh "$mod_dir" | cut -f1)" - echo " $name: $size" -done diff --git a/test_wheels_in_image.sh b/test_wheels_in_image.sh new file mode 100755 index 0000000..995aaac --- /dev/null +++ b/test_wheels_in_image.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +# a small script for testing our built wheels in a variety of linux distros + +IMAGE="${1:?usage: ./test_wheels_in_image.sh }" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$ROOT_DIR" + +# setup our deps - should only need a python3 build +docker build -t wheeltest -f - . </dev/null; then \ + apk add --no-cache python3 py3-pip bash; \ + elif command -v apt-get >/dev/null; then \ + apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \ + elif command -v dnf >/dev/null; then \ + dnf install -y python3 python3-pip; \ + elif command -v pacman >/dev/null; then \ + pacman -Sy --noconfirm python python-pip; \ + elif command -v zypper >/dev/null; then \ + zypper install -y python3 python3-pip; \ + elif command -v xbps-install >/dev/null; then \ + xbps-install -Sy python3 python3-pip bash; \ + fi +DOCKERFILE + +# install + smoketest +docker run --rm -v "$PWD:/work" -w /work wheeltest bash -lc ' + set -euo pipefail + python3 -m venv /tmp/venv + source /tmp/venv/bin/activate + pip install dist/*.whl + for toml in */pyproject.toml; do + module="$(basename "$(dirname "$toml")" | tr "-" "_")" + python -c "import $module; $module.smoketest()" && echo "$module: OK" + done +' + +echo +echo "All good!"