simplify building (#26)

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  build:
+    name: build (${{ matrix.platform }})
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux-x86_64
+            runs-on: ubuntu-latest
+          - platform: linux-arm64
+            runs-on: ubuntu-24.04-arm
+          - platform: macos
+            runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/cache@v4
+        with:
+          key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }}
+          restore-keys: build-cache-v1-${{ matrix.platform }}-
+          path: |
+            .uv-cache
+            */*/install
+            */*/toolchain
+            */*/bin
+      - name: Build wheels
+        env:
+          MANYLINUX: ${{ runner.os == 'Linux' && '1' || '0' }}
+          UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
+          BUILD_SH_REUSE_MANYLINUX_ARTIFACTS: "1"
+        run: ./build.sh
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.platform }}
+          path: dist/*.whl
+
+  test:
+    name: test distro (${{ matrix.image }})
+    needs: build
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: debian:bookworm-slim
+            runs-on: ubuntu-latest
+          - image: ubuntu:24.04
+            runs-on: ubuntu-latest
+          - image: ubuntu:20.04
+            runs-on: ubuntu-latest
+          - image: fedora:41
+            runs-on: ubuntu-latest
+          - image: archlinux:latest
+            runs-on: ubuntu-latest
+          - image: opensuse/tumbleweed:latest
+            runs-on: ubuntu-latest
+          - image: ghcr.io/void-linux/void-glibc:latest
+            runs-on: ubuntu-latest
+          - image: debian:bookworm-slim
+            runs-on: ubuntu-24.04-arm
+          # TODO: add musl test
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ contains(matrix.runs-on, 'arm') && 'wheels-linux-arm64' || 'wheels-linux-x86_64' }}
+          path: dist/
+      - run: ./test_wheels_in_image.sh "${{ matrix.image }}"
+
+  publish:
+    name: publish wheels
+    if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+    needs: [build, test]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          merge-multiple: true
+          path: dist
+      - uses: actions/cache/restore@v4
+        with:
+          key: build-cache-v1-${{ matrix.platform }}-${{ hashFiles('build.sh', 'setup.sh', '*/pyproject.toml', '*/setup.py', '*/build.sh') }}
+          restore-keys: build-cache-v1-${{ matrix.platform }}-
+          path: |
+            .uv-cache
+            */*/install
+            */*/toolchain
+            */*/bin
+      - name: Publish wheels and shims
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./release.sh --publish-only

.github/workflows/release.yml

@@ -1,217 +0,0 @@
-name: release wheels
-
-on:
-  push:
-    branches: [master]
-
-jobs:
-  build:
-    name: build - ${{ matrix.platform }}
-    runs-on: ${{ matrix.runs-on }}
-    permissions:
-      contents: write
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: linux-x86_64
-            runs-on: ubuntu-latest
-          - platform: linux-arm64
-            runs-on: ubuntu-24.04-arm
-          - platform: macos
-            runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v6
-
-      - uses: actions/setup-python@v6
-        if: runner.os != 'Linux'
-        with:
-          python-version: "3.14"
-
-      - name: Build wheels
-        run: ./build_wheels.sh dist/
-
-      - name: Upload releases
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          for toml in */pyproject.toml; do
-            [ -f "$toml" ] || continue
-            pkg="$(dirname "$toml")"
-            module="${pkg//-/_}"
-            version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
-            tag="${pkg}/v${version}"
-
-            echo "========================================="
-            echo "Uploading: $pkg v$version"
-            echo "========================================="
-
-            whl=$(ls dist/${module}-${version}-*.whl 2>/dev/null) || continue
-            echo "Wheel: $whl"
-            if gh release view "$tag" --repo "$GITHUB_REPOSITORY" &>/dev/null; then
-              gh release upload "$tag" "$whl" --repo "$GITHUB_REPOSITORY" --clobber
-            else
-              gh release create "$tag" "$whl" \
-                --repo "$GITHUB_REPOSITORY" \
-                --title "$pkg v$version" \
-                --notes "Platform wheels for $pkg $version"
-            fi
-          done
-
-  publish-shims:
-    name: publish releases branch
-    needs: build
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v6
-
-      - name: build and push shim packages
-        run: |
-          REPO_URL="https://github.com/${GITHUB_REPOSITORY}"
-
-          # set up a temp dir for the releases branch
-          tmp="$(mktemp -d)"
-          trap 'rm -rf "$tmp"' EXIT
-
-          for toml in */pyproject.toml; do
-            [ -f "$toml" ] || continue
-            pkg="$(dirname "$toml")"
-            version=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['version'])")
-            tag="${pkg}/v${version}"
-            module="${pkg//-/_}"
-            # detect the package-data directory (e.g. "toolchain" or "install")
-            datadir=$(python3 -c "
-          import tomllib
-          t = tomllib.load(open('$toml', 'rb'))
-          pat = t.get('tool', {}).get('setuptools', {}).get('package-data', {}).get('$module', [''])[0]
-          print(pat.split('/')[0])
-          ")
-            description=$(python3 -c "import tomllib; print(tomllib.load(open('$toml', 'rb'))['project']['description'])")
-
-            echo "========================================="
-            echo "Creating shim: $pkg v$version (datadir=$datadir)"
-            echo "========================================="
-
-            pkg_dir="$tmp/$pkg"
-            mod_dir="$pkg_dir/$module"
-            mkdir -p "$mod_dir"
-
-            # copy __init__.py from source
-            cp "$pkg/$module/__init__.py" "$mod_dir/"
-
-            cat > "$pkg_dir/pyproject.toml" << TOML
-          [build-system]
-          requires = ["setuptools>=64", "wheel"]
-          build-backend = "setuptools.build_meta"
-
-          [project]
-          name = "$pkg"
-          version = "$version"
-          description = "$description (pre-built)"
-          requires-python = ">=3.8"
-
-          [project.scripts]
-          $(grep -A100 '^\[project\.scripts\]' "$pkg/pyproject.toml" | tail -n +2 | sed '/^\[/q' | grep -v '^\[')
-
-          [tool.setuptools.packages.find]
-          include = ["${module}*"]
-
-          [tool.setuptools.package-data]
-          $module = ["${datadir}/**/*"]
-          TOML
-
-            # setup.py — downloads pre-built wheel from GH releases and extracts data
-            cat > "$pkg_dir/setup.py" << 'SETUP'
-          import os
-          import platform
-          import zipfile
-          from io import BytesIO
-          from urllib.request import urlopen
-
-          from setuptools.command.build_py import build_py
-
-          REPO_URL = "REPO_URL_PLACEHOLDER"
-          TAG = "TAG_PLACEHOLDER"
-          VERSION = "VERSION_PLACEHOLDER"
-          MODULE = "MODULE_PLACEHOLDER"
-          DATADIR = "DATADIR_PLACEHOLDER"
-
-          PLATFORM_MAP = {
-            ("Linux", "x86_64"): "linux_x86_64",
-            ("Linux", "aarch64"): "linux_aarch64",
-            ("Darwin", "arm64"): "macosx_11_0_arm64",
-          }
-
-
-          class InstallPrebuilt(build_py):
-            """Download pre-built wheel from GitHub Releases and extract binaries."""
-
-            def run(self):
-              pkg_dir = os.path.dirname(os.path.abspath(__file__))
-              data_dir = os.path.join(pkg_dir, MODULE, DATADIR)
-
-              if not os.path.exists(os.path.join(data_dir, "bin")):
-                key = (platform.system(), platform.machine())
-                plat = PLATFORM_MAP.get(key)
-                if plat is None:
-                  raise RuntimeError(f"unsupported platform: {key}")
-
-                whl_name = f"{MODULE}-{VERSION}-py3-none-{plat}.whl"
-                url = f"{REPO_URL}/releases/download/{TAG}/{whl_name}"
-
-                print(f"Downloading {url} ...")
-                data = urlopen(url).read()
-
-                print(f"Extracting {DATADIR} ...")
-                with zipfile.ZipFile(BytesIO(data)) as zf:
-                  prefix = f"{MODULE}/{DATADIR}/"
-                  alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/"
-                  for info in zf.infolist():
-                    for p in (prefix, alt_prefix):
-                      if info.filename.startswith(p):
-                        rel = info.filename[len(p):]
-                        if not rel:
-                          continue
-                        dest = os.path.join(data_dir, rel)
-                        if info.is_dir():
-                          os.makedirs(dest, exist_ok=True)
-                        else:
-                          os.makedirs(os.path.dirname(dest), exist_ok=True)
-                          with open(dest, "wb") as f:
-                            f.write(zf.read(info))
-                          if info.external_attr >> 16 & 0o111:
-                            os.chmod(dest, 0o755)
-                        break
-
-              super().run()
-
-
-          def setup():
-            from setuptools import setup as _setup
-            _setup(cmdclass={"build_py": InstallPrebuilt})
-
-
-          if __name__ == "__main__":
-            setup()
-          SETUP
-
-            # template in the actual values
-            sed -i "s|REPO_URL_PLACEHOLDER|${REPO_URL}|" "$pkg_dir/setup.py"
-            sed -i "s|TAG_PLACEHOLDER|${tag}|" "$pkg_dir/setup.py"
-            sed -i "s|VERSION_PLACEHOLDER|${version}|" "$pkg_dir/setup.py"
-            sed -i "s|MODULE_PLACEHOLDER|${module}|" "$pkg_dir/setup.py"
-            sed -i "s|DATADIR_PLACEHOLDER|${datadir}|" "$pkg_dir/setup.py"
-          done
-
-          # push to releases branch
-          cd "$tmp"
-          git init
-          git checkout -b releases
-          git add .
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git commit -m "update shim packages"
-          git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${GITHUB_REPOSITORY}.git"
-          git push -f origin releases

.github/workflows/test.yml

@@ -1,90 +0,0 @@
-name: ci
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-
-jobs:
-  build:
-    name: build (${{ matrix.platform }})
-    runs-on: ${{ matrix.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - platform: macos
-            runs-on: macos-latest
-          - platform: linux-x86_64
-            runs-on: ubuntu-latest
-          - platform: linux-arm64
-            runs-on: ubuntu-24.04-arm
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/setup-python@v6
-        if: runner.os != 'Linux'
-        with:
-          python-version: "3.14"
-      - name: Build wheels
-        run: ./build_wheels.sh dist/
-      - name: Smoketest
-        run: bash smoketest.sh dist/
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.platform }}
-          path: dist/*.whl
-
-  test:
-    name: test (${{ matrix.image }})
-    needs: build
-    runs-on: ${{ matrix.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          # test on vanilla linux images
-          - image: debian:bookworm-slim
-            runs-on: ubuntu-latest
-          - image: ubuntu:24.04
-            runs-on: ubuntu-latest
-          - image: ubuntu:20.04
-            runs-on: ubuntu-latest
-          - image: fedora:41
-            runs-on: ubuntu-latest
-          - image: archlinux:latest
-            runs-on: ubuntu-latest
-          # TODO: make musl work
-          #- image: alpine:3.21
-          #  runs-on: ubuntu-latest
-          - image: opensuse/tumbleweed:latest
-            runs-on: ubuntu-latest
-          - image: ghcr.io/void-linux/void-glibc:latest
-            runs-on: ubuntu-latest
-          - image: debian:bookworm-slim
-            runs-on: ubuntu-24.04-arm
-    steps:
-      - uses: actions/checkout@v6
-      - uses: actions/download-artifact@v4
-        with:
-          name: wheels-${{ contains(matrix.runs-on, 'arm') && 'linux-arm64' || 'linux-x86_64' }}
-          path: wheels
-      - name: build docker
-        run: |
-          docker build -t smoketest -f - . <<'DOCKERFILE'
-          FROM ${{ matrix.image }}
-          RUN if command -v apk >/dev/null; then \
-                apk add --no-cache python3 py3-pip bash; \
-              elif command -v apt-get >/dev/null; then \
-                apt-get update && apt-get install -y --no-install-recommends python3 python3-pip python3-venv; \
-              elif command -v dnf >/dev/null; then \
-                dnf install -y python3 python3-pip; \
-              elif command -v pacman >/dev/null; then \
-                pacman -Sy --noconfirm python python-pip; \
-              elif command -v zypper >/dev/null; then \
-                zypper install -y python3 python3-pip; \
-              elif command -v xbps-install >/dev/null; then \
-                xbps-install -Sy python3 python3-pip bash; \
-              fi
-          DOCKERFILE
-      - name: ./smoketest.sh
-        run: docker run --rm -v "$PWD:/work" -w /work smoketest bash smoketest.sh wheels/