diff --git a/.gitignore b/.gitignore index bdc037d..6d0cd60 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,9 @@ git-lfs/git_lfs/bin/ # downloaded mdbook mdbook/mdbook/bin/ + +# vendored at build time +acados/acados/acados_template/ +acados/casadi/ +acados/casadi-venv/ +acados/casadi-wheel/ diff --git a/_shim_setup.py b/_shim_setup.py index 488aea5..8b58b1c 100644 --- a/_shim_setup.py +++ b/_shim_setup.py @@ -25,6 +25,17 @@ DATADIR = _cfg["tool"]["shim"]["datadir"] VERSION = _cfg["project"]["version"] MODULE = _cfg["project"]["name"].replace("-", "_") +# all top-level packages bundled in this wheel (e.g. acados + casadi). +# `packages` may be a flat list or a dict with find.include patterns. +_pkgs = _cfg.get("tool", {}).get("setuptools", {}).get("packages", [f"{MODULE}*"]) +if isinstance(_pkgs, list): + _INCLUDE = _pkgs +elif isinstance(_pkgs, dict): + _INCLUDE = _pkgs.get("find", {}).get("include", [f"{MODULE}*"]) +else: + _INCLUDE = [f"{MODULE}*"] +TOP_PACKAGES = sorted({p.rstrip("*").rstrip("/") for p in _INCLUDE if p.rstrip("*").rstrip("/")}) + PLATFORM_MAP = { ("Linux", "x86_64"): "linux_x86_64", ("Linux", "aarch64"): "linux_aarch64", @@ -67,44 +78,28 @@ class InstallPrebuilt(build_py): if raw is not None: break - print(f"Extracting {DATADIR} ...") + print("Extracting wheel ...") with zipfile.ZipFile(BytesIO(raw)) as zf: - prefix = f"{MODULE}/{DATADIR}/" - alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/{DATADIR}/" + purelib_data = f"{MODULE}-{VERSION}.data/purelib/" for info in zf.infolist(): - for p in (prefix, alt_prefix): - if info.filename.startswith(p): - rel = info.filename[len(p):] - if not rel: - continue - dest = os.path.join(data_dir, rel) - if info.is_dir(): - os.makedirs(dest, exist_ok=True) - else: - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, "wb") as f: - f.write(zf.read(info)) - if info.external_attr >> 16 & 0o111: - os.chmod(dest, 0o755) - break - - # Also extract compiled extension modules (.so) - ext_prefix = f"{MODULE}/" - ext_alt_prefix = f"{MODULE}-{VERSION}.data/purelib/{MODULE}/" - for info in zf.infolist(): - if not info.filename.endswith('.so'): + if info.is_dir(): continue - for p in (ext_prefix, ext_alt_prefix): - if info.filename.startswith(p): - rel = info.filename[len(p):] - if rel and '/' not in rel: - dest = os.path.join(module_dir, rel) - os.makedirs(os.path.dirname(dest), exist_ok=True) - with open(dest, "wb") as f: - f.write(zf.read(info)) - if info.external_attr >> 16 & 0o111: - os.chmod(dest, 0o755) - break + name = info.filename + if name.startswith(purelib_data): + rel = name[len(purelib_data):] + else: + rel = name + if "/" not in rel: + continue + top = rel.split("/", 1)[0] + if top not in TOP_PACKAGES: + continue + dest = os.path.join(_HERE, rel) + os.makedirs(os.path.dirname(dest), exist_ok=True) + with open(dest, "wb") as f: + f.write(zf.read(info)) + if info.external_attr >> 16 & 0o111: + os.chmod(dest, 0o755) super().run() diff --git a/acados/acados/__init__.py b/acados/acados/__init__.py new file mode 100644 index 0000000..2c84aef --- /dev/null +++ b/acados/acados/__init__.py @@ -0,0 +1,48 @@ +import os + +DIR = os.path.join(os.path.dirname(__file__), "install") +INCLUDE_DIR = os.path.join(DIR, "include") +LIB_DIR = os.path.join(DIR, "lib") +BIN_DIR = os.path.join(DIR, "bin") +TERA_PATH = os.path.join(BIN_DIR, "t_renderer") + +TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "acados_template") + + +def smoketest(): + import sys + + lib_ext = ".dylib" if sys.platform == "darwin" else ".so" + for lib in ("libacados", "libblasfeo", "libhpipm", "libqpOASES_e"): + path = os.path.join(LIB_DIR, lib + lib_ext) + assert os.path.isfile(path), f"missing lib: {path}" + + assert os.path.isfile(os.path.join(INCLUDE_DIR, "acados_c", "ocp_nlp_interface.h")) + assert os.path.isfile(os.path.join(INCLUDE_DIR, "blasfeo", "include", "blasfeo.h")) + assert os.path.isfile(os.path.join(INCLUDE_DIR, "hpipm", "include", "hpipm_common.h")) + + assert os.path.isfile(TERA_PATH) and os.access(TERA_PATH, os.X_OK), f"t_renderer missing/not executable: {TERA_PATH}" + + assert os.path.isfile(os.path.join(TEMPLATE_DIR, "__init__.py")) + assert os.path.isfile(os.path.join(TEMPLATE_DIR, "acados_layout.json")) + assert os.path.isdir(os.path.join(TEMPLATE_DIR, "c_templates_tera")) + + # the vendored slim casadi shipped in this wheel is built against cpython 3.12 + # and pulls in numpy; only exercise it when both are available (real consumers + # like openpilot install via the shim, which declares numpy as a dep). + try: + import numpy # noqa: F401 + except ImportError: + return + if sys.version_info[:2] != (3, 12): + return + + from casadi import SX, MX, DM, Function, CasadiMeta, vertcat, jacobian, sin, cos, n_nodes # noqa: F401 + + x = SX.sym("x") + y = SX.sym("y") + expr = vertcat(sin(x), cos(y)) + J = jacobian(expr, vertcat(x, y)) + assert J.shape == (2, 2) + assert n_nodes(J) > 0 + assert isinstance(CasadiMeta.version(), str) diff --git a/acados/build.sh b/acados/build.sh new file mode 100755 index 0000000..bbae346 --- /dev/null +++ b/acados/build.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd "$DIR" + +# v0.2.2 +VERSION="8af9b0ad180940ef611884574a0b27a43504311d" +INSTALL_DIR="$DIR/acados/install" +TEMPLATE_DIR="$DIR/acados/acados_template" +CASADI_DIR="$DIR/casadi" +CASADI_VERSION="3.6.7" + +NJOBS="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" + +# pick BLAS target per host arch +ARCH="$(uname -m)" +if [[ "$OSTYPE" == "darwin"* ]]; then + # this BLASFEO version doesn't have an Apple Silicon target; Cortex-A57 + # baseline ARMv8 SIMD compiles and runs fine on M1+. + BLAS_TARGET="ARMV8A_ARM_CORTEX_A57" +elif [[ "$ARCH" == "aarch64" ]]; then + # Cortex-A57 = TICI baseline; safe for any modern aarch64 + BLAS_TARGET="ARMV8A_ARM_CORTEX_A57" +else + BLAS_TARGET="X64_AUTOMATIC" +fi + +ACADOS_FLAGS=( + -DACADOS_WITH_QPOASES=ON + -UBLASFEO_TARGET + -DBLASFEO_TARGET="$BLAS_TARGET" + -DACADOS_INSTALL_DIR="$INSTALL_DIR" + # acados (and several of its submodules) still pin cmake_minimum_required <3.5; + # CMake 4 removed that compatibility, so re-enable it here. + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + # qpOASES + blasfeo both call malloc()/posix_memalign() without including + # . C99/C2x and modern gcc/clang reject implicit declarations as + # errors. qpOASES also has a real Constraints*/Constraints** pointer bug + # that gcc 14+ now flags as an error too. Downgrade both so the upstream + # (pinned) sources keep compiling. + "-DCMAKE_C_FLAGS=-Wno-implicit-function-declaration -Wno-incompatible-pointer-types" +) +if [[ "$OSTYPE" == "darwin"* ]]; then + ACADOS_FLAGS+=( + -DCMAKE_OSX_ARCHITECTURES=arm64 + -DCMAKE_MACOSX_RPATH=1 + ) +fi + +# clone/update source +if [ ! -d "acados-src/.git" ]; then + rm -rf acados-src + git clone https://github.com/acados/acados.git acados-src +fi +git -C acados-src fetch --all --tags +git -C acados-src checkout --force "$VERSION" +git -C acados-src submodule update --init --recursive --depth=1 + +# build acados +mkdir -p build +cd build +cmake "${ACADOS_FLAGS[@]}" "$DIR/acados-src" +make -j"$NJOBS" install +cd "$DIR" + +# we don't ship sample json templates +rm -f "$INSTALL_DIR"/lib/*.json + +# python interface package (acados_template) +rm -rf "$TEMPLATE_DIR" +cp -r acados-src/interfaces/acados_template/acados_template "$TEMPLATE_DIR" + +# strip future_fstrings (avoids needing the compatibility package on py>=3.6). +# Cython chokes on the unknown encoding in .pyx/.pxd too, not just .py. +find "$TEMPLATE_DIR" -type f \( -name '*.py' -o -name '*.pyx' -o -name '*.pxd' \) \ + -exec sed -i.bak '/future.fstrings/d' {} + +find "$TEMPLATE_DIR" -name '*.bak' -delete + +# acados_template's gnsf/check_reformulation.py uses an absolute `from +# acados_template.utils import ...` that only worked when acados_template was +# itself a top-level package on the path. We ship it as `acados.acados_template`, +# so rewrite to the relative form the rest of gnsf already uses. +if [ -f "$TEMPLATE_DIR/gnsf/check_reformulation.py" ]; then + sed -i.bak 's/^from acados_template\.utils /from ..utils /' "$TEMPLATE_DIR/gnsf/check_reformulation.py" + rm -f "$TEMPLATE_DIR/gnsf/check_reformulation.py.bak" +fi + +# build tera renderer (needs cargo) +if ! command -v cargo >/dev/null 2>&1; then + echo "installing rust toolchain (needed for tera_renderer)..." + curl -LsSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable --profile minimal + # shellcheck disable=SC1091 + source "$HOME/.cargo/env" +fi + +mkdir -p "$INSTALL_DIR/bin" +cd "$DIR/acados-src/interfaces/acados_template/tera_renderer/" +if [[ "$OSTYPE" == "darwin"* ]]; then + cargo build --release --target aarch64-apple-darwin + cp target/aarch64-apple-darwin/release/t_renderer "$INSTALL_DIR/bin/t_renderer" +else + cargo build --release + cp target/release/t_renderer "$INSTALL_DIR/bin/t_renderer" +fi + +cd "$DIR" + +# vendor a slim casadi: install the upstream wheel into a throwaway cp312 venv +# (uv resolves the right platform wheel automatically), then move the casadi/ +# tree out and slim it. +echo "vendoring casadi $CASADI_VERSION ..." +rm -rf "$CASADI_DIR" casadi-venv +uv venv --python 3.12 --quiet casadi-venv +uv pip install --python casadi-venv/bin/python --no-deps --quiet "casadi==$CASADI_VERSION" +mv casadi-venv/lib/python3.12/site-packages/casadi "$CASADI_DIR" +rm -rf casadi-venv + +# drop everything except the bits openpilot actually needs: +# - __init__.py, casadi.py, tools/ (Python wrapper) +# - _casadi.so (CPython extension; same name on linux+darwin) +# - libcasadi.{so,dylib}* (the C++ runtime that _casadi.so links to) +# openpilot only uses symbolic SX/MX/Function/jacobian etc., which live in +# libcasadi + _casadi. Solver plugins (conic_*, nlpsol_*, integrator_*, ...) +# and their third-party backends (ipopt, bonmin, hpipm, fatrop, ...) are +# loaded lazily via dlopen and never reached. +cd "$CASADI_DIR" +shopt -s extglob +# libc++.*.dylib only exists on darwin and is needed by _casadi.so via @rpath +rm -rf !(__init__.py|casadi.py|_casadi.so|tools|libcasadi.*|libc++.*) +shopt -u extglob + +cd "$DIR" +rm -rf casadi-wheel + +echo "Installed acados to $INSTALL_DIR" +du -sh "$INSTALL_DIR" +echo "Vendored casadi (slim) at $CASADI_DIR" +du -sh "$CASADI_DIR" diff --git a/acados/pyproject.toml b/acados/pyproject.toml new file mode 100644 index 0000000..4625284 --- /dev/null +++ b/acados/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "acados" +version = "0.2.2" +description = "acados solvers + Python interface, with a slimmed casadi vendored alongside" +requires-python = ">=3.8" +dependencies = [ + # the vendored casadi.py shim imports numpy at module load time + "numpy", +] + +# `find` runs at setup() init time, before build.sh creates these dirs, so +# enumerate packages explicitly. acados.acados_template + casadi.tools are +# subpackages and ship via package-data. +[tool.setuptools] +packages = ["acados", "casadi"] + +[tool.setuptools.package-data] +acados = ["install/**/*", "acados_template/**/*"] +casadi = ["**/*"] diff --git a/acados/setup.py b/acados/setup.py new file mode 100644 index 0000000..0dcbc2f --- /dev/null +++ b/acados/setup.py @@ -0,0 +1,68 @@ +import os +import platform +import subprocess + +from setuptools.command.build_py import build_py + +# `casadi/` is generated by build.sh (vendored from the upstream wheel), but +# setuptools requires every declared package to have a directory + __init__.py +# *before* any command runs. Drop a stub if it isn't there yet — build.sh +# replaces it with the real slim casadi during build_py. +_HERE = os.path.dirname(os.path.abspath(__file__)) +_casadi_init = os.path.join(_HERE, "casadi", "__init__.py") +if not os.path.exists(_casadi_init): + os.makedirs(os.path.dirname(_casadi_init), exist_ok=True) + open(_casadi_init, "w").close() + +try: + from wheel.bdist_wheel import bdist_wheel +except ImportError: + bdist_wheel = None + + +class BuildAcados(build_py): + """Run build.sh to compile acados before collecting package data.""" + + def run(self): + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + build_script = os.path.join(pkg_dir, "build.sh") + subprocess.check_call(["bash", build_script], cwd=pkg_dir) + + super().run() + + +cmdclass = {"build_py": BuildAcados} + +if bdist_wheel is not None: + + class PlatformWheel(bdist_wheel): + """Produce a platform-specific, Python-version-agnostic wheel.""" + + def finalize_options(self): + super().finalize_options() + self.root_is_pure = False + + def get_tag(self): + system = platform.system() + machine = platform.machine() + + if system == "Linux": + plat = f"linux_{machine}" + elif system == "Darwin": + plat = "macosx_11_0_arm64" + else: + plat = f"{system.lower()}_{machine}" + + return "py3", "none", plat + + cmdclass["bdist_wheel"] = PlatformWheel + + +def setup(): + from setuptools import setup as _setup + + _setup(cmdclass=cmdclass) + + +if __name__ == "__main__": + setup() diff --git a/pyproject.toml b/pyproject.toml index 575c582..e55a2f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,6 @@ [tool.uv.workspace] members = [ + "acados", "bzip2", "capnproto", "catch2", diff --git a/release.sh b/release.sh index 75db728..ad08fce 100755 --- a/release.sh +++ b/release.sh @@ -65,18 +65,37 @@ for toml in sorted(pathlib.Path(".").glob("*/pyproject.toml")): for py_file in src_mod.glob("*.py"): shutil.copy2(py_file, mod_dir / py_file.name) - # copy extra packages (e.g. pyray for raylib) - pkgs_val = data.get("tool", {}).get("setuptools", {}).get("packages", {}) - include_patterns = pkgs_val.get("find", {}).get("include", []) if isinstance(pkgs_val, dict) else [] + # copy extra packages (e.g. pyray for raylib, slim casadi for acados) + # binaries (.so/.dylib) are fetched from the wheel at install time, so the + # shim repo only carries Python sources to keep it small + def _ignore_binaries(_dir, names): + return [n for n in names if n.endswith((".so", ".dylib", ".a")) or ".so." in n or n == "__pycache__"] + + # `packages` is either a flat list (["acados", "casadi"]) or a dict with + # `find.include` patterns (["acados*", "casadi*"]) — handle both + pkgs_val = data.get("tool", {}).get("setuptools", {}).get("packages", []) + if isinstance(pkgs_val, list): + include_patterns = list(pkgs_val) + elif isinstance(pkgs_val, dict): + include_patterns = pkgs_val.get("find", {}).get("include", []) + else: + include_patterns = [] extra_packages = [] for pattern in include_patterns: p = pattern.rstrip("*") if p and p != module and p != f"{module}/": src_extra = pathlib.Path(pkg) / p + dst_extra = pkg_dir / p if src_extra.is_dir(): - dst_extra = pkg_dir / p - shutil.copytree(src_extra, dst_extra, dirs_exist_ok=True) - extra_packages.append(pattern) + shutil.copytree(src_extra, dst_extra, dirs_exist_ok=True, ignore=_ignore_binaries) + else: + # source not in the workspace (typically built lazily by build.sh and + # not cached into the publish job). Drop a placeholder __init__.py so + # setuptools.packages.find picks it up; the shim's setup.py overwrites + # it from the wheel at install time. + dst_extra.mkdir(parents=True, exist_ok=True) + (dst_extra / "__init__.py").write_text("") + extra_packages.append(pattern) shutil.copy2(shim_setup, pkg_dir / "setup.py") @@ -102,14 +121,28 @@ for toml in sorted(pathlib.Path(".").glob("*/pyproject.toml")): for name, target in scripts.items(): lines.append(f'"{name}" = {json.dumps(target)}') - find_include = [f"{module}*"] + extra_packages + # propagate the workspace's package-data so the shim wheel knows to ship + # everything the upstream wheel ships (e.g. acados_template/, casadi/*.so). + workspace_pkgdata = data.get("tool", {}).get("setuptools", {}).get("package-data", {}) + + # extra_packages may be plain names (["casadi"]) or globs (["casadi*"]). + # Normalise to plain package names for both `packages` and `package-data`. + extra_pkg_names = [p.rstrip("*").rstrip("/") for p in extra_packages] + lines += [ "", - "[tool.setuptools.packages.find]", - f"include = {json.dumps(find_include)}", + "[tool.setuptools]", + f"packages = {json.dumps([module] + extra_pkg_names)}", "", "[tool.setuptools.package-data]", - f'{module} = ["{datadir}/**/*", "*.so"]', + ] + module_data = sorted(set(workspace_pkgdata.get(module, [f"{datadir}/**/*"]) + ["*.so"])) + lines.append(f"{module} = {json.dumps(module_data)}") + for name in extra_pkg_names: + extra_data = workspace_pkgdata.get(name, ["**/*"]) + lines.append(f"{name} = {json.dumps(list(extra_data))}") + + lines += [ "", "[tool.shim]", f'repo_url = "{repo_url}"',