IQ.Pilot/iqpilot/system/manager/build.py

#!/usr/bin/env python3
import os
import subprocess
import sys
from pathlib import Path

# NOTE: Do NOT import anything here that needs be built (e.g. params)
from iqpilot.common.basedir import BASEDIR
from iqpilot.common.spinner import Spinner
from iqpilot.common.text_window import TextWindow
from iqpilot.common.swaglog import cloudlog, add_file_handler
from iqpilot.system.hardware import HARDWARE, AGNOS
from iqpilot.system.version import get_build_metadata

MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")

TOTAL_SCONS_NODES = 5500
MAX_BUILD_PROGRESS = 100

def get_job_sequence() -> list[int]:
  env_override = os.environ.get("SCONS_MAX_JOBS")
  if env_override is not None:
    try:
      max_jobs = max(1, int(env_override))
    except ValueError:
      max_jobs = 1
  else:
    detected_jobs = os.cpu_count() or 2
    max_jobs = min(detected_jobs, 3 if AGNOS else detected_jobs)

  candidates = [max_jobs, max_jobs // 2, 1]
  jobs: list[int] = []
  for candidate in candidates:
    candidate = max(1, int(candidate))
    if candidate not in jobs:
      jobs.append(candidate)
  return jobs

class _SilentProgress:
  """Not a UI element: a do-nothing sink for build()'s progress calls, used when
  updated.py builds in the background at update-apply time so that NO spinner or
  window is shown. The normal boot path still uses the real Spinner."""
  def update(self, spinner_text: str) -> None:
    pass

  def update_progress(self, cur: float, total: float) -> None:
    pass

  def close(self) -> None:
    pass


REGISTRY_ARTIFACTS = [
  "iqpilot/cereal/services.h",
  "iqpilot/cereal/messaging/socketmaster.o",
  "iqpilot/cereal/libsocketmaster.a",
  "iqpilot/cereal/messaging/bridge",
  "iqpilot/selfdrive/iqlocd/iqlocd",
  "iqpilot/selfdrive/pandad/pandad",
  "iqpilot/system/camerad/camerad",
  "iqpilot/system/loggerd/loggerd",
  "iqpilot/system/loggerd/encoderd",
  "iqpilot/system/loggerd/bootlog",
]


def stale_registry_artifacts(basedir: str = BASEDIR) -> list[str]:
  from iqpilot.cereal.services import REGISTRY_TAG_PREFIX, registry_tag
  expected = registry_tag().encode()
  prefix = REGISTRY_TAG_PREFIX.encode()
  stale = []
  for rel in REGISTRY_ARTIFACTS:
    path = os.path.join(basedir, rel)
    if not os.path.isfile(path):
      continue
    with open(path, "rb") as f:
      data = f.read()
    if prefix in data and expected not in data:
      stale.append(rel)
  return stale


def purge_stale_registry_header(basedir: str = BASEDIR) -> bool:
  from iqpilot.cereal.services import registry_tag
  header = os.path.join(basedir, REGISTRY_ARTIFACTS[0])
  if not os.path.isfile(header):
    return False
  with open(header, "rb") as f:
    stamped = registry_tag().encode() in f.read()
  if stamped:
    return False
  purge_registry_artifacts([], basedir)
  return True


def purge_registry_artifacts(stale: list[str], basedir: str = BASEDIR) -> None:
  for rel in set(stale) | set(REGISTRY_ARTIFACTS[:3]):
    path = os.path.join(basedir, rel)
    if os.path.isfile(path):
      os.remove(path)


def build(spinner, dirty: bool = False, minimal: bool = False, show_error_window: bool = True, registry_retry: bool = False) -> None:
  env = os.environ.copy()
  env.pop('PWD', None)
  env['SCONS_PROGRESS'] = "1"

  extra_args = ["--minimal"] if minimal else []

  if AGNOS:
    HARDWARE.set_power_save(False)
    os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores

  # building with all cores can result in using too
  # much memory, so retry with less parallelism
  if purge_stale_registry_header():
    cloudlog.error("generated service registry header was stale, regenerating")

  compile_output: list[bytes] = []
  for n in get_job_sequence():
    compile_output.clear()
    command = [sys.executable, "-m", "SCons", f"-j{int(n)}", "--cache-populate", *extra_args]
    scons: subprocess.Popen = subprocess.Popen(command, cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
    assert scons.stderr is not None

    # Read progress from stderr and update spinner
    while scons.poll() is None:
      try:
        line = scons.stderr.readline()
        if line is None:
          continue
        line = line.rstrip()

        prefix = b'progress: '
        if line.startswith(prefix):
          i = int(line[len(prefix):])
          spinner.update_progress(MAX_BUILD_PROGRESS * min(0.99, i / TOTAL_SCONS_NODES), 100.)
        elif len(line):
          compile_output.append(line)
          print(line.decode('utf8', 'replace'))
      except Exception:
        pass

    if scons.returncode == 0:
      spinner.update_progress(100, 100.)
      break

  if scons.returncode != 0:
    # Read remaining output
    if scons.stderr is not None:
      compile_output += scons.stderr.read().split(b'\n')

    # Build failed log errors
    error_s = b"\n".join(compile_output).decode('utf8', 'replace')
    add_file_handler(cloudlog)
    cloudlog.error("scons build failed\n" + error_s)

    # Show TextWindow
    spinner.close()
    if not os.getenv("CI") and show_error_window:
      with TextWindow("IQ.Pilot failed to build\n \n" + error_s) as t:
        t.wait_for_exit()
    exit(1)

  stale = stale_registry_artifacts()
  if stale and not registry_retry:
    cloudlog.error(f"compiled service registry is stale in {', '.join(stale)}, rebuilding messaging")
    purge_registry_artifacts(stale)
    build(spinner, dirty, minimal, show_error_window, registry_retry=True)
    return
  if stale:
    error_s = "compiled service registry is still stale after rebuild: " + ", ".join(stale)
    add_file_handler(cloudlog)
    cloudlog.error(error_s)
    spinner.close()
    if not os.getenv("CI") and show_error_window:
      with TextWindow("IQ.Pilot failed to build\n \n" + error_s) as t:
        t.wait_for_exit()
    exit(1)

  # enforce max cache size
  cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
  cache_files.sort(key=lambda f: f.stat().st_mtime)
  cache_size = sum(f.stat().st_size for f in cache_files)
  for f in cache_files:
    if cache_size < MAX_CACHE_SIZE:
      break
    cache_size -= f.stat().st_size
    f.unlink()


if __name__ == "__main__":
  headless = "--headless" in sys.argv
  spinner = _SilentProgress() if headless else Spinner()
  spinner.update_progress(0, 100)
  build_metadata = get_build_metadata()
  build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS, show_error_window = not headless)