IQ.Pilot Release Commit @ bec7652

iqpilot/system/manager/build.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+# NOTE: Do NOT import anything here that needs be built (e.g. params)
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.spinner import Spinner
+from iqpilot.common.text_window import TextWindow
+from iqpilot.common.swaglog import cloudlog, add_file_handler
+from iqpilot.system.hardware import HARDWARE, AGNOS
+from iqpilot.system.version import get_build_metadata
+
+MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
+CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")
+
+TOTAL_SCONS_NODES = 5500
+MAX_BUILD_PROGRESS = 100
+
+def get_job_sequence() -> list[int]:
+  env_override = os.environ.get("SCONS_MAX_JOBS")
+  if env_override is not None:
+    try:
+      max_jobs = max(1, int(env_override))
+    except ValueError:
+      max_jobs = 1
+  else:
+    detected_jobs = os.cpu_count() or 2
+    max_jobs = min(detected_jobs, 3 if AGNOS else detected_jobs)
+
+  candidates = [max_jobs, max_jobs // 2, 1]
+  jobs: list[int] = []
+  for candidate in candidates:
+    candidate = max(1, int(candidate))
+    if candidate not in jobs:
+      jobs.append(candidate)
+  return jobs
+
+class _SilentProgress:
+  """Not a UI element: a do-nothing sink for build()'s progress calls, used when
+  updated.py builds in the background at update-apply time so that NO spinner or
+  window is shown. The normal boot path still uses the real Spinner."""
+  def update(self, spinner_text: str) -> None:
+    pass
+
+  def update_progress(self, cur: float, total: float) -> None:
+    pass
+
+  def close(self) -> None:
+    pass
+
+
+def build(spinner, dirty: bool = False, minimal: bool = False, show_error_window: bool = True) -> None:
+  env = os.environ.copy()
+  env.pop('PWD', None)
+  env['SCONS_PROGRESS'] = "1"
+
+  extra_args = ["--minimal"] if minimal else []
+
+  if AGNOS:
+    HARDWARE.set_power_save(False)
+    os.sched_setaffinity(0, range(8))  # ensure we can use the isolcpus cores
+
+  # building with all cores can result in using too
+  # much memory, so retry with less parallelism
+  compile_output: list[bytes] = []
+  for n in get_job_sequence():
+    compile_output.clear()
+    command = [sys.executable, "-m", "SCons", f"-j{int(n)}", "--cache-populate", *extra_args]
+    scons: subprocess.Popen = subprocess.Popen(command, cwd=BASEDIR, env=env, stderr=subprocess.PIPE)
+    assert scons.stderr is not None
+
+    # Read progress from stderr and update spinner
+    while scons.poll() is None:
+      try:
+        line = scons.stderr.readline()
+        if line is None:
+          continue
+        line = line.rstrip()
+
+        prefix = b'progress: '
+        if line.startswith(prefix):
+          i = int(line[len(prefix):])
+          spinner.update_progress(MAX_BUILD_PROGRESS * min(0.99, i / TOTAL_SCONS_NODES), 100.)
+        elif len(line):
+          compile_output.append(line)
+          print(line.decode('utf8', 'replace'))
+      except Exception:
+        pass
+
+    if scons.returncode == 0:
+      spinner.update_progress(100, 100.)
+      break
+
+  if scons.returncode != 0:
+    # Read remaining output
+    if scons.stderr is not None:
+      compile_output += scons.stderr.read().split(b'\n')
+
+    # Build failed log errors
+    error_s = b"\n".join(compile_output).decode('utf8', 'replace')
+    add_file_handler(cloudlog)
+    cloudlog.error("scons build failed\n" + error_s)
+
+    # Show TextWindow
+    spinner.close()
+    if not os.getenv("CI") and show_error_window:
+      with TextWindow("IQ.Pilot failed to build\n \n" + error_s) as t:
+        t.wait_for_exit()
+    exit(1)
+
+  # enforce max cache size
+  cache_files = [f for f in CACHE_DIR.rglob('*') if f.is_file()]
+  cache_files.sort(key=lambda f: f.stat().st_mtime)
+  cache_size = sum(f.stat().st_size for f in cache_files)
+  for f in cache_files:
+    if cache_size < MAX_CACHE_SIZE:
+      break
+    cache_size -= f.stat().st_size
+    f.unlink()
+
+
+if __name__ == "__main__":
+  headless = "--headless" in sys.argv
+  spinner = _SilentProgress() if headless else Spinner()
+  spinner.update_progress(0, 100)
+  build_metadata = get_build_metadata()
+  build(spinner, build_metadata.openpilot.is_dirty, minimal = AGNOS, show_error_window = not headless)

iqpilot/system/manager/github_runner.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Define the service name
+SERVICE_NAME="actions.runner.iqpilot.$(uname -n)"
+
+# Function to control the service
+control_service() {
+    local action=$1  # Store the function argument in a local variable
+    sudo systemctl $action ${SERVICE_NAME}
+}
+
+service_exists_and_is_loaded() {
+    sudo systemctl status ${SERVICE_NAME} &>/dev/null
+    if [[ $? -ne 4 ]]; then
+        return 0  # Service is known to systemd (i.e., loaded)
+    else
+        return 1  # Service is unknown to systemd (i.e., not loaded)
+    fi
+}
+
+# Check for required argument
+if [[ -z $1 ]] || { [[ $1 != "start" ]] && [[ $1 != "stop" ]]; }; then
+    echo "Usage: $0 {start|stop}"
+    exit 1
+fi
+
+# Store the script argument in a descriptive variable
+ACTION=$1
+
+# Trap EXIT signal (Ctrl+C) and stop the service
+trap 'control_service stop ; exit' SIGINT SIGKILL EXIT
+
+# Enter the main loop
+while true; do
+    # Check if the service is actually present on the system
+    if service_exists_and_is_loaded; then
+        control_service $ACTION  # Call the function with the specified action
+    fi
+    sleep 1  # Pause before the next iteration
+done

iqpilot/system/manager/helpers.py

@@ -0,0 +1,104 @@
+import errno
+import fcntl
+import os
+import sys
+import pathlib
+import shutil
+import signal
+import subprocess
+import tempfile
+import threading
+
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.params import Params
+
+def unblock_stdout() -> None:
+  # get a non-blocking stdout
+  child_pid, child_pty = os.forkpty()
+  if child_pid != 0:  # parent
+
+    # child is in its own process group, manually pass kill signals
+    signal.signal(signal.SIGINT, lambda signum, frame: os.kill(child_pid, signal.SIGINT))
+    signal.signal(signal.SIGTERM, lambda signum, frame: os.kill(child_pid, signal.SIGTERM))
+
+    fcntl.fcntl(sys.stdout, fcntl.F_SETFL, fcntl.fcntl(sys.stdout, fcntl.F_GETFL) | os.O_NONBLOCK)
+
+    while True:
+      try:
+        dat = os.read(child_pty, 4096)
+      except OSError as e:
+        if e.errno == errno.EIO:
+          break
+        continue
+
+      if not dat:
+        break
+
+      try:
+        sys.stdout.write(dat.decode('utf8'))
+      except (OSError, UnicodeDecodeError):
+        pass
+
+    # os.wait() returns a tuple with the pid and a 16 bit value
+    # whose low byte is the signal number and whose high byte is the exit status
+    exit_status = os.wait()[1] >> 8
+    os._exit(exit_status)
+
+
+def write_onroad_params(started, params):
+  params.put_bool("IsOnroad", started)
+  params.put_bool("IsOffroad", not started)
+
+
+def heal_param_perms():
+  """Self-heal for a boot-bricking failure mode: a stray root process occasionally
+  writes a param (seen with RouteCount/CurrentRoute) as root:root 0600, which manager
+  (comma) then can't read — crashing save_bootlog and any param read. Detect params we
+  don't own and chown them back + make them readable. Needs root to chown another user's
+  file, so it shells to passwordless sudo (device grants it); best-effort, never raises,
+  never blocks boot. No-op when everything is already ours (the common case: no sudo)."""
+  try:
+    param_path = Params().get_param_path()
+    uid, gid = os.getuid(), os.getgid()
+    stray = []
+    for name in os.listdir(param_path):
+      p = os.path.join(param_path, name)
+      try:
+        if os.stat(p).st_uid != uid:
+          stray.append(p)
+      except OSError:
+        pass
+    if stray:
+      subprocess.run(["sudo", "-n", "chown", f"{uid}:{gid}", *stray], check=False, timeout=15,
+                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+      subprocess.run(["sudo", "-n", "chmod", "644", *stray], check=False, timeout=15,
+                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+  except Exception:
+    pass
+
+
+def save_bootlog():
+  # copy current params
+  tmp = tempfile.mkdtemp()
+  params_dirname = pathlib.Path(Params().get_param_path()).name
+  params_dir = os.path.join(tmp, params_dirname)
+
+  # Params are rewritten atomically (unlink + rename) by other processes, so a
+  # value can vanish between copytree's listing and the copy; a param may also be
+  # unreadable (e.g. a root-owned RouteCount/CurrentRoute). Skip any file we can't
+  # copy instead of raising — the bootlog snapshot is best-effort and must NOT block boot.
+  def _copy_skip_missing(src, dst, *, follow_symlinks=True):
+    try:
+      shutil.copy2(src, dst, follow_symlinks=follow_symlinks)
+    except OSError:
+      pass
+  shutil.copytree(Params().get_param_path(), params_dir, dirs_exist_ok=True, copy_function=_copy_skip_missing)
+
+  def fn(tmpdir):
+    env = os.environ.copy()
+    env['PARAMS_COPY_PATH'] = tmpdir
+    subprocess.call("./bootlog", cwd=os.path.join(BASEDIR, "iqpilot/system/loggerd"), env=env)
+    shutil.rmtree(tmpdir)
+  t = threading.Thread(target=fn, args=(tmp, ))
+  t.daemon = True
+  t.start()

iqpilot/system/manager/manager.py

@@ -0,0 +1,273 @@
+#!/usr/bin/env python3
+import datetime
+import os
+import signal
+import sys
+import time
+import traceback
+
+# sync $PWD before cereal/kj loads (and for spawned procs) or kj warns "PWD doesn't match"
+os.environ['PWD'] = os.getcwd()
+
+from iqpilot.cereal import log
+import iqpilot.cereal.messaging as messaging
+import iqpilot.system.sentry as sentry
+from iqpilot.common.utils import atomic_write
+from iqpilot.common.params import Params, ParamKeyFlag
+from iqpilot.common.text_window import TextWindow
+from iqpilot.system.hardware import HARDWARE
+from iqpilot.system.loggerd.crash_recovery import recover_unclean_segments
+from iqpilot.system.manager.helpers import unblock_stdout, write_onroad_params, save_bootlog, heal_param_perms
+from iqpilot.system.manager.process import ensure_running
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.konn3kt.registration import register, UNREGISTERED_DONGLE_ID
+from iqpilot.common.swaglog import cloudlog, add_file_handler
+from iqpilot.system.version import get_build_metadata
+from iqpilot.system.hardware.hw import Paths
+
+
+def manager_init() -> None:
+  heal_param_perms()
+  save_bootlog()
+
+  # loggerd isn't running yet, so any leftover .lock marks an unclean shutdown:
+  # unlock those segments and preserve them (dashcam footage from power cuts)
+  try:
+    recover_unclean_segments()
+  except Exception:
+    cloudlog.exception("recover_unclean_segments failed")
+
+  build_metadata = get_build_metadata()
+
+  params = Params()
+  params.clear_all(ParamKeyFlag.CLEAR_ON_MANAGER_START)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)
+  params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)
+  if build_metadata.release_channel:
+    params.clear_all(ParamKeyFlag.DEVELOPMENT_ONLY)
+
+  # device boot mode
+  if params.get("DeviceBootMode") == 1:  # start in Always Offroad mode
+    params.put_bool("IQAlwaysOffroad", True)
+
+  if params.get_bool("RecordFrontLock"):
+    params.put_bool("RecordFront", True)
+
+  # set unset params to their default value
+  for k in params.all_keys():
+    default_value = params.get_default_value(k)
+    if default_value is not None and params.get(k) is None:
+      params.put(k, default_value)
+
+  try:
+    from iqpilot.selfdrive.iqmodeld.models.helpers import seed_default_bundle_if_unset
+    seed_default_bundle_if_unset(params)
+  except Exception:
+    cloudlog.exception("failed to seed default model bundle")
+
+  # Create folders needed for msgq
+  try:
+    os.mkdir(Paths.shm_path())
+  except FileExistsError:
+    pass
+  except PermissionError:
+    print(f"WARNING: failed to make {Paths.shm_path()}")
+
+  # set params
+  serial = HARDWARE.get_serial()
+  params.put("Version", build_metadata.openpilot.version)
+  params.put("GitCommit", build_metadata.openpilot.git_commit)
+  params.put("GitCommitDate", build_metadata.openpilot.git_commit_date)
+  params.put("GitBranch", build_metadata.channel)
+  params.put("GitRemote", build_metadata.openpilot.git_origin)
+  params.put_bool("IsDevelopmentBranch", build_metadata.development_channel)
+  params.put_bool("IsTestedBranch", build_metadata.tested_channel)
+  params.put_bool("IsReleaseBranch", build_metadata.release_channel)
+  params.put_bool("IsReleaseIqBranch", build_metadata.release_channel)
+  params.put("HardwareSerial", serial)
+
+  # set dongle id
+  reg_res = register(show_spinner=True)
+  if reg_res:
+    dongle_id = reg_res
+  else:
+    raise Exception(f"Registration failed for device {serial}")
+  os.environ['DONGLE_ID'] = dongle_id  # Needed for swaglog
+  os.environ['GIT_ORIGIN'] = build_metadata.openpilot.git_normalized_origin # Needed for swaglog
+  os.environ['GIT_BRANCH'] = build_metadata.channel # Needed for swaglog
+  os.environ['GIT_COMMIT'] = build_metadata.openpilot.git_commit # Needed for swaglog
+
+  if not build_metadata.openpilot.is_dirty:
+    os.environ['CLEAN'] = '1'
+
+  # init logging
+  sentry.init(sentry.SentryProject.SELFDRIVE)
+  cloudlog.bind_global(dongle_id=dongle_id,
+                       version=build_metadata.openpilot.version,
+                       origin=build_metadata.openpilot.git_normalized_origin,
+                       branch=build_metadata.channel,
+                       commit=build_metadata.openpilot.git_commit,
+                       dirty=build_metadata.openpilot.is_dirty,
+                       device=HARDWARE.get_device_type())
+
+  # preimport all processes
+  for p in managed_processes.values():
+    p.prepare()
+
+
+def manager_cleanup() -> None:
+  # send signals to kill all procs
+  for p in managed_processes.values():
+    p.stop(block=False)
+
+  # ensure all are killed
+  for p in managed_processes.values():
+    p.stop(block=True)
+
+  cloudlog.info("everything is dead")
+
+
+def manager_thread() -> None:
+  cloudlog.bind(daemon="manager")
+  cloudlog.info("manager start")
+  cloudlog.info({"environ": os.environ})
+
+  params = Params()
+
+  ignore: list[str] = []
+  if params.get("DongleId") in (None, UNREGISTERED_DONGLE_ID):
+    ignore += ["manage_hephaestusd", "iquploaderd"]
+  if os.getenv("NOBOARD") is not None:
+    ignore.append("pandad")
+  ignore += [x for x in os.getenv("BLOCK", "").split(",") if len(x) > 0]
+
+  sm = messaging.SubMaster(['deviceState', 'carParams', 'pandaStates'], poll='deviceState')
+  pm = messaging.PubMaster(['managerState'])
+
+  write_onroad_params(False, params)
+  ensure_running(managed_processes.values(), False, params=params, CP=sm['carParams'], not_run=ignore)
+
+  started_prev = False
+  ignition_prev = False
+  running_prev = None
+
+  while True:
+    sm.update(1000)
+
+    started = sm['deviceState'].started
+
+    if started and not started_prev:
+      try:
+        HARDWARE.set_power_save(False)
+      except Exception:
+        cloudlog.exception("failed to leave power save on onroad transition")
+      params.clear_all(ParamKeyFlag.CLEAR_ON_ONROAD_TRANSITION)
+    elif not started and started_prev:
+      params.clear_all(ParamKeyFlag.CLEAR_ON_OFFROAD_TRANSITION)
+
+    ignition = any(ps.ignitionLine or ps.ignitionCan for ps in sm['pandaStates'] if ps.pandaType != log.PandaState.PandaType.unknown)
+    if ignition and not ignition_prev:
+      params.clear_all(ParamKeyFlag.CLEAR_ON_IGNITION_ON)
+
+    # update onroad params, which drives pandad's safety setter thread
+    if started != started_prev:
+      write_onroad_params(started, params)
+
+    started_prev = started
+    ignition_prev = ignition
+
+    ensure_running(managed_processes.values(), started, params=params, CP=sm['carParams'], not_run=ignore)
+
+    # print only on change (reprinting every loop floods the shared tmux); always logged
+    procs = [p for p in managed_processes.values() if p.proc]
+    running = ' '.join(
+      ("\u001b[32m{}\u001b[0m".format(p.name) if p.proc.is_alive()
+       else "\u001b[1;31m\u2717 {}\u001b[0m".format(p.name))
+      for p in procs)
+    cloudlog.debug(running)
+    alive = tuple(p.proc.is_alive() for p in procs)
+    if alive != running_prev:
+      print(running)
+      running_prev = alive
+
+    # send managerState
+    msg = messaging.new_message('managerState', valid=True)
+    msg.managerState.processes = [p.get_process_state_msg() for p in managed_processes.values()]
+    pm.send('managerState', msg)
+
+    # kick AGNOS power monitoring watchdog
+    try:
+      if sm.all_checks(['deviceState']):
+        with atomic_write("/var/tmp/power_watchdog", "w", overwrite=True) as f:
+          f.write(str(time.monotonic()))
+    except Exception:
+      pass
+
+    # Exit main loop when uninstall/shutdown/reboot is needed
+    shutdown = False
+    for param in ("DoUninstall", "DoShutdown", "DoReboot"):
+      if params.get_bool(param):
+        shutdown = True
+        params.put("LastManagerExitReason", f"{param} {datetime.datetime.now()}")
+        cloudlog.warning(f"Shutting down manager - {param} set")
+
+    if shutdown:
+      break
+
+
+def main() -> None:
+  manager_init()
+  if os.getenv("PREPAREONLY") is not None:
+    return
+
+  # SystemExit on sigterm
+  signal.signal(signal.SIGTERM, lambda signum, frame: sys.exit(1))
+
+  try:
+    manager_thread()
+  except Exception:
+    traceback.print_exc()
+    sentry.capture_exception()
+  finally:
+    manager_cleanup()
+
+  params = Params()
+  if params.get_bool("DoUninstall"):
+    cloudlog.warning("uninstalling")
+    HARDWARE.uninstall()
+  elif params.get_bool("DoReboot"):
+    cloudlog.warning("reboot")
+    HARDWARE.reboot()
+  elif params.get_bool("DoShutdown"):
+    cloudlog.warning("shutdown")
+    HARDWARE.shutdown()
+
+
+if __name__ == "__main__":
+  if not (os.getenv("SIMULATION") and sys.platform == "darwin"):
+    unblock_stdout()
+
+  try:
+    main()
+  except KeyboardInterrupt:
+    print("got CTRL-C, exiting")
+  except Exception:
+    add_file_handler(cloudlog)
+    cloudlog.exception("Manager failed to start")
+
+    try:
+      managed_processes['ui'].stop()
+    except Exception:
+      pass
+
+    # Show last 3 lines of traceback
+    error = traceback.format_exc(-3)
+    error = "Manager failed to start\n\n" + error
+    with TextWindow(error) as t:
+      t.wait_for_exit()
+
+    raise
+
+  # manual exit because we are forked
+  sys.exit(0)

iqpilot/system/manager/process.py

@@ -0,0 +1,365 @@
+import importlib
+import os
+import signal
+import time
+import subprocess
+from pathlib import Path
+from collections.abc import Callable, ValuesView
+from abc import ABC, abstractmethod
+from multiprocessing import Process
+
+from setproctitle import setproctitle
+
+from iqpilot.cereal import car, log
+import iqpilot.cereal.messaging as messaging
+import iqpilot.system.sentry as sentry
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.params import Params
+from iqpilot.common.swaglog import cloudlog
+
+MAX_CRASH_BACKOFF = 300.0
+CRASH_RESET_TIME = 60.0
+CRASH_LOOP_THRESHOLD = 6
+
+try:
+  from iqpilot.system.proprietary_runtime.runtime_paths import preferred_runner_path
+except ModuleNotFoundError:
+  _VERIFIED_RUNNER_PATH = Path("/usr/libexec/iqpilot/iqpilot_bundle_runner")
+  _FALLBACK_RUNNER_PATH = Path("/data/openpilot/iqpilot/system/proprietary_runtime/iqpilot_bundle_runner")
+
+  def preferred_runner_path() -> Path:
+    if _VERIFIED_RUNNER_PATH.is_file() and os.access(_VERIFIED_RUNNER_PATH, os.X_OK):
+      return _VERIFIED_RUNNER_PATH
+    if os.getenv("IQPILOT_ALLOW_DEV_FALLBACKS") == "1" and _FALLBACK_RUNNER_PATH.is_file():
+      return _FALLBACK_RUNNER_PATH
+    return _VERIFIED_RUNNER_PATH
+
+
+def launcher(proc: str, name: str) -> None:
+  try:
+    # import the process
+    mod = importlib.import_module(proc)
+
+    # rename the process
+    setproctitle(proc)
+
+    # create new context since we forked
+    messaging.reset_context()
+
+    # add daemon name tag to logs
+    cloudlog.bind(daemon=name)
+    sentry.set_tag("daemon", name)
+
+    # exec the process
+    mod.main()
+  except KeyboardInterrupt:
+    cloudlog.warning(f"child {proc} got SIGINT")
+  except Exception:
+    # can't install the crash handler because sys.excepthook doesn't play nice
+    # with threads, so catch it here.
+    sentry.capture_exception()
+    raise
+
+
+def nativelauncher(pargs: list[str], cwd: str, name: str) -> None:
+  os.environ['MANAGER_DAEMON'] = name
+
+  # exec the process
+  os.chdir(cwd)
+  os.environ['PWD'] = cwd
+  os.execvp(pargs[0], pargs)
+
+
+def join_process(process: Process, timeout: float) -> None:
+  # Process().join(timeout) will hang due to a python 3 bug: https://bugs.python.org/issue28382
+  # We have to poll the exitcode instead
+  t = time.monotonic()
+  while time.monotonic() - t < timeout and process.exitcode is None:
+    time.sleep(0.001)
+
+
+class ManagerProcess(ABC):
+  daemon = False
+  sigkill = False
+  should_run: Callable[[bool, Params, car.CarParams], bool]
+  proc: Process | None = None
+  enabled = True
+  name = ""
+  shutting_down = False
+  restart_if_crash = False
+  crash_count = 0
+  last_restart_time = 0.0
+  last_alive_time = 0.0
+  crash_loop_logged = False
+
+  @abstractmethod
+  def prepare(self) -> None:
+    pass
+
+  @abstractmethod
+  def start(self) -> None:
+    pass
+
+  def restart(self) -> None:
+    self.stop(sig=signal.SIGKILL)
+    self.start()
+
+  def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals | None = None, timeout: float = 5) -> int | None:
+    if self.proc is None:
+      return None
+
+    if self.proc.exitcode is None:
+      if not self.shutting_down:
+        cloudlog.info(f"killing {self.name}")
+        if sig is None:
+          sig = signal.SIGKILL if self.sigkill else signal.SIGINT
+        self.signal(sig)
+        self.shutting_down = True
+
+        if not block:
+          return None
+
+      join_process(self.proc, timeout)
+
+      # If process failed to die send SIGKILL
+      if self.proc.exitcode is None and retry:
+        cloudlog.info(f"killing {self.name} with SIGKILL")
+        self.signal(signal.SIGKILL)
+        self.proc.join()
+
+    ret = self.proc.exitcode
+    cloudlog.info(f"{self.name} is dead with {ret}")
+
+    if self.proc.exitcode is not None:
+      self.shutting_down = False
+      self.proc = None
+
+    return ret
+
+  def signal(self, sig: int) -> None:
+    if self.proc is None:
+      return
+
+    # Don't signal if already exited
+    if self.proc.exitcode is not None and self.proc.pid is not None:
+      return
+
+    # Can't signal if we don't have a pid
+    if self.proc.pid is None:
+      return
+
+    cloudlog.info(f"sending signal {sig} to {self.name}")
+    os.kill(self.proc.pid, sig)
+
+  def get_process_state_msg(self):
+    state = log.ManagerState.ProcessState.new_message()
+    state.name = self.name
+    if self.proc:
+      state.running = self.proc.is_alive()
+      state.shouldBeRunning = self.proc is not None and not self.shutting_down
+      state.pid = self.proc.pid or 0
+      state.exitCode = self.proc.exitcode or 0
+    return state
+
+
+class NativeProcess(ManagerProcess):
+  def __init__(self, name, cwd, cmdline, should_run, enabled=True, sigkill=False, restart_if_crash=False):
+    self.name = name
+    self.cwd = cwd
+    self.cmdline = cmdline
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.launcher = nativelauncher
+    self.restart_if_crash = restart_if_crash
+
+  def prepare(self) -> None:
+    pass
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    cwd = os.path.join(BASEDIR, self.cwd)
+    cloudlog.info(f"starting process {self.name}")
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.cmdline, cwd, self.name))
+    self.proc.start()
+    self.shutting_down = False
+
+
+def _normalize_bundle_modes(bundle: str) -> None:
+  import json
+  candidates = []
+  if env_root := os.environ.get("IQPILOT_PROPRIETARY_ROOT"):
+    candidates += [os.path.join(env_root, bundle), env_root]
+  candidates += [
+    os.path.join(BASEDIR, ".iqpilot", "bundles", bundle),
+    os.path.join(os.path.dirname(BASEDIR), ".iqpilot", "bundles", bundle),
+    os.path.join(BASEDIR, "artifacts", bundle),
+  ]
+  root = next((c for c in candidates if os.path.isfile(os.path.join(c, "manifest.json"))), None)
+  if root is None:
+    return
+  try:
+    with open(os.path.join(root, "manifest.json")) as f:
+      manifest = json.load(f)
+    for rel, meta in manifest.items():
+      if not (isinstance(meta, dict) and "mode" in meta and "sha256" in meta):
+        continue
+      path = os.path.join(root, rel)
+      if os.path.isfile(path) and (os.stat(path).st_mode & 0o777) != meta["mode"]:
+        os.chmod(path, meta["mode"])
+  except Exception:
+    cloudlog.exception(f"failed to normalize bundle modes for {bundle}")
+
+
+class BundleProcess(NativeProcess):
+  def __init__(self, name, bundle, entry, should_run, enabled=True, sigkill=False, restart_if_crash=False):
+    self.bundle = bundle
+    self.entry = entry
+    self.restart_if_crash = restart_if_crash
+    runner_path = preferred_runner_path()
+    runner_cmd = str(runner_path) if runner_path.is_absolute() else "./iqpilot_bundle_runner"
+    runner_cwd = ".iqpilot/runtime_root" if runner_path.is_absolute() else "system/proprietary_runtime"
+    super().__init__(
+      name=name,
+      cwd=runner_cwd,
+      cmdline=[
+        runner_cmd,
+        "--bundle", bundle,
+        "--mode", "python-module",
+        "--entry", entry,
+        "--daemon-name", name,
+      ],
+      should_run=should_run,
+      enabled=enabled,
+      sigkill=sigkill,
+    )
+
+  def start(self) -> None:
+    if self.proc is None:
+      _normalize_bundle_modes(self.bundle)
+    super().start()
+
+  def stop(self, retry: bool = True, block: bool = True, sig: signal.Signals | None = None, timeout: float = 5) -> int | None:
+    return super().stop(retry=retry, block=block, sig=signal.SIGTERM if sig is None else sig, timeout=timeout)
+
+
+class PythonProcess(ManagerProcess):
+  def __init__(self, name, module, should_run, enabled=True, sigkill=False, restart_if_crash=False):
+    self.name = name
+    self.module = module
+    self.should_run = should_run
+    self.enabled = enabled
+    self.sigkill = sigkill
+    self.launcher = launcher
+    self.restart_if_crash = restart_if_crash
+
+  def prepare(self) -> None:
+    if self.enabled:
+      cloudlog.info(f"preimporting {self.module}")
+      importlib.import_module(self.module)
+
+  def start(self) -> None:
+    # In case we only tried a non blocking stop we need to stop it before restarting
+    if self.shutting_down:
+      self.stop()
+
+    if self.proc is not None:
+      return
+
+    cloudlog.info(f"starting python {self.module}")
+    self.proc = Process(name=self.name, target=self.launcher, args=(self.module, self.name))
+    self.proc.start()
+    self.shutting_down = False
+
+
+class DaemonProcess(ManagerProcess):
+  """Python process that has to stay running across manager restart.
+  This is used for athena so you don't lose SSH access when restarting manager."""
+  def __init__(self, name, module, param_name, enabled=True):
+    self.name = name
+    self.module = module
+    self.param_name = param_name
+    self.enabled = enabled
+    self.params = None
+
+  @staticmethod
+  def should_run(started, params, CP):
+    return True
+
+  def prepare(self) -> None:
+    pass
+
+  def start(self) -> None:
+    if self.params is None:
+      self.params = Params()
+
+    pid = self.params.get(self.param_name)
+    if pid is not None:
+      try:
+        os.kill(int(pid), 0)
+        with open(f'/proc/{pid}/cmdline') as f:
+          if self.module in f.read():
+            # daemon is running
+            return
+      except (OSError, FileNotFoundError):
+        # process is dead
+        pass
+
+    cloudlog.info(f"starting daemon {self.name}")
+    proc = subprocess.Popen(['python', '-m', self.module],
+                               stdin=open('/dev/null'),
+                               stdout=open('/dev/null', 'w'),
+                               stderr=open('/dev/null', 'w'),
+                               preexec_fn=os.setpgrp)
+
+    self.params.put(self.param_name, proc.pid)
+
+  def stop(self, retry=True, block=True, sig=None) -> None:
+    pass
+
+
+def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None, CP: car.CarParams=None,
+                   not_run: list[str] | None=None) -> list[ManagerProcess]:
+  if not_run is None:
+    not_run = []
+
+  running = []
+  now = time.monotonic()
+  for p in procs:
+    if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
+      if p.restart_if_crash and p.proc is not None and p.proc.is_alive():
+        p.last_alive_time = now
+      elif p.restart_if_crash and p.proc is not None:
+        # uptime, not time-since-restart: the latter also counts the backoff wait,
+        # which would reset the counter as soon as backoff exceeds CRASH_RESET_TIME
+        if p.last_alive_time - p.last_restart_time > CRASH_RESET_TIME:
+          p.crash_count = 0
+          p.crash_loop_logged = False
+
+        backoff = 0.0 if not p.crash_count else min(MAX_CRASH_BACKOFF, 2.0 ** (p.crash_count - 1))
+        if now - p.last_restart_time >= backoff:
+          p.crash_count += 1
+          p.last_restart_time = now
+          cloudlog.error(f'Restarting {p.name} (exitcode {p.proc.exitcode}) [crash {p.crash_count}]')
+          if p.crash_count >= CRASH_LOOP_THRESHOLD and not p.crash_loop_logged:
+            # never stop retrying: giving up on hardwared or ui is worse than restarting slowly
+            cloudlog.error(f'{p.name} is in a crash loop, backing off to {MAX_CRASH_BACKOFF}s between restarts')
+            p.crash_loop_logged = True
+          p.restart()
+      running.append(p)
+    else:
+      p.crash_count = 0
+      p.crash_loop_logged = False
+      p.last_alive_time = 0.0
+      p.stop(block=False)
+
+  for p in running:
+    p.start()
+
+  return running

iqpilot/system/manager/process_config.py

@@ -0,0 +1,221 @@
+import os
+import platform
+from pathlib import Path
+
+from iqpilot.cereal import car, custom
+from iqpilot.common.params import Params
+from iqpilot.system.hardware import HARDWARE, PC, TICI
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.system.manager.process import PythonProcess, NativeProcess, BundleProcess
+
+from iqpilot.selfdrive.iqmodeld.models.helpers import get_active_model_runner
+from iqpilot.konn3kt.service_health import hephaestus_ready
+
+def driverview(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started or params.get_bool("IsDriverViewEnabled")
+
+def driver_monitoring(started: bool, params: Params, CP: car.CarParams) -> bool:
+  if os.path.exists('/tmp/lite_hw'):
+    return False
+  return driverview(started, params, CP)
+
+def notcar(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and CP.notCar
+
+def iscar(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not CP.notCar
+
+def logging(started: bool, params: Params, CP: car.CarParams) -> bool:
+  run = (not CP.notCar) or not params.get_bool("DisableLogging")
+  return started and run and params.get_bool("DashcamEnabled")
+
+def ublox_available() -> bool:
+  if HARDWARE.get_device_type() == "tizi" or os.path.exists('/tmp/lite_hw'):
+    return False
+
+  quectel_override = Path(Paths.persist_root()) / "comma" / "use-quectel-gps"
+  return os.path.exists('/dev/ttyHS0') and not quectel_override.exists()
+
+def ublox(started: bool, params: Params, CP: car.CarParams) -> bool:
+  use_ublox = ublox_available()
+  if use_ublox != params.get_bool("UbloxAvailable"):
+    params.put_bool("UbloxAvailable", use_ublox)
+  return started and use_ublox
+
+def joystick(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("JoystickDebugMode")
+
+def not_joystick(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not params.get_bool("JoystickDebugMode")
+
+def long_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("LongitudinalManeuverMode")
+
+def not_long_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not params.get_bool("LongitudinalManeuverMode")
+
+def lat_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("LateralManeuverMode")
+
+def not_lat_maneuver(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not params.get_bool("LateralManeuverMode")
+
+def qcomgps(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and not ublox_available()
+
+def always_run(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return True
+
+def only_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started
+
+def navd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("NavigationEnabled")
+
+def navrenderd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("NavigationEnabled") and params.get_bool("OnScreenNavigation")
+
+def navincidentd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("NavigationEnabled") and bool(params.get("WazePoliceApiKey")) and (
+    params.get_int("WazePoliceAlertMode") > 0 or params.get_bool("WazePoliceShadow")
+  )
+
+def iqmapd_needed(params: Params) -> bool:
+  return (
+    params.get_bool("IQRoadNameOverlay")
+    or params.get_bool("ShowSpeedLimits")
+    or params.get_bool("SpeedLimitController")
+    or params.get_bool("EnableSpeedLimitControl")
+    or params.get_bool("EnableSpeedLimitPredicative")
+    or params.get_bool("MapCurveSpeedController")
+    or params.get_bool("VisionCurveSpeedController")
+  )
+
+def iqmapd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("NavigationEnabled") and iqmapd_needed(params)
+
+def mapd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and iqmapd_needed(params)
+
+def constructiond_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return started and params.get_bool("ConstructionZoneAssist")
+
+def iqvd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  # held for 1.0d: iqvd runs a detector per frame and the added load is not
+  # something 1.0c needs to carry. re-enable by restoring the param check.
+  return False
+
+def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool:
+  return not started
+
+def livestream(started: bool, params: Params, CP: car.CarParams) -> bool:
+  # Konn3kt Live View: hephaestusd sets IsLiveStreaming when a viewer connects, so the
+  # manager brings up the stream encoder (and camerad/webrtcd when offroad) and tears them
+  # down cleanly when the session ends — no subprocess management inside hephaestusd.
+  return params.get_bool("IsLiveStreaming")
+
+def canlive(started: bool, params: Params, CP: car.CarParams) -> bool:
+  # Remote live CAN debugging via konn3kt. hephaestusd sets CanLiveStreaming when a viewer
+  # connects (startCanLive) and clears it when the last one leaves (stopCanLive), so canlived
+  # runs only during an active debug session — no idle connection or battery cost otherwise.
+  return params.get_bool("CanLiveStreaming")
+
+def is_tinygrad_model(started, params, CP: car.CarParams) -> bool:
+  """Check if the active model runner is tinygrad."""
+  return bool(get_active_model_runner(params, not started) == custom.IQModelManager.Runner.tinygrad)
+
+def hephaestus_ready_shim(started, params, CP: car.CarParams) -> bool:
+  return hephaestus_ready(params)
+
+def not_low_power(started: bool, params: Params, CP: car.CarParams) -> bool:
+  # FastSleep deep standby: heavy processes are shed offroad while DevicePowerState is low_power
+  return started or params.get("DevicePowerState") != "low_power"
+
+def iquploaderd_ready(started: bool, params: Params, CP: car.CarParams) -> bool:
+  if not params.get_bool("OnroadUploads"):
+    return only_offroad(started, params, CP)
+
+  return always_run(started, params, CP)
+
+def or_(*fns):
+  return lambda *args: any(fn(*args) for fn in fns)
+
+def and_(*fns):
+  return lambda *args: all(fn(*args) for fn in fns)
+
+procs = [
+  NativeProcess("loggerd", "iqpilot/system/loggerd", ["./loggerd"], logging),
+  NativeProcess("encoderd", "iqpilot/system/loggerd", ["./encoderd"], only_onroad),
+  NativeProcess("stream_encoderd", "iqpilot/system/loggerd", ["./encoderd", "--stream"], or_(notcar, livestream)),
+  PythonProcess("logmessaged", "iqpilot.system.logmessaged", always_run, restart_if_crash=True),
+
+  NativeProcess("camerad", "iqpilot/system/camerad", ["./camerad"], or_(driverview, livestream), restart_if_crash=True),
+  PythonProcess("proclogd", "iqpilot.system.proclogd", only_onroad, enabled=platform.system() != "Darwin"),
+  PythonProcess("journald", "iqpilot.system.journald", only_onroad, platform.system() != "Darwin"),
+  PythonProcess("micd", "iqpilot.system.micd", or_(iscar, livestream)),
+  PythonProcess("timed", "iqpilot.system.timed", always_run, enabled=not PC),
+
+  PythonProcess("dmonitoringmodeld", "iqpilot.selfdrive.dmonitoringmodeld.dmonitoringmodeld", driver_monitoring, enabled=not PC),
+
+  PythonProcess("sensord", "iqpilot.system.sensord.sensord", only_onroad, enabled=not PC),
+  PythonProcess("ui", "iqpilot.selfdrive.ui.ui", not_low_power, restart_if_crash=True),
+  PythonProcess("soundd", "iqpilot.selfdrive.ui.soundd", driverview),
+  PythonProcess("locationd", "iqpilot.selfdrive.locationd.locationd", only_onroad),
+  NativeProcess("_pandad", "iqpilot/selfdrive/pandad", ["./pandad"], always_run, enabled=False),
+  PythonProcess("calibrationd", "iqpilot.selfdrive.locationd.calibrationd", only_onroad),
+  PythonProcess("controlsd", "iqpilot.selfdrive.controls.controlsd", and_(not_joystick, iscar)),
+  PythonProcess("joystickd", "iqpilot.tools.joystick.joystickd", or_(joystick, notcar)),
+  PythonProcess("selfdrived", "iqpilot.selfdrive.selfdrived.selfdrived", only_onroad),
+  PythonProcess("card", "iqpilot.selfdrive.car.card", only_onroad),
+  PythonProcess("deleter", "iqpilot.system.loggerd.deleter", always_run),
+  PythonProcess("dmonitoringd", "iqpilot.selfdrive.monitoring.dmonitoringd", driver_monitoring, enabled=not PC),
+  PythonProcess("qcomgpsd", "iqpilot.system.qcomgpsd.qcomgpsd", qcomgps, enabled=TICI),
+  PythonProcess("pandad", "iqpilot.selfdrive.pandad.pandad", always_run),
+  PythonProcess("estimatord", "iqpilot.selfdrive.locationd.estimatord", only_onroad),
+  PythonProcess("ubloxd", "iqpilot.system.ubloxd.ubloxd", ublox, enabled=TICI),
+  PythonProcess("pigeond", "iqpilot.system.ubloxd.pigeond", ublox, enabled=TICI),
+  PythonProcess("plannerd", "iqpilot.selfdrive.controls.plannerd", not_long_maneuver),
+  PythonProcess("maneuversd", "iqpilot.tools.maneuvers.longitudinal_maneuversd", long_maneuver),
+  PythonProcess("lateral_maneuversd", "iqpilot.tools.maneuvers.lateral_maneuversd", lat_maneuver),
+  PythonProcess("radard", "iqpilot.selfdrive.controls.radard", only_onroad),
+  PythonProcess("hardwared", "iqpilot.system.hardware.hardwared", always_run, restart_if_crash=True),
+  PythonProcess("tombstoned", "iqpilot.system.tombstoned", always_run, enabled=not PC),
+  PythonProcess("updated", "iqpilot.system.updated.updated", and_(only_offroad, not_low_power), enabled=not PC),
+  BundleProcess("iquploaderd", "iqpilot_hephaestusd_private", "iqpilot_private.konn3kt.uploaderd.iquploaderd",
+                and_(iquploaderd_ready, not_low_power), restart_if_crash=True),
+  PythonProcess("feedbackd", "iqpilot.selfdrive.ui.feedback.feedbackd", and_(only_onroad, not_lat_maneuver)),
+
+  # debug procs
+  NativeProcess("bridge", "iqpilot/cereal/messaging", ["./bridge"], notcar),
+  PythonProcess("webrtcd", "iqpilot.system.webrtc.webrtcd", or_(iscar, livestream)),
+  PythonProcess("canlived", "iqpilot.konn3kt.canlive.canlived", canlive),
+]
+
+# iqpilot
+procs += [
+  # Models
+  BundleProcess("models_manager", "iqpilot_model_selector_private", "iqpilot_private.models.manager", and_(only_offroad, not_low_power)),
+  NativeProcess("iqmodeld", "iqpilot/selfdrive/iqmodeld", ["./iqmodeld"], and_(only_onroad, is_tinygrad_model), restart_if_crash=True),
+
+  BundleProcess("backup_manager_k3", "iqpilot_hephaestusd_private", "iqpilot_private.konn3kt.backups.backup_orchestrator",
+                and_(only_offroad, hephaestus_ready_shim, not_low_power)),
+  BundleProcess("navd", "iqpilot_navd_private", "iqpilot_private.navd.navd", navd_onroad, restart_if_crash=True),
+  BundleProcess("navincidentd", "iqpilot_navd_private", "iqpilot_private.navd.navincidentd", navincidentd_onroad, restart_if_crash=True),
+  BundleProcess("navrenderd", "iqpilot_navd_private", "iqpilot_private.navd.navrenderd", navrenderd_onroad, restart_if_crash=True),
+  BundleProcess("iqmapd", "iqpilot_navd_private", "iqpilot_private.navd.iqmapd", iqmapd_onroad, restart_if_crash=True),
+
+  # work-zone detector for Speed Limit Assist
+  PythonProcess("constructiond", "iqpilot.selfdrive.constructiond", constructiond_onroad, restart_if_crash=True),
+
+  # iqvd: vision vehicle detector for UI ambient track dots
+  BundleProcess("iqvd", "iqpilot_iqvd_private", "iqpilot_private.iqvd.iqvd", iqvd_onroad, restart_if_crash=True),
+
+  # mapd
+  NativeProcess("mapd", "iqpilot/third_party/mapd_pfeiferj", ["./mapd"], mapd_onroad, restart_if_crash=True),
+  PythonProcess("mapd_manager", "iqpilot.iq_maps.orchestrator", and_(only_offroad, not_low_power)),
+
+  # locationd
+  NativeProcess("iqlocd", "iqpilot/selfdrive/iqlocd", ["./iqlocd"], only_onroad),
+]
+
+managed_processes = {p.name: p for p in procs}

iqpilot/system/manager/test/test_manager.py

@@ -0,0 +1,111 @@
+import os
+import pytest
+import signal
+import time
+
+from iqpilot.cereal import car
+from iqpilot.common.params import Params
+import iqpilot.system.manager.manager as manager
+from iqpilot.system.manager.process import BundleProcess, NativeProcess, ensure_running
+from iqpilot.system.manager.process_config import managed_processes, procs
+from iqpilot.system.hardware import HARDWARE
+
+os.environ['FAKEUPLOAD'] = "1"
+
+MAX_STARTUP_TIME = 3
+BLACKLIST_PROCS = ['manage_hephaestusd', 'pandad', 'pigeond']
+
+
+class TestManager:
+  def setup_method(self):
+    HARDWARE.set_power_save(False)
+
+    # ensure clean CarParams
+    params = Params()
+    params.clear_all()
+
+  def teardown_method(self):
+    manager.manager_cleanup()
+
+  @pytest.mark.linux
+  def test_manager_prepare(self):
+    os.environ['PREPAREONLY'] = '1'
+    manager.main()
+
+  def test_duplicate_procs(self):
+    assert len(procs) == len(managed_processes), "Duplicate process names"
+
+  def test_models_manager_uses_private_bundle(self):
+    proc = managed_processes["models_manager"]
+
+    assert isinstance(proc, BundleProcess)
+    assert proc.bundle == "iqpilot_model_selector_private"
+    assert proc.entry == "iqpilot_private.models.manager"
+
+  def test_bundle_process_stops_with_sigterm(self, mocker):
+    proc = BundleProcess("test", "bundle", "entry", lambda *_: True)
+    proc.proc = mocker.Mock(exitcode=None, pid=123)
+    proc.proc.is_alive.return_value = True
+    signal_mock = mocker.patch.object(proc, "signal")
+
+    proc.stop(block=False)
+
+    signal_mock.assert_called_once_with(signal.SIGTERM)
+
+  def test_native_process_stop_timeout(self, mocker):
+    proc = NativeProcess("test", ".", ["true"], lambda *_: True)
+    native_process = mocker.Mock(exitcode=None, pid=123)
+    proc.proc = native_process
+    signal_mock = mocker.patch.object(proc, "signal")
+    join_mock = mocker.patch("iqpilot.system.manager.process.join_process", side_effect=lambda process, _: setattr(process, "exitcode", 0))
+
+    assert proc.stop(timeout=30) == 0
+    signal_mock.assert_called_once_with(signal.SIGINT)
+    join_mock.assert_called_once_with(native_process, 30)
+
+  def test_blacklisted_procs(self):
+    # TODO: ensure there are blacklisted procs until we have a dedicated test
+    assert len(BLACKLIST_PROCS), "No blacklisted procs to test not_run"
+
+  @pytest.mark.linux
+  def test_set_params_with_default_value(self):
+    params = Params()
+    params.clear_all()
+
+    os.environ['PREPAREONLY'] = '1'
+    manager.main()
+    for k in params.all_keys():
+      default_value = params.get_default_value(k)
+      if default_value is not None:
+        assert params.get(k) == default_value
+    assert params.get("OpenpilotEnabledToggle")
+    assert params.get("RouteCount") == 0
+
+  @pytest.mark.tici
+  def test_clean_exit(self, subtests):
+    """
+      Ensure all processes exit cleanly when stopped.
+    """
+    HARDWARE.set_power_save(False)
+    manager.manager_init()
+
+    CP = car.CarParams.new_message()
+    procs = ensure_running(managed_processes.values(), True, Params(), CP, not_run=BLACKLIST_PROCS)
+
+    time.sleep(10)
+
+    for p in procs:
+      with subtests.test(proc=p.name):
+        state = p.get_process_state_msg()
+        assert state.running, f"{p.name} not running"
+        exit_code = p.stop(retry=False)
+
+        assert p.name not in BLACKLIST_PROCS, f"{p.name} was started"
+
+        assert exit_code is not None, f"{p.name} failed to exit"
+
+        # TODO: interrupted blocking read exits with 1 in cereal. use a more unique return code
+        exit_codes = [0, 1]
+        if p.sigkill:
+          exit_codes = [-signal.SIGKILL]
+        assert exit_code in exit_codes, f"{p.name} died with {exit_code}"