IQ.Pilot Release Commit @ e46d557

system/manager/process.py

@@ -17,6 +17,10 @@ from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
 from openpilot.common.swaglog import cloudlog
 
+MAX_CRASH_BACKOFF = 300.0
+CRASH_RESET_TIME = 60.0
+CRASH_LOOP_THRESHOLD = 6
+
 try:
   from openpilot.system.proprietary_runtime.runtime_paths import preferred_runner_path
 except ModuleNotFoundError:
@@ -82,6 +86,10 @@ class ManagerProcess(ABC):
   name = ""
   shutting_down = False
   restart_if_crash = False
+  crash_count = 0
+  last_restart_time = 0.0
+  last_alive_time = 0.0
+  crash_loop_logged = False
 
   @abstractmethod
   def prepare(self) -> None:
@@ -318,13 +326,33 @@ def ensure_running(procs: ValuesView[ManagerProcess], started: bool, params=None
     not_run = []
 
   running = []
+  now = time.monotonic()
   for p in procs:
     if p.enabled and p.name not in not_run and p.should_run(started, params, CP):
-      if p.restart_if_crash and p.proc is not None and not p.proc.is_alive():
-        cloudlog.error(f'Restarting {p.name} (exitcode {p.proc.exitcode})')
-        p.restart()
+      if p.restart_if_crash and p.proc is not None and p.proc.is_alive():
+        p.last_alive_time = now
+      elif p.restart_if_crash and p.proc is not None:
+        # uptime, not time-since-restart: the latter also counts the backoff wait,
+        # which would reset the counter as soon as backoff exceeds CRASH_RESET_TIME
+        if p.last_alive_time - p.last_restart_time > CRASH_RESET_TIME:
+          p.crash_count = 0
+          p.crash_loop_logged = False
+
+        backoff = 0.0 if not p.crash_count else min(MAX_CRASH_BACKOFF, 2.0 ** (p.crash_count - 1))
+        if now - p.last_restart_time >= backoff:
+          p.crash_count += 1
+          p.last_restart_time = now
+          cloudlog.error(f'Restarting {p.name} (exitcode {p.proc.exitcode}) [crash {p.crash_count}]')
+          if p.crash_count >= CRASH_LOOP_THRESHOLD and not p.crash_loop_logged:
+            # never stop retrying: giving up on hardwared or ui is worse than restarting slowly
+            cloudlog.error(f'{p.name} is in a crash loop, backing off to {MAX_CRASH_BACKOFF}s between restarts')
+            p.crash_loop_logged = True
+          p.restart()
       running.append(p)
     else:
+      p.crash_count = 0
+      p.crash_loop_logged = False
+      p.last_alive_time = 0.0
       p.stop(block=False)
 
   for p in running:

system/manager/process_config.py

@@ -98,7 +98,9 @@ def constructiond_onroad(started: bool, params: Params, CP: car.CarParams) -> bo
   return started and params.get_bool("ConstructionZoneAssist")
 
 def iqvd_onroad(started: bool, params: Params, CP: car.CarParams) -> bool:
-  return started and params.get_bool("VisionVehicleTracks")
+  # held for 1.0d: iqvd runs a detector per frame and the added load is not
+  # something 1.0c needs to carry. re-enable by restoring the param check.
+  return False
 
 def only_offroad(started: bool, params: Params, CP: car.CarParams) -> bool:
   return not started
@@ -109,6 +111,12 @@ def livestream(started: bool, params: Params, CP: car.CarParams) -> bool:
   # down cleanly when the session ends — no subprocess management inside hephaestusd.
   return params.get_bool("IsLiveStreaming")
 
+def canlive(started: bool, params: Params, CP: car.CarParams) -> bool:
+  # Remote live CAN debugging via konn3kt. hephaestusd sets CanLiveStreaming when a viewer
+  # connects (startCanLive) and clears it when the last one leaves (stopCanLive), so canlived
+  # runs only during an active debug session — no idle connection or battery cost otherwise.
+  return params.get_bool("CanLiveStreaming")
+
 def is_tinygrad_model(started, params, CP: car.CarParams) -> bool:
   """Check if the active model runner is tinygrad."""
   return bool(get_active_model_runner(params, not started) == custom.IQModelManager.Runner.tinygrad)
@@ -179,6 +187,7 @@ procs = [
   # debug procs
   NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar),
   PythonProcess("webrtcd", "system.webrtc.webrtcd", or_(iscar, livestream)),
+  PythonProcess("canlived", "iqpilot.konn3kt.canlive.canlived", canlive),
   PythonProcess("webjoystick", "tools.bodyteleop.web", notcar),
 ]