IQ.Pilot Prebuilt Release @ 27f668a

iqpilot/selfdrive/test/.gitignore

@@ -0,0 +1,9 @@
+out/
+docker_out/
+
+process_replay/diff.txt
+process_replay/model_diff.txt
+valgrind_logs.txt
+
+*.bz2
+*.hevc

iqpilot/selfdrive/test/cpp_harness.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+from pathlib import Path
+
+from iqpilot.common.prefix import OpenpilotPrefix
+
+command = [str(Path(sys.argv[1]).resolve()), *sys.argv[2:]]
+with OpenpilotPrefix():
+  ret = subprocess.call(command)
+
+sys.exit(ret)

iqpilot/selfdrive/test/docker_build.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e
+
+# To build sim and docs, you can run the following to mount the scons cache to the same place as in CI:
+# mkdir -p .ci_cache/scons_cache
+# sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache
+
+SCRIPT_DIR=$(dirname "$0")
+OPENPILOT_DIR=$SCRIPT_DIR/../../../
+if [ -n "$TARGET_ARCHITECTURE" ]; then
+  PLATFORM="linux/$TARGET_ARCHITECTURE"
+  TAG_SUFFIX="-$TARGET_ARCHITECTURE"
+else
+  PLATFORM="linux/$(uname -m)"
+  TAG_SUFFIX=""
+fi
+
+source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX"
+
+DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
+
+if [ -n "$PUSH_IMAGE" ]; then
+  docker push $REMOTE_TAG
+  docker tag $REMOTE_TAG $REMOTE_SHA_TAG
+  docker push $REMOTE_SHA_TAG
+fi

iqpilot/selfdrive/test/docker_common.sh

@@ -0,0 +1,18 @@
+if [ "$1" = "base" ]; then
+  export DOCKER_IMAGE=openpilot-base
+  export DOCKER_FILE=Dockerfile.openpilot_base
+elif [ "$1" = "prebuilt" ]; then
+  export DOCKER_IMAGE=openpilot-prebuilt
+  export DOCKER_FILE=Dockerfile.openpilot
+else
+  echo "Invalid docker build image: '$1'"
+  exit 1
+fi
+
+export DOCKER_REGISTRY=ghcr.io/commaai
+export COMMIT_SHA=$(git rev-parse HEAD)
+
+TAG_SUFFIX=$2
+LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
+REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
+REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA

iqpilot/selfdrive/test/fuzzy_generation.py

@@ -0,0 +1,81 @@
+import capnp
+import hypothesis.strategies as st
+from typing import Any
+from collections.abc import Callable
+from functools import cache
+
+from iqpilot.cereal import log
+
+DrawType = Callable[[st.SearchStrategy], Any]
+
+
+class FuzzyGenerator:
+  def __init__(self, draw: DrawType, real_floats: bool):
+    self.draw = draw
+    self.native_type_map = FuzzyGenerator._get_native_type_map(real_floats)
+
+  def generate_native_type(self, field: str) -> st.SearchStrategy[bool | int | float | str | bytes]:
+    value_func = self.native_type_map.get(field)
+    if value_func is not None:
+      return value_func
+    else:
+      raise NotImplementedError(f'Invalid type: {field}')
+
+  def generate_field(self, field: capnp.lib.capnp._StructSchemaField) -> st.SearchStrategy:
+    def rec(field_type: capnp.lib.capnp._DynamicStructReader) -> st.SearchStrategy:
+      type_which = field_type.which()
+      if type_which == 'struct':
+        return self.generate_struct(field.schema.elementType if base_type == 'list' else field.schema)
+      elif type_which == 'list':
+        return st.lists(rec(field_type.list.elementType))
+      elif type_which == 'enum':
+        schema = field.schema.elementType if base_type == 'list' else field.schema
+        return st.sampled_from(list(schema.enumerants.keys()))
+      else:
+        return self.generate_native_type(type_which)
+
+    try:
+      if hasattr(field.proto, 'slot'):
+        slot_type =  field.proto.slot.type
+        base_type = slot_type.which()
+        return rec(slot_type)
+      else:
+        return self.generate_struct(field.schema)
+    except capnp.lib.capnp.KjException:
+      return self.generate_struct(field.schema)
+
+  def generate_struct(self, schema: capnp.lib.capnp._StructSchema, event: str | None = None) -> st.SearchStrategy[dict[str, Any]]:
+    single_fill: tuple[str, ...] = (event,) if event else (self.draw(st.sampled_from(schema.union_fields)),) if schema.union_fields else ()
+    fields_to_generate = schema.non_union_fields + single_fill
+    return st.fixed_dictionaries({field: self.generate_field(schema.fields[field]) for field in fields_to_generate if not field.endswith('DEPRECATED')})
+
+  @staticmethod
+  @cache
+  def _get_native_type_map(real_floats: bool) -> dict[str, st.SearchStrategy]:
+    return {
+      'bool': st.booleans(),
+      'int8': st.integers(min_value=-2**7, max_value=2**7-1),
+      'int16': st.integers(min_value=-2**15, max_value=2**15-1),
+      'int32': st.integers(min_value=-2**31, max_value=2**31-1),
+      'int64': st.integers(min_value=-2**63, max_value=2**63-1),
+      'uint8': st.integers(min_value=0, max_value=2**8-1),
+      'uint16': st.integers(min_value=0, max_value=2**16-1),
+      'uint32': st.integers(min_value=0, max_value=2**32-1),
+      'uint64': st.integers(min_value=0, max_value=2**64-1),
+      'float32': st.floats(width=32, allow_nan=not real_floats, allow_infinity=not real_floats),
+      'float64': st.floats(width=64, allow_nan=not real_floats, allow_infinity=not real_floats),
+      'text': st.text(max_size=1000),
+      'data': st.binary(max_size=1000),
+      'anyPointer': st.text(),  # Note: No need to define a separate function for anyPointer
+    }
+
+  @classmethod
+  def get_random_msg(cls, draw: DrawType, struct: capnp.lib.capnp._StructModule, real_floats: bool = False) -> dict[str, Any]:
+    fg = cls(draw, real_floats=real_floats)
+    data: dict[str, Any] = draw(fg.generate_struct(struct.schema))
+    return data
+
+  @classmethod
+  def get_random_event_msg(cls, draw: DrawType, events: list[str], real_floats: bool = False) -> list[dict[str, Any]]:
+    fg = cls(draw, real_floats=real_floats)
+    return [draw(fg.generate_struct(log.Event.schema, e)) for e in sorted(events)]

iqpilot/selfdrive/test/helpers.py

@@ -0,0 +1,123 @@
+import contextlib
+import http.server
+import os
+import threading
+import time
+import pytest
+
+from functools import wraps
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.common.params import Params
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.system.version import training_version, terms_version
+
+
+def set_params_enabled():
+  os.environ['FINGERPRINT'] = "TOYOTA_COROLLA_TSS2"
+  os.environ['LOGPRINT'] = "debug"
+
+  params = Params()
+  params.put("HasAcceptedTerms", terms_version)
+  params.put("CompletedTrainingVersion", training_version)
+  params.put_bool("OpenpilotEnabledToggle", True)
+
+  # valid calib
+  msg = messaging.new_message('extrinsicsCalibration')
+  msg.extrinsicsCalibration.validBlocks = 20
+  msg.extrinsicsCalibration.rpyCalib = [0.0, 0.0, 0.0]
+  params.put("CalibrationParams", msg.to_bytes())
+
+def release_only(f):
+  return pytest.mark.release(f)
+
+
+@contextlib.contextmanager
+def processes_context(processes, init_time=0, ignore_stopped=None):
+  ignore_stopped = [] if ignore_stopped is None else ignore_stopped
+
+  # start and assert started
+  for n, p in enumerate(processes):
+    managed_processes[p].start()
+    if n < len(processes) - 1:
+      time.sleep(init_time)
+
+  assert all(managed_processes[name].proc.exitcode is None for name in processes)
+
+  try:
+    yield [managed_processes[name] for name in processes]
+    # assert processes are still started
+    assert all(managed_processes[name].proc.exitcode is None for name in processes if name not in ignore_stopped)
+  finally:
+    for p in processes:
+      managed_processes[p].stop()
+
+
+def with_processes(processes, init_time=0, ignore_stopped=None):
+  def wrapper(func):
+    @wraps(func)
+    def wrap(*args, **kwargs):
+      with processes_context(processes, init_time, ignore_stopped):
+        return func(*args, **kwargs)
+
+    return wrap
+  return wrapper
+
+
+def noop(*args, **kwargs):
+  pass
+
+
+def read_segment_list(segment_list_path):
+  with open(segment_list_path) as f:
+    seg_list = f.read().splitlines()
+
+  return [(platform[2:], segment) for platform, segment in zip(seg_list[::2], seg_list[1::2], strict=True)]
+
+
+@contextlib.contextmanager
+def http_server_context(handler, setup=None):
+  host = '127.0.0.1'
+  server = http.server.HTTPServer((host, 0), handler)
+  port = server.server_port
+  t = threading.Thread(target=server.serve_forever)
+  t.start()
+
+  if setup is not None:
+    setup(host, port)
+
+  try:
+    yield (host, port)
+  finally:
+    server.shutdown()
+    server.server_close()
+    t.join()
+
+
+def with_http_server(func, handler=http.server.BaseHTTPRequestHandler, setup=None):
+  @wraps(func)
+  def inner(*args, **kwargs):
+    with http_server_context(handler, setup) as (host, port):
+      return func(*args, f"http://{host}:{port}", **kwargs)
+  return inner
+
+
+def DirectoryHttpServer(directory) -> type[http.server.SimpleHTTPRequestHandler]:
+  # creates an http server that serves files from directory
+  class Handler(http.server.SimpleHTTPRequestHandler):
+    API_NO_RESPONSE = False
+    API_BAD_RESPONSE = False
+
+    def do_GET(self):
+      if self.API_NO_RESPONSE:
+        return
+
+      if self.API_BAD_RESPONSE:
+        self.send_response(500, "")
+        return
+      super().do_GET()
+
+    def __init__(self, *args, **kwargs):
+      super().__init__(*args, directory=str(directory), **kwargs)
+
+  return Handler

iqpilot/selfdrive/test/longitudinal_maneuvers/.gitignore

@@ -0,0 +1 @@
+out/*

iqpilot/selfdrive/test/longitudinal_maneuvers/maneuver.py

@@ -0,0 +1,88 @@
+import numpy as np
+from iqpilot.selfdrive.test.longitudinal_maneuvers.plant import Plant
+
+
+class Maneuver:
+  def __init__(self, title, duration, **kwargs):
+    # Was tempted to make a builder class
+    self.distance_lead = kwargs.get("initial_distance_lead", 200.0)
+    self.speed = kwargs.get("initial_speed", 0.0)
+    self.lead_relevancy = kwargs.get("lead_relevancy", 0)
+
+    self.breakpoints = kwargs.get("breakpoints", [0.0, duration])
+    self.speed_lead_values = kwargs.get("speed_lead_values", [0.0 for i in range(len(self.breakpoints))])
+    self.prob_lead_values = kwargs.get("prob_lead_values", [1.0 for i in range(len(self.breakpoints))])
+    self.prob_throttle_values = kwargs.get("prob_throttle_values", [1.0 for i in range(len(self.breakpoints))])
+    self.cruise_values = kwargs.get("cruise_values", [50.0 for i in range(len(self.breakpoints))])
+    self.pitch_values = kwargs.get("pitch_values", [0.0 for i in range(len(self.breakpoints))])
+
+    self.only_lead2 = kwargs.get("only_lead2", False)
+    self.only_radar = kwargs.get("only_radar", False)
+    self.ensure_start = kwargs.get("ensure_start", False)
+    self.ensure_slowdown = kwargs.get("ensure_slowdown", False)
+    self.enabled = kwargs.get("enabled", True)
+    self.e2e = kwargs.get("e2e", False)
+    self.personality = kwargs.get("personality", 0)
+    self.force_decel = kwargs.get("force_decel", False)
+
+    self.duration = duration
+    self.title = title
+
+  def evaluate(self):
+    plant = Plant(
+      lead_relevancy=self.lead_relevancy,
+      speed=self.speed,
+      distance_lead=self.distance_lead,
+      enabled=self.enabled,
+      only_lead2=self.only_lead2,
+      only_radar=self.only_radar,
+      e2e=self.e2e,
+      personality=self.personality,
+      force_decel=self.force_decel,
+    )
+
+    valid = True
+    started = not self.ensure_start
+    logs = []
+    while plant.current_time < self.duration:
+      speed_lead = np.interp(plant.current_time, self.breakpoints, self.speed_lead_values)
+      prob_lead = np.interp(plant.current_time, self.breakpoints, self.prob_lead_values)
+      cruise = np.interp(plant.current_time, self.breakpoints, self.cruise_values)
+      pitch = np.interp(plant.current_time, self.breakpoints, self.pitch_values)
+      prob_throttle = np.interp(plant.current_time, self.breakpoints, self.prob_throttle_values)
+      log = plant.step(speed_lead, prob_lead, cruise, pitch, prob_throttle)
+
+      d_rel = log['distance_lead'] - log['distance'] if self.lead_relevancy else 200.
+      v_rel = speed_lead - log['speed'] if self.lead_relevancy else 0.
+      log['d_rel'] = d_rel
+      log['v_rel'] = v_rel
+      logs.append(np.array([plant.current_time,
+                            log['distance'],
+                            log['distance_lead'],
+                            log['speed'],
+                            speed_lead,
+                            log['acceleration'],
+                            log['d_rel']]))
+
+      if d_rel < .4 and (self.only_radar or prob_lead > 0.5):
+        print("Crashed!!!!")
+        valid = False
+
+      if self.ensure_start and log['v_rel'] > 0 and log['acceleration'] >= 1e-3:
+        started = True
+
+    if self.ensure_slowdown and log['speed'] > 5.5:
+      print('LongitudinalPlanner not slowing down!')
+      valid = False
+
+    if not started:
+      print('LongitudinalPlanner not starting!')
+      valid = False
+
+    if self.force_decel and log['speed'] > 1e-1 and log['acceleration'] > -0.04:
+      print('Not stopping with force decel')
+      valid = False
+
+
+    print("maneuver end", valid)
+    return valid, np.array(logs)

iqpilot/selfdrive/test/longitudinal_maneuvers/plant.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import time
+import numpy as np
+
+from iqpilot.cereal import log
+import iqpilot.cereal.messaging as messaging
+from iqpilot.common.realtime import Ratekeeper, DT_MDL
+from iqpilot.selfdrive.controls.lib.longcontrol import LongCtrlState
+from iqpilot.selfdrive.iqmodeld.config import ModelConstants
+from iqpilot.selfdrive.controls.lib.longitudinal_planner import LongitudinalPlanner
+from iqpilot.selfdrive.controls.radard import _LEAD_ACCEL_TAU
+
+
+class Plant:
+  messaging_initialized = False
+
+  def __init__(self, lead_relevancy=False, speed=0.0, distance_lead=2.0,
+               enabled=True, only_lead2=False, only_radar=False, e2e=False, personality=0, force_decel=False):
+    self.rate = 1. / DT_MDL
+
+    if not Plant.messaging_initialized:
+      Plant.radar = messaging.pub_sock('radarState')
+      Plant.controls_state = messaging.pub_sock('controlsState')
+      Plant.selfdrive_state = messaging.pub_sock('selfdriveState')
+      Plant.car_state = messaging.pub_sock('carState')
+      Plant.plan = messaging.sub_sock('longitudinalPlan')
+      Plant.messaging_initialized = True
+
+    self.v_lead_prev = 0.0
+
+    self.distance = 0.
+    self.speed = speed
+    self.should_stop = False
+    self.acceleration = 0.0
+
+    # lead car
+    self.lead_relevancy = lead_relevancy
+    self.distance_lead = distance_lead
+    self.enabled = enabled
+    self.only_lead2 = only_lead2
+    self.only_radar = only_radar
+    self.e2e = e2e
+    self.personality = personality
+    self.force_decel = force_decel
+
+    self.rk = Ratekeeper(self.rate, print_delay_threshold=100.0)
+    self.ts = 1. / self.rate
+    time.sleep(0.1)
+    self.sm = messaging.SubMaster(['longitudinalPlan'])
+
+    from iqdbc.car.honda.values import CAR
+    from iqdbc.car.honda.interface import CarInterface
+
+    CP = CarInterface.get_non_essential_params(CAR.HONDA_CIVIC)
+    CP_IQ = CarInterface.get_non_essential_params_iq(CP, CAR.HONDA_CIVIC)
+    self.planner = LongitudinalPlanner(CP, CP_IQ, init_v=self.speed)
+
+  @property
+  def current_time(self):
+    return float(self.rk.frame) / self.rate
+
+  def step(self, v_lead=0.0, prob_lead=1.0, v_cruise=50., pitch=0.0, prob_throttle=1.0):
+    # ******** publish a fake model going straight and fake calibration ********
+    # note that this is worst case for MPC, since model will delay long mpc by one time step
+    radar = messaging.new_message('radarState')
+    control = messaging.new_message('controlsState')
+    ss = messaging.new_message('selfdriveState')
+    car_state = messaging.new_message('carState')
+    lp = messaging.new_message('vehicleParameters')
+    car_control = messaging.new_message('carControl')
+    model = messaging.new_message('modelV2')
+    iq_car_state = messaging.new_message('iqCarState')
+    iq_nav_state = messaging.new_message('iqNavState')
+    live_map_data_iq = messaging.new_message('iqLiveData')
+    gps_data = messaging.new_message('gpsLocation')
+    a_lead = (v_lead - self.v_lead_prev)/self.ts
+    self.v_lead_prev = v_lead
+
+    if self.lead_relevancy:
+      d_rel = np.maximum(0., self.distance_lead - self.distance)
+      v_rel = v_lead - self.speed
+      if self.only_radar:
+        status = True
+      elif prob_lead > .5:
+        status = True
+      else:
+        status = False
+    else:
+      d_rel = 200.
+      v_rel = 0.
+      prob_lead = 0.0
+      status = False
+
+    lead = log.RadarState.LeadData.new_message()
+    lead.dRel = float(d_rel)
+    lead.yRel = 0.0
+    lead.vRel = float(v_rel)
+    lead.aRel = float(a_lead - self.acceleration)
+    lead.vLead = float(v_lead)
+    lead.vLeadK = float(v_lead)
+    lead.aLeadK = float(a_lead)
+    # TODO use real radard logic for this
+    lead.aLeadTau = float(_LEAD_ACCEL_TAU)
+    lead.status = status
+    lead.modelProb = float(prob_lead)
+    if not self.only_lead2:
+      radar.radarState.leadOne = lead
+    radar.radarState.leadTwo = lead
+
+    # Simulate model predicting slightly faster speed
+    # this is to ensure lead policy is effective when model
+    # does not predict slowdown in e2e mode
+    position = log.XYZTData.new_message()
+    position.x = [float(x) for x in (self.speed + 0.5) * np.array(ModelConstants.T_IDXS)]
+    model.modelV2.position = position
+    model.modelV2.action.desiredAcceleration = float(self.acceleration + 0.5)
+    velocity = log.XYZTData.new_message()
+    velocity.x = [float(x) for x in (self.speed + 0.5) * np.ones_like(ModelConstants.T_IDXS)]
+    velocity.x[0] = float(self.speed) # always start at current speed
+    model.modelV2.velocity = velocity
+    acceleration = log.XYZTData.new_message()
+    acceleration.x = [float(x) for x in np.zeros_like(ModelConstants.T_IDXS)]
+    model.modelV2.acceleration = acceleration
+    model.modelV2.meta.disengagePredictions.gasPressProbs = [float(prob_throttle) for _ in range(6)]
+    lead_times = np.asarray(ModelConstants.LEAD_T_IDXS)
+    lead_accel = np.clip(a_lead, -10.0, 5.0)
+    stop_time = -v_lead / lead_accel if lead_accel < 0.0 else np.inf
+    motion_times = np.minimum(lead_times, stop_time)
+    lead_positions = d_rel + v_lead * motion_times + 0.5 * lead_accel * motion_times**2
+    lead_velocities = np.maximum(v_lead + lead_accel * lead_times, 0.0)
+    for lead_prediction in model.modelV2.leadsV3:
+      lead_prediction.prob = float(prob_lead)
+      lead_prediction.x = [float(x) for x in lead_positions]
+      lead_prediction.v = [float(v) for v in lead_velocities]
+
+    control.controlsState.longControlState = LongCtrlState.pid if self.enabled else LongCtrlState.off
+    ss.selfdriveState.experimentalMode = self.e2e
+    ss.selfdriveState.personality = self.personality
+    control.controlsState.forceDecel = self.force_decel
+    car_state.carState.vEgo = float(self.speed)
+    car_state.carState.standstill = bool(self.speed < 0.01)
+    car_state.carState.vCruise = float(v_cruise * 3.6)
+    car_control.carControl.orientationNED = [0., float(pitch), 0.]
+
+    # ******** get controlsState messages for plotting ***
+    sm = {'radarState': radar.radarState,
+          'carState': car_state.carState,
+          'carControl': car_control.carControl,
+          'controlsState': control.controlsState,
+          'selfdriveState': ss.selfdriveState,
+          'vehicleParameters': lp.vehicleParameters,
+          'modelV2': model.modelV2,
+          'iqCarState': iq_car_state.iqCarState,
+          'iqNavState': iq_nav_state.iqNavState,
+          'iqLiveData': live_map_data_iq.iqLiveData,
+          'gpsLocation': gps_data.gpsLocation}
+    self.planner.update(sm)
+    self.acceleration = self.planner.output_a_target
+    if self.planner.output_should_stop:
+      self.acceleration = min(-0.5, self.acceleration)
+    self.speed = self.speed + self.acceleration * self.ts
+    self.should_stop = self.planner.output_should_stop
+    fcw = self.planner.fcw
+    self.distance_lead = self.distance_lead + v_lead * self.ts
+
+    # ******** run the car ********
+    #print(self.distance, speed)
+    if self.speed <= 0:
+      self.speed = 0
+      self.acceleration = 0
+    self.distance = self.distance + self.speed * self.ts
+
+    # *** radar model ***
+    if self.lead_relevancy:
+      d_rel = np.maximum(0., self.distance_lead - self.distance)
+      v_rel = v_lead - self.speed
+    else:
+      d_rel = 200.
+      v_rel = 0.
+
+    # print at 5hz
+    # if (self.rk.frame % (self.rate // 5)) == 0:
+    #   print("%2.2f sec   %6.2f m  %6.2f m/s  %6.2f m/s2   lead_rel: %6.2f m  %6.2f m/s"
+    #         % (self.current_time, self.distance, self.speed, self.acceleration, d_rel, v_rel))
+
+
+    # ******** update prevs ********
+    self.rk.monitor_time()
+
+    return {
+      "distance": self.distance,
+      "speed": self.speed,
+      "acceleration": self.acceleration,
+      "should_stop": self.should_stop,
+      "distance_lead": self.distance_lead,
+      "fcw": fcw,
+    }
+
+# simple engage in standalone mode
+def plant_thread():
+  plant = Plant()
+  while 1:
+    plant.step()
+
+
+if __name__ == "__main__":
+  plant_thread()

iqpilot/selfdrive/test/longitudinal_maneuvers/test_longitudinal.py

@@ -0,0 +1,191 @@
+import itertools
+from parameterized import parameterized_class
+
+from iqpilot.selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import STOP_DISTANCE
+from iqpilot.selfdrive.test.longitudinal_maneuvers.maneuver import Maneuver
+
+
+# TODO: make new FCW tests
+def create_maneuvers(kwargs):
+  maneuvers = [
+    Maneuver(
+      'approach stopped car at 25m/s, initial distance: 120m',
+      duration=20.,
+      initial_speed=25.,
+      lead_relevancy=True,
+      initial_distance_lead=120.,
+      speed_lead_values=[30., 0.],
+      breakpoints=[0., 1.],
+      **kwargs,
+    ),
+    Maneuver(
+      'approach stopped car at 20m/s, initial distance 90m',
+      duration=20.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=90.,
+      speed_lead_values=[20., 0.],
+      breakpoints=[0., 1.],
+      **kwargs,
+    ),
+    Maneuver(
+      'steady state following a car at 20m/s, then lead decel to 0mph at 1m/s^2',
+      duration=50.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=35.,
+      speed_lead_values=[20., 20., 0.],
+      breakpoints=[0., 15., 35.0],
+      **kwargs,
+    ),
+    Maneuver(
+      'steady state following a car at 20m/s, then lead decel to 0mph at 2m/s^2',
+      duration=50.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=35.,
+      speed_lead_values=[20., 20., 0.],
+      breakpoints=[0., 15., 25.0],
+      **kwargs,
+    ),
+    Maneuver(
+      'steady state following a car at 20m/s, then lead decel to 0mph at 3m/s^2',
+      duration=50.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=35.,
+      speed_lead_values=[20., 20., 0.],
+      breakpoints=[0., 15., 21.66],
+      **kwargs,
+    ),
+    Maneuver(
+      'steady state following a car at 20m/s, then lead decel to 0mph at 3+m/s^2',
+      duration=40.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=35.,
+      speed_lead_values=[20., 20., 0.],
+      prob_lead_values=[0., 1., 1.],
+      cruise_values=[20., 20., 20.],
+      breakpoints=[2., 2.01, 8.8],
+      **kwargs,
+    ),
+    Maneuver(
+      "approach stopped car at 20m/s, with prob_lead_values",
+      duration=30.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=120.,
+      speed_lead_values=[0.0, 0., 0.],
+      prob_lead_values=[0.0, 0., 1.],
+      cruise_values=[20., 20., 20.],
+      breakpoints=[0.0, 2., 2.01],
+      **kwargs,
+    ),
+    Maneuver(
+      "approach stopped car at 20m/s, with prob_throttle_values and pitch = -0.1",
+      duration=30.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=120.,
+      speed_lead_values=[0.0, 0., 0.],
+      prob_throttle_values=[1., 0., 0.],
+      cruise_values=[20., 20., 20.],
+      pitch_values=[0., -0.1, -0.1],
+      breakpoints=[0.0, 2., 2.01],
+      **kwargs,
+    ),
+    Maneuver(
+      "approach stopped car at 20m/s, with prob_throttle_values and pitch = +0.1",
+      duration=30.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=120.,
+      speed_lead_values=[0.0, 0., 0.],
+      prob_throttle_values=[1., 0., 0.],
+      cruise_values=[20., 20., 20.],
+      pitch_values=[0., 0.1, 0.1],
+      breakpoints=[0.0, 2., 2.01],
+      **kwargs,
+    ),
+    Maneuver(
+      "approach slower cut-in car at 20m/s",
+      duration=20.,
+      initial_speed=20.,
+      lead_relevancy=True,
+      initial_distance_lead=50.,
+      speed_lead_values=[15., 15.],
+      breakpoints=[1., 11.],
+      only_lead2=True,
+      **kwargs,
+    ),
+    Maneuver(
+      "stay stopped behind radar override lead",
+      duration=20.,
+      initial_speed=0.,
+      lead_relevancy=True,
+      initial_distance_lead=10.,
+      speed_lead_values=[0., 0.],
+      prob_lead_values=[0., 0.],
+      breakpoints=[1., 11.],
+      only_radar=True,
+      **kwargs,
+    ),
+    Maneuver(
+      "NaN recovery",
+      duration=30.,
+      initial_speed=15.,
+      lead_relevancy=True,
+      initial_distance_lead=60.,
+      speed_lead_values=[0., 0., 0.0],
+      breakpoints=[1., 1.01, 11.],
+      cruise_values=[float("nan"), 15., 15.],
+      **kwargs,
+    ),
+    Maneuver(
+      'cruising at 25 m/s while disabled',
+      duration=20.,
+      initial_speed=25.,
+      lead_relevancy=False,
+      enabled=False,
+      **kwargs,
+    ),
+    Maneuver(
+      "slow to 5m/s with allow_throttle = False and pitch = +0.1",
+      duration=30.,
+      initial_speed=20.,
+      lead_relevancy=False,
+      prob_throttle_values=[1., 0., 0.],
+      cruise_values=[20., 20., 20.],
+      pitch_values=[0., 0.1, 0.1],
+      breakpoints=[0.0, 2., 2.01],
+      ensure_slowdown=True,
+      **kwargs,
+    )]
+  if not kwargs['force_decel']:
+    # controls relies on planner commanding to move for stock-ACC resume spamming
+    maneuvers.append(Maneuver(
+      "resume from a stop",
+      duration=20.,
+      initial_speed=0.,
+      lead_relevancy=True,
+      initial_distance_lead=STOP_DISTANCE,
+      speed_lead_values=[0., 0., 2.],
+      breakpoints=[1., 10., 15.],
+      ensure_start=True,
+      **kwargs,
+    ))
+  return maneuvers
+
+
+@parameterized_class(("e2e", "force_decel"), itertools.product([True, False], repeat=2))
+class TestLongitudinalControl:
+  e2e: bool
+  force_decel: bool
+
+  def test_maneuver(self, subtests):
+    for maneuver in create_maneuvers({"e2e": self.e2e, "force_decel": self.force_decel}):
+      with subtests.test(title=maneuver.title, e2e=maneuver.e2e, force_decel=maneuver.force_decel):
+        print(maneuver.title, f'in {"e2e" if maneuver.e2e else "acc"} mode')
+        valid, _ = maneuver.evaluate()
+        assert valid

iqpilot/selfdrive/test/process_replay/.gitignore

@@ -0,0 +1 @@
+fakedata/

iqpilot/selfdrive/test/process_replay/README.md

@@ -0,0 +1,122 @@
+# Process replay
+
+Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment.
+
+If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated.
+
+Use `test_processes.py` to run the test locally.
+Log files are cached by default. Use `DISABLE_FILEREADER_CACHE='1' test_processes.py` to disable caching.
+
+Currently the following processes are tested:
+
+* controlsd
+* radard
+* plannerd
+* calibrationd
+* dmonitoringd
+* locationd
+* ubloxd
+
+### Usage
+```
+Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS]
+                         [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--update-refs] [--upload-only]
+Regression test to identify changes in a process's output
+optional arguments:
+  -h, --help            show this help message and exit
+  --whitelist-procs PROCS               Whitelist given processes from the test (e.g. controlsd)
+  --whitelist-cars WHITELIST_CARS       Whitelist given cars from the test (e.g. HONDA)
+  --blacklist-procs BLACKLIST_PROCS     Blacklist given processes from the test (e.g. controlsd)
+  --blacklist-cars BLACKLIST_CARS       Blacklist given cars from the test (e.g. HONDA)
+  --ignore-fields IGNORE_FIELDS         Extra fields or msgs to ignore (e.g. driverMonitoringState.events)
+  --ignore-msgs IGNORE_MSGS             Msgs to ignore (e.g. onroadEvents)
+  --update-refs                         Updates reference logs using current commit
+  --upload-only                         Skips testing processes and uploads logs from previous test run
+```
+
+## Forks
+
+openpilot forks can use this test with their own reference logs, by default `test_proccesses.py` saves logs locally.
+
+To generate new logs:
+
+`./test_processes.py`
+
+Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit.
+
+## API
+
+Process replay test suite exposes programmatic APIs for simultaneously running processes or groups of processes on provided logs.
+
+```py
+def replay_process_with_name(name: Union[str, Iterable[str]], lr: LogIterable, *args, **kwargs) -> List[capnp._DynamicStructReader]:
+
+def replay_process(
+  cfg: Union[ProcessConfig, Iterable[ProcessConfig]], lr: LogIterable, frs: Optional[Dict[str, Any]] = None,
+  fingerprint: Optional[str] = None, return_all_logs: bool = False, custom_params: Optional[Dict[str, Any]] = None, disable_progress: bool = False
+) -> List[capnp._DynamicStructReader]:
+```
+
+Example usage:
+```py
+from iqpilot.selfdrive.test.process_replay import replay_process_with_name
+from iqpilot.tools.lib.logreader import LogReader
+
+lr = LogReader(...)
+
+# provide a name of the process to replay
+output_logs = replay_process_with_name('locationd', lr)
+
+# or list of names
+output_logs = replay_process_with_name(['ubloxd', 'locationd'], lr)
+```
+
+Supported processes:
+* controlsd
+* radard
+* plannerd
+* calibrationd
+* dmonitoringd
+* locationd
+* ubloxd
+* modeld
+* dmonitoringmodeld
+
+Certain processes may require an initial state, which is usually supplied within `Params` and persisting from segment to segment (e.g CalibrationParams, LiveParameters). The `custom_params` is dictionary  used to prepopulate `Params` with arbitrary values. The `get_custom_params_from_lr` helper is provided to fetch meaningful values from log files.
+
+```py
+from iqpilot.selfdrive.test.process_replay import get_custom_params_from_lr
+
+previous_segment_lr = LogReader(...)
+current_segment_lr = LogReader(...)
+
+custom_params = get_custom_params_from_lr(previous_segment_lr, 'last')
+
+output_logs = replay_process_with_name('calibrationd', lr, custom_params=custom_params)
+```
+
+Replaying processes that use VisionIPC (e.g. modeld, dmonitoringmodeld) require additional `frs` dictionary with camera states as keys and `FrameReader` objects as values.
+
+```py
+from iqpilot.tools.lib.framereader import FrameReader
+
+frs = {
+  'roadCameraState': FrameReader(...),
+  'wideRoadCameraState': FrameReader(...),
+  'driverCameraState': FrameReader(...),
+}
+
+output_logs = replay_process_with_name(['modeld', 'dmonitoringmodeld'], lr, frs=frs)
+```
+
+To capture stdout/stderr of the replayed process, `captured_output_store` can be provided.
+
+```py
+output_store = dict()
+# pass dictionary by reference, it will be filled with standard outputs - even if process replay fails
+output_logs = replay_process_with_name(['radard', 'plannerd'], lr, captured_output_store=output_store)
+
+# entries with captured output in format { 'out': '...', 'err': '...' } will be added to provided dictionary for each replayed process
+print(output_store['radard']['out']) # radard stdout
+print(output_store['radard']['err']) # radard stderr
+```

iqpilot/selfdrive/test/process_replay/__init__.py

@@ -0,0 +1,2 @@
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, get_process_config, get_custom_params_from_lr, \
+                                                                  replay_process, replay_process_with_name  # noqa: F401

iqpilot/selfdrive/test/process_replay/capture.py

@@ -0,0 +1,59 @@
+import os
+import sys
+
+from typing import no_type_check
+
+class FdRedirect:
+  def __init__(self, file_prefix: str, fd: int):
+    fname = os.path.join("/tmp", f"{file_prefix}.{fd}")
+    if os.path.exists(fname):
+      os.unlink(fname)
+    self.dest_fd = os.open(fname, os.O_WRONLY | os.O_CREAT)
+    self.dest_fname = fname
+    self.source_fd = fd
+    os.set_inheritable(self.dest_fd, True)
+
+  def __del__(self):
+    os.close(self.dest_fd)
+
+  def purge(self) -> None:
+    os.unlink(self.dest_fname)
+
+  def read(self) -> bytes:
+    with open(self.dest_fname, "rb") as f:
+      return f.read() or b""
+
+  def link(self) -> None:
+    os.dup2(self.dest_fd, self.source_fd)
+
+
+class ProcessOutputCapture:
+  def __init__(self, proc_name: str, prefix: str):
+    prefix = f"{proc_name}_{prefix}"
+    self.stdout_redirect = FdRedirect(prefix, 1)
+    self.stderr_redirect = FdRedirect(prefix, 2)
+
+  def __del__(self):
+    self.stdout_redirect.purge()
+    self.stderr_redirect.purge()
+
+  @no_type_check # ipython classes have incompatible signatures
+  def link_with_current_proc(self) -> None:
+    try:
+      # prevent ipykernel from redirecting stdout/stderr of python subprocesses
+      from ipykernel.iostream import OutStream
+      if isinstance(sys.stdout, OutStream):
+        sys.stdout = sys.__stdout__
+      if isinstance(sys.stderr, OutStream):
+        sys.stderr = sys.__stderr__
+    except ImportError:
+      pass
+
+    # link stdout/stderr to the fifo
+    self.stdout_redirect.link()
+    self.stderr_redirect.link()
+
+  def read_outerr(self) -> tuple[str, str]:
+    out_str = self.stdout_redirect.read().decode()
+    err_str = self.stderr_redirect.read().decode()
+    return out_str, err_str

iqpilot/selfdrive/test/process_replay/compare_logs.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+import sys
+import math
+import capnp
+import numbers
+import dictdiffer
+from collections import Counter
+
+from iqpilot.tools.lib.logreader import LogReader
+
+EPSILON = sys.float_info.epsilon
+
+
+def remove_ignored_fields(msg, ignore):
+  msg = msg.as_builder()
+  for key in ignore:
+    attr = msg
+    keys = key.split(".")
+    if msg.which() != keys[0] and len(keys) > 1:
+      continue
+
+    for k in keys[:-1]:
+      # indexing into list
+      if k.isdigit():
+        attr = attr[int(k)]
+      else:
+        attr = getattr(attr, k)
+
+    v = getattr(attr, keys[-1])
+    if isinstance(v, bool):
+      val = False
+    elif isinstance(v, numbers.Number):
+      val = 0
+    elif isinstance(v, (list, capnp.lib.capnp._DynamicListBuilder)):
+      val = []
+    elif isinstance(v, str):
+      val = ""
+    elif isinstance(v, capnp.lib.capnp._DynamicEnum):
+      val = 0
+    else:
+      raise NotImplementedError(f"Unknown type: {type(v)}")
+    setattr(attr, keys[-1], val)
+  return msg
+
+
+def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None,):
+  if ignore_fields is None:
+    ignore_fields = []
+  if ignore_msgs is None:
+    ignore_msgs = []
+  tolerance = EPSILON if tolerance is None else tolerance
+
+  log1, log2 = (
+    [m for m in log if m.which() not in ignore_msgs]
+    for log in (log1, log2)
+  )
+
+  if len(log1) != len(log2):
+    cnt1 = Counter(m.which() for m in log1)
+    cnt2 = Counter(m.which() for m in log2)
+    raise Exception(f"logs are not same length: {len(log1)} VS {len(log2)}\n\t\t{cnt1}\n\t\t{cnt2}")
+
+  diff = []
+  for msg1, msg2 in zip(log1, log2, strict=True):
+    if msg1.which() != msg2.which():
+      raise Exception("msgs not aligned between logs")
+
+    msg1 = remove_ignored_fields(msg1, ignore_fields)
+    msg2 = remove_ignored_fields(msg2, ignore_fields)
+
+    if msg1.to_bytes() != msg2.to_bytes():
+      msg1_dict = msg1.as_reader().to_dict(verbose=True)
+      msg2_dict = msg2.as_reader().to_dict(verbose=True)
+
+      dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
+
+      # Dictdiffer only supports relative tolerance, we also want to check for absolute
+      # TODO: add this to dictdiffer
+      def outside_tolerance(diff):
+        try:
+          if diff[0] == "change":
+            a, b = diff[2]
+            finite = math.isfinite(a) and math.isfinite(b)
+            if finite and isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
+              return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
+        except TypeError:
+          pass
+        return True
+
+      dd = list(filter(outside_tolerance, dd))
+
+      diff.extend(dd)
+  return diff
+
+
+def format_process_diff(diff):
+  diff_short, diff_long = "", ""
+
+  if isinstance(diff, str):
+    diff_short += f"        {diff}\n"
+    diff_long += f"\t{diff}\n"
+  else:
+    cnt: dict[str, int] = {}
+    for d in diff:
+      diff_long += f"\t{str(d)}\n"
+
+      k = str(d[1])
+      cnt[k] = 1 if k not in cnt else cnt[k] + 1
+
+    for k, v in sorted(cnt.items()):
+      diff_short += f"        {k}: {v}\n"
+
+  return diff_short, diff_long
+
+
+def format_diff(results, log_paths, ref_commit):
+  diff_short, diff_long = "", ""
+  diff_long += f"***** tested against commit {ref_commit} *****\n"
+
+  failed = False
+  for segment, result in list(results.items()):
+    diff_short += f"***** results for segment {segment} *****\n"
+    diff_long += f"***** differences for segment {segment} *****\n"
+
+    for proc, diff in list(result.items()):
+      diff_long += f"*** process: {proc} ***\n"
+      diff_long += f"\tref: {log_paths[segment][proc]['ref']}\n"
+      diff_long += f"\tnew: {log_paths[segment][proc]['new']}\n\n"
+
+      diff_short += f"    {proc}\n"
+
+      if isinstance(diff, str) or len(diff):
+        diff_short += f"        ref: {log_paths[segment][proc]['ref']}\n"
+        diff_short += f"        new: {log_paths[segment][proc]['new']}\n\n"
+        failed = True
+
+        proc_diff_short, proc_diff_long = format_process_diff(diff)
+
+        diff_long += proc_diff_long
+        diff_short += proc_diff_short
+
+  return diff_short, diff_long, failed
+
+
+if __name__ == "__main__":
+  log1 = list(LogReader(sys.argv[1]))
+  log2 = list(LogReader(sys.argv[2]))
+  ignore_fields = sys.argv[3:] or ["logMonoTime"]
+  results = {"segment": {"proc": compare_logs(log1, log2, ignore_fields)}}
+  log_paths = {"segment": {"proc": {"ref": sys.argv[1], "new": sys.argv[2]}}}
+  diff_short, diff_long, failed = format_diff(results, log_paths, None)
+
+  print(diff_long)
+  print(diff_short)

iqpilot/selfdrive/test/process_replay/migration.py

@@ -0,0 +1,477 @@
+from collections import defaultdict
+from collections.abc import Callable
+from typing import cast
+import capnp
+import functools
+import traceback
+
+from iqpilot.cereal import messaging, car, log
+from iqdbc.car.fingerprints import MIGRATION
+from iqdbc.car.toyota.values import EPS_SCALE, ToyotaSafetyFlags
+from iqdbc.car.ford.values import CAR as FORD, FordFlags, FordSafetyFlags
+from iqdbc.car.hyundai.values import HyundaiSafetyFlags
+from iqdbc.car.gm.values import GMSafetyFlags
+from iqpilot.selfdrive.iqmodeld.config import ModelConstants
+from iqpilot.selfdrive.iqmodeld.messaging import fill_xyz_poly, fill_lane_line_meta
+from iqpilot.selfdrive.test.process_replay.vision_meta import meta_from_encode_index
+from iqpilot.selfdrive.controls.lib.longitudinal_planner import get_accel_from_plan, CONTROL_N_T_IDX
+from iqpilot.tools.lib.logreader import LogIterable
+
+MessageWithIndex = tuple[int, capnp.lib.capnp._DynamicStructReader]
+MigrationOps = tuple[list[tuple[int, capnp.lib.capnp._DynamicStructReader]], list[capnp.lib.capnp._DynamicStructReader], list[int]]
+MigrationFunc = Callable[[list[MessageWithIndex]], MigrationOps]
+
+
+# rules for migration functions
+# 1. must use the decorator @migration(inputs=[...], product="...") and MigrationFunc signature
+# 2. it only gets the messages that are in the inputs list
+# 3. product is the message type created by the migration function, and the function will be skipped if product type already exists in lr
+# 4. it must return a list of operations to be applied to the logreader (replace, add, delete)
+# 5. all migration functions must be independent of each other
+def migrate_all(lr: LogIterable, manager_states: bool = False, panda_states: bool = False, camera_states: bool = False,
+                live_location_kalman: bool = True):
+  migrations = [
+    migrate_sensorEvents,
+    migrate_carParams,
+    migrate_gpsLocation,
+    migrate_deviceState,
+    migrate_carOutput,
+    migrate_controlsState,
+    migrate_carState,
+    migrate_radarTracks,
+    migrate_driverAssistance,
+    migrate_drivingModelData,
+    migrate_onroadEvents,
+    migrate_driverMonitoringState,
+    migrate_longitudinalPlan,
+  ]
+  if manager_states:
+    migrations.append(migrate_managerState)
+  if panda_states:
+    migrations.extend([migrate_pandaStates, migrate_peripheralState])
+  if camera_states:
+    migrations.append(migrate_cameraStates)
+  if live_location_kalman:
+    migrations.append(migrate_liveLocationKalman)
+
+  return migrate(lr, migrations)
+
+
+def migrate(lr: LogIterable, migration_funcs: list[MigrationFunc]):
+  lr = list(lr)
+  grouped = defaultdict(list)
+  for i, msg in enumerate(lr):
+    grouped[msg.which()].append(i)
+
+  replace_ops, add_ops, del_ops = [], [], []
+  for migration in migration_funcs:
+    assert hasattr(migration, "inputs") and hasattr(migration, "product"), "Migration functions must use @migration decorator"
+    if migration.product in grouped: # skip if product already exists
+      continue
+
+    sorted_indices = sorted(ii for i in cast(list[str], migration.inputs) for ii in grouped.get(i, []))
+    msg_gen = [(i, lr[i]) for i in sorted_indices]
+    r_ops, a_ops, d_ops = migration(msg_gen)
+    replace_ops.extend(r_ops)
+    add_ops.extend(a_ops)
+    del_ops.extend(d_ops)
+
+  for index, msg in replace_ops:
+    lr[index] = msg
+  for index in sorted(del_ops, reverse=True):
+    del lr[index]
+  for msg in add_ops:
+    lr.append(msg)
+  lr = sorted(lr, key=lambda x: x.logMonoTime)
+
+  return lr
+
+
+def migration(inputs: list[str], product: str|None=None):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      return func(*args, **kwargs)
+    wrapper.inputs = inputs
+    wrapper.product = product
+    return wrapper
+  return decorator
+
+
+@migration(inputs=["longitudinalPlan", "carParams"])
+def migrate_longitudinalPlan(msgs):
+  ops = []
+
+  needs_migration = all(msg.longitudinalPlan.aTarget == 0.0 for _, msg in msgs if msg.which() == 'longitudinalPlan')
+  CP = next((m.carParams for _, m in msgs if m.which() == 'carParams'), None)
+  if not needs_migration or CP is None:
+    return [], [], []
+
+  for index, msg in msgs:
+    if msg.which() != 'longitudinalPlan':
+      continue
+    new_msg = msg.as_builder()
+    a_target, should_stop = get_accel_from_plan(msg.longitudinalPlan.speeds, msg.longitudinalPlan.accels, CONTROL_N_T_IDX)
+    new_msg.longitudinalPlan.aTarget, new_msg.longitudinalPlan.shouldStop = float(a_target), bool(should_stop)
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["longitudinalPlan"], product="driverAssistance")
+def migrate_driverAssistance(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    new_msg = messaging.new_message('driverAssistance', valid=True, logMonoTime=msg.logMonoTime)
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["modelV2"], product="drivingModelData")
+def migrate_drivingModelData(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    dmd = messaging.new_message('drivingModelData', valid=msg.valid, logMonoTime=msg.logMonoTime)
+    for field in ["frameId", "frameIdExtra", "frameDropPerc", "modelExecutionTime", "action"]:
+      setattr(dmd.drivingModelData, field, getattr(msg.modelV2, field))
+    for meta_field in ["laneChangeState", "laneChangeState"]:
+      setattr(dmd.drivingModelData.meta, meta_field, getattr(msg.modelV2.meta, meta_field))
+    lane_lines = msg.modelV2.laneLines
+    lane_probs = msg.modelV2.laneLineProbs
+    if len(lane_lines) > 2 and len(lane_probs) > 2 and len(lane_lines[1].y) and len(lane_lines[2].y):
+      fill_lane_line_meta(dmd.drivingModelData.laneLineMeta, msg.modelV2.laneLines, msg.modelV2.laneLineProbs)
+    if all(len(a) for a in [msg.modelV2.position.x, msg.modelV2.position.y, msg.modelV2.position.z]):
+      fill_xyz_poly(dmd.drivingModelData.path, ModelConstants.POLY_PATH_DEGREE, msg.modelV2.position.x, msg.modelV2.position.y, msg.modelV2.position.z)
+    add_ops.append( dmd.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["radarTracksDEPRECATED"], product="radarTracks")
+def migrate_radarTracks(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = messaging.new_message('radarTracks')
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+
+    pts = []
+    for track in msg.radarTracksDEPRECATED:
+      pt = car.RadarData.RadarPoint()
+      pt.trackId = track.trackId
+
+      pt.dRel = track.dRel
+      pt.yRel = track.yRel
+      pt.vRel = track.vRel
+      pt.aRel = track.aRel
+      pt.measured = True
+      pts.append(pt)
+
+    new_msg.radarTracks.points = pts
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["liveLocationKalmanDEPRECATED"], product="deviceMotion")
+def migrate_liveLocationKalman(msgs):
+  nans = [float('nan')] * 3
+  ops = []
+  for index, msg in msgs:
+    m = messaging.new_message('deviceMotion')
+    m.valid = msg.valid
+    m.logMonoTime = msg.logMonoTime
+    for field in ["orientationNED", "velocityDevice", "accelerationDevice", "angularVelocityDevice"]:
+      lp_field, llk_field = getattr(m.deviceMotion, field), getattr(msg.liveLocationKalmanDEPRECATED, field)
+      lp_field.x, lp_field.y, lp_field.z = llk_field.value or nans
+      lp_field.xStd, lp_field.yStd, lp_field.zStd = llk_field.std or nans
+      lp_field.valid = llk_field.valid
+    for flag in ["inputsOK", "posenetOK", "sensorsOK"]:
+      setattr(m.deviceMotion, flag, getattr(msg.liveLocationKalmanDEPRECATED, flag))
+    ops.append((index, m.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["controlsState"], product="selfdriveState")
+def migrate_controlsState(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    m = messaging.new_message('selfdriveState')
+    m.valid = msg.valid
+    m.logMonoTime = msg.logMonoTime
+    ss = m.selfdriveState
+    for field in ("enabled", "active", "state", "engageable", "alertText1", "alertText2",
+                  "alertStatus", "alertSize", "alertType", "experimentalMode",
+                  "personality"):
+      setattr(ss, field, getattr(msg.controlsState, field+"DEPRECATED"))
+    add_ops.append(m.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["carState", "controlsState"])
+def migrate_carState(msgs):
+  ops = []
+  last_cs = None
+  for index, msg in msgs:
+    if msg.which() == 'controlsState':
+      last_cs = msg
+    elif msg.which() == 'carState' and last_cs is not None:
+      if last_cs.controlsState.vCruiseDEPRECATED - msg.carState.vCruise > 0.1:
+        msg = msg.as_builder()
+        msg.carState.vCruise = last_cs.controlsState.vCruiseDEPRECATED
+        msg.carState.vCruiseCluster = last_cs.controlsState.vCruiseClusterDEPRECATED
+        ops.append((index, msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["managerState"])
+def migrate_managerState(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = msg.as_builder()
+    for process in new_msg.managerState.processes:
+      process.running = True
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["gpsLocation", "gpsLocationExternal"])
+def migrate_gpsLocation(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = msg.as_builder()
+    g = getattr(new_msg, new_msg.which())
+    # hasFix is a newer field
+    if not g.hasFix and g.flags == 1:
+      g.hasFix = True
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["deviceState", "initData"])
+def migrate_deviceState(msgs):
+  init_data = next((m.initData for _, m in msgs if m.which() == 'initData'), None)
+  device_state = next((m.deviceState for _, m in msgs if m.which() == 'deviceState'), None)
+  if init_data is None or device_state is None:
+    return [], [], []
+
+  ops = []
+  for i, msg in msgs:
+    if msg.which() == 'deviceState':
+      n = msg.as_builder()
+      n.deviceState.deviceType = init_data.deviceType
+      ops.append((i, n.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["carControl"], product="carOutput")
+def migrate_carOutput(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    co = messaging.new_message('carOutput')
+    co.valid = msg.valid
+    co.logMonoTime = msg.logMonoTime
+    co.carOutput.actuatorsOutput = msg.carControl.actuatorsOutputDEPRECATED
+    add_ops.append(co.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["pandaStates", "pandaStateDEPRECATED", "carParams"])
+def migrate_pandaStates(msgs):
+  # TODO: safety param migration should be handled automatically
+  safety_param_migration = {
+    "TOYOTA_PRIUS": EPS_SCALE["TOYOTA_PRIUS"] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
+    "TOYOTA_RAV4": EPS_SCALE["TOYOTA_RAV4"] | ToyotaSafetyFlags.ALT_BRAKE,
+    "KIA_EV6": HyundaiSafetyFlags.EV_GAS | HyundaiSafetyFlags.CANFD_LKA_STEERING,
+    "CHEVROLET_VOLT": GMSafetyFlags.EV,
+    "CHEVROLET_BOLT_EUV": GMSafetyFlags.EV | GMSafetyFlags.HW_CAM,
+  }
+  # TODO: get new Ford route
+  safety_param_migration |= dict.fromkeys((set(FORD) - FORD.with_flags(FordFlags.CANFD)), FordSafetyFlags.LONG_CONTROL)
+
+  # Migrate safety param base on carParams
+  CP = next((m.carParams for _, m in msgs if m.which() == 'carParams'), None)
+  assert CP is not None, "carParams message not found"
+  fingerprint = MIGRATION.get(CP.carFingerprint, CP.carFingerprint)
+  if fingerprint in safety_param_migration:
+    safety_param = safety_param_migration[fingerprint].value
+  elif len(CP.safetyConfigs):
+    safety_param = CP.safetyConfigs[0].safetyParam
+    if CP.safetyConfigs[0].safetyParamDEPRECATED != 0:
+      safety_param = CP.safetyConfigs[0].safetyParamDEPRECATED
+  else:
+    safety_param = CP.safetyParamDEPRECATED
+
+  ops = []
+  for index, msg in msgs:
+    if msg.which() == 'pandaStateDEPRECATED':
+      new_msg = messaging.new_message('pandaStates', 1)
+      new_msg.valid = msg.valid
+      new_msg.logMonoTime = msg.logMonoTime
+      new_msg.pandaStates[0] = msg.pandaStateDEPRECATED
+      new_msg.pandaStates[0].safetyParam = safety_param
+      ops.append((index, new_msg.as_reader()))
+    elif msg.which() == 'pandaStates':
+      new_msg = msg.as_builder()
+      new_msg.pandaStates[-1].safetyParam = safety_param
+      # Clear DISABLE_DISENGAGE_ON_GAS bit to fix controls mismatch
+      new_msg.pandaStates[-1].alternativeExperience &= ~1
+      ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["pandaStates", "pandaStateDEPRECATED"], product="peripheralState")
+def migrate_peripheralState(msgs):
+  add_ops = []
+
+  which = "pandaStates" if any(msg.which() == "pandaStates" for _, msg in msgs) else "pandaStateDEPRECATED"
+  for _, msg in msgs:
+    if msg.which() != which:
+      continue
+    new_msg = messaging.new_message("peripheralState")
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["roadEncodeIdx", "wideRoadEncodeIdx", "driverEncodeIdx", "roadCameraState", "wideRoadCameraState", "driverCameraState"])
+def migrate_cameraStates(msgs):
+  add_ops, del_ops = [], []
+  frame_to_encode_id = defaultdict(dict)
+  # just for encodeId fallback mechanism
+  min_frame_id = defaultdict(lambda: float('inf'))
+
+  for _, msg in msgs:
+    if msg.which() not in ["roadEncodeIdx", "wideRoadEncodeIdx", "driverEncodeIdx"]:
+      continue
+
+    encode_index = getattr(msg, msg.which())
+    meta = meta_from_encode_index(msg.which())
+
+    assert encode_index.segmentId < 1200, f"Encoder index segmentId greater that 1200: {msg.which()} {encode_index.segmentId}"
+    frame_to_encode_id[meta.camera_state][encode_index.frameId] = encode_index.segmentId
+
+  for index, msg in msgs:
+    if msg.which() not in ["roadCameraState", "wideRoadCameraState", "driverCameraState"]:
+      continue
+
+    camera_state = getattr(msg, msg.which())
+    min_frame_id[msg.which()] = min(min_frame_id[msg.which()], camera_state.frameId)
+
+    encode_id = frame_to_encode_id[msg.which()].get(camera_state.frameId)
+    if encode_id is None:
+      print(f"Missing encoded frame for camera feed {msg.which()} with frameId: {camera_state.frameId}")
+      if len(frame_to_encode_id[msg.which()]) != 0:
+        del_ops.append(index)
+        continue
+
+      # fallback mechanism for logs without encodeIdx (e.g. logs from before 2022 with dcamera recording disabled)
+      # try to fake encode_id by subtracting lowest frameId
+      encode_id = camera_state.frameId - min_frame_id[msg.which()]
+      print(f"Faking encodeId to {encode_id} for camera feed {msg.which()} with frameId: {camera_state.frameId}")
+
+    new_msg = messaging.new_message(msg.which())
+    new_camera_state = getattr(new_msg, new_msg.which())
+    new_camera_state.sensor = camera_state.sensor
+    new_camera_state.frameId = encode_id
+    new_camera_state.encodeId = encode_id
+    # timestampSof was added later so it might be missing on some old segments
+    if camera_state.timestampSof == 0 and camera_state.timestampEof > 25000000:
+      new_camera_state.timestampSof = camera_state.timestampEof - 18000000
+    else:
+      new_camera_state.timestampSof = camera_state.timestampSof
+    new_camera_state.timestampEof = camera_state.timestampEof
+    new_msg.logMonoTime = msg.logMonoTime
+    new_msg.valid = msg.valid
+
+    del_ops.append(index)
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, del_ops
+
+
+@migration(inputs=["carParams"])
+def migrate_carParams(msgs):
+  ops = []
+  for index, msg in msgs:
+    CP = msg.as_builder()
+    CP.carParams.carFingerprint = MIGRATION.get(CP.carParams.carFingerprint, CP.carParams.carFingerprint)
+    for car_fw in CP.carParams.carFw:
+      car_fw.brand = CP.carParams.brand
+    ops.append((index, CP.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["sensorEventsDEPRECATED"], product="sensorEvents")
+def migrate_sensorEvents(msgs):
+  add_ops, del_ops = [], []
+  for index, msg in msgs:
+    # migrate to split sensor events
+    for evt in msg.sensorEventsDEPRECATED:
+      # build new message for each sensor type
+      sensor_service = ''
+      if evt.which() == 'acceleration':
+        sensor_service = 'accelerometer'
+      elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated':
+        sensor_service = 'gyroscope'
+      elif evt.which() == 'light' or evt.which() == 'proximity':
+        sensor_service = 'lightSensor'
+      elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated':
+        sensor_service = 'magnetometer'
+      elif evt.which() == 'temperature':
+        sensor_service = 'temperatureSensor'
+
+      m = messaging.new_message(sensor_service)
+      m.valid = True
+      m.logMonoTime = msg.logMonoTime
+
+      m_dat = getattr(m, sensor_service)
+      m_dat.version = evt.version
+      m_dat.sensor = evt.sensor
+      m_dat.type = evt.type
+      m_dat.source = evt.source
+      m_dat.timestamp = evt.timestamp
+      setattr(m_dat, evt.which(), getattr(evt, evt.which()))
+
+      add_ops.append(m.as_reader())
+    del_ops.append(index)
+  return [], add_ops, del_ops
+
+
+@migration(inputs=["onroadEventsDEPRECATED"], product="onroadEvents")
+def migrate_onroadEvents(msgs):
+  ops = []
+  for index, msg in msgs:
+    onroadEvents = []
+    for event in msg.onroadEventsDEPRECATED:
+      try:
+        if not str(event.name).endswith('DEPRECATED'):
+          # dict converts name enum into string representation
+          onroadEvents.append(log.OnroadEvent(**event.to_dict()))
+      except RuntimeError:  # Member was null
+        traceback.print_exc()
+
+    new_msg = messaging.new_message('onroadEvents', len(msg.onroadEventsDEPRECATED))
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+    new_msg.onroadEvents = onroadEvents
+    ops.append((index, new_msg.as_reader()))
+
+  return ops, [], []
+
+
+@migration(inputs=["driverMonitoringState"])
+def migrate_driverMonitoringState(msgs):
+  ops = []
+  for index, msg in msgs:
+    msg = msg.as_builder()
+    events = []
+    for event in msg.driverMonitoringState.eventsDEPRECATED:
+      try:
+        if not str(event.name).endswith('DEPRECATED'):
+          # dict converts name enum into string representation
+          events.append(log.OnroadEvent(**event.to_dict()))
+      except RuntimeError:  # Member was null
+        traceback.print_exc()
+
+    msg.driverMonitoringState.events = events
+    ops.append((index, msg.as_reader()))
+
+  return ops, [], []

iqpilot/selfdrive/test/process_replay/model_replay.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+import os
+import pickle
+import sys
+from collections import defaultdict
+from typing import Any
+
+import numpy as np
+from tabulate import tabulate
+
+from iqpilot.system.hardware import PC
+from iqpilot.tools.lib.logreader import get_url
+from iqpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
+from iqpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
+from iqpilot.tools.lib.framereader import FrameReader
+from iqpilot.tools.lib.logreader import LogReader, save_log
+
+TEST_ROUTE = "8494c69d3c710e81|000001d4--2648a9a404"
+SEGMENT = 4
+START_FRAME = 0
+END_FRAME = 60
+
+SEND_EXTRA_INPUTS = bool(int(os.getenv("SEND_EXTRA_INPUTS", "0")))
+
+MODEL_REPLAY_BUCKET="model_replay_master"
+
+EXEC_TIMINGS = [
+  # model, instant max, average max
+  ("modelV2", 0.035, 0.025),
+  ("driverStateV2", 0.02, 0.015),
+]
+
+def get_log_fn(test_route, ref="master"):
+  return f"{test_route}_model_tici_{ref}.zst"
+
+def get_model_replay_url(filename):
+  return f"https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/{MODEL_REPLAY_BUCKET}/{filename}"
+
+def trim_logs(logs, start_frame, end_frame, frs_types, include_all_types):
+  all_msgs = []
+  cam_state_counts = defaultdict(int)
+  for msg in sorted(logs, key=lambda m: m.logMonoTime):
+    if msg.which() in frs_types:
+      cam_state_counts[msg.which()] += 1
+    if any(cam_state_counts[state]  >= start_frame for state in frs_types):
+      all_msgs.append(msg)
+    if all(cam_state_counts[state] == end_frame for state in frs_types):
+      break
+
+  if len(include_all_types) != 0:
+    other_msgs = [m for m in logs if m.which() in include_all_types]
+    all_msgs.extend(other_msgs)
+
+  return all_msgs
+
+
+def model_replay(lr, frs):
+  # modeld is using frame pairs
+  modeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"roadCameraState", "wideRoadCameraState"},
+                                                                         {"roadEncodeIdx", "wideRoadEncodeIdx", "carParams", "carState", "carControl", "can"})
+  dmodeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"driverCameraState"}, {"driverEncodeIdx", "carParams", "can"})
+
+  if not SEND_EXTRA_INPUTS:
+    modeld_logs = [msg for msg in modeld_logs if msg.which() != 'extrinsicsCalibration']
+    dmodeld_logs = [msg for msg in dmodeld_logs if msg.which() != 'extrinsicsCalibration']
+
+  # initial setup
+  for s in ('extrinsicsCalibration', 'deviceState'):
+    msg = next(msg for msg in lr if msg.which() == s).as_builder()
+    msg.logMonoTime = lr[0].logMonoTime
+    modeld_logs.insert(1, msg.as_reader())
+    dmodeld_logs.insert(1, msg.as_reader())
+
+  modeld = get_process_config("modeld")
+  dmonitoringmodeld = get_process_config("dmonitoringmodeld")
+
+  modeld_msgs = replay_process(modeld, modeld_logs, frs)
+  dmonitoringmodeld_msgs = replay_process(dmonitoringmodeld, dmodeld_logs, frs)
+
+  msgs = modeld_msgs + dmonitoringmodeld_msgs
+
+  header = ['model', 'max instant', 'max instant allowed', 'average', 'max average allowed', 'test result']
+  rows = []
+  timings_ok = True
+  for (s, instant_max, avg_max) in EXEC_TIMINGS:
+    ts = [getattr(m, s).modelExecutionTime for m in msgs if m.which() == s]
+    # TODO some init can happen in first iteration
+    ts = ts[1:]
+
+    errors = []
+    if np.max(ts) > instant_max:
+      errors.append("❌ FAILED MAX TIMING CHECK ❌")
+    if np.mean(ts) > avg_max:
+      errors.append("❌ FAILED AVG TIMING CHECK ❌")
+
+    timings_ok = not errors and timings_ok
+    rows.append([s, np.max(ts), instant_max, np.mean(ts), avg_max, "\n".join(errors) or "✅"])
+
+  print("------------------------------------------------")
+  print("----------------- Model Timing -----------------")
+  print("------------------------------------------------")
+  print(tabulate(rows, header, tablefmt="simple_grid", stralign="center", numalign="center", floatfmt=".4f"))
+  assert timings_ok or PC
+
+  return msgs
+
+
+def get_frames():
+  regen_cache = "--regen-cache" in sys.argv
+  frames_cache = '/tmp/model_replay_cache' if PC else '/data/model_replay_cache'
+  os.makedirs(frames_cache, exist_ok=True)
+
+  cache_name = f'{frames_cache}/{TEST_ROUTE}_{SEGMENT}_{START_FRAME}_{END_FRAME}.pkl'
+  if os.path.isfile(cache_name) and not regen_cache:
+    try:
+      print(f"Loading frames from cache {cache_name}")
+      return pickle.load(open(cache_name, "rb"))
+    except Exception as e:
+      print(f"Failed to load frames from cache {cache_name}: {e}")
+
+  frs = {
+    'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+    'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+    'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+  }
+  for fr in frs.values():
+    for fidx in range(START_FRAME, END_FRAME):
+      fr.get(fidx)
+    fr.it = None
+  print(f"Dumping frame cache {cache_name}")
+  pickle.dump(frs, open(cache_name, "wb"))
+  return frs
+
+if __name__ == "__main__":
+  update = "--update" in sys.argv or (os.getenv("GIT_BRANCH", "") == 'master')
+  replay_dir = os.path.dirname(os.path.abspath(__file__))
+
+  # load logs
+  lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.zst")))
+  frs = get_frames()
+
+  log_msgs = []
+  # run replays
+  log_msgs += model_replay(lr, frs)
+
+  # get diff
+  failed = False
+  if not update:
+    log_fn = get_log_fn(TEST_ROUTE)
+    try:
+      all_logs = list(LogReader(get_model_replay_url(log_fn)))
+      cmp_log = []
+      model_start_index = next(i for i, m in enumerate(all_logs) if m.which() in ("modelV2", "drivingModelData", "cameraOdometry"))
+      cmp_log += all_logs[model_start_index+START_FRAME*3:model_start_index + END_FRAME*3]
+      dmon_start_index = next(i for i, m in enumerate(all_logs) if m.which() == "driverStateV2")
+      cmp_log += all_logs[dmon_start_index+START_FRAME:dmon_start_index + END_FRAME]
+
+      ignore = [
+        'logMonoTime',
+        'drivingModelData.frameDropPerc',
+        'drivingModelData.modelExecutionTime',
+        'modelV2.frameDropPerc',
+        'modelV2.modelExecutionTime',
+        'driverStateV2.modelExecutionTime',
+        'driverStateV2.gpuExecutionTime'
+      ]
+      if PC:
+        # TODO We ignore whole bunch so we can compare important stuff
+        # like posenet with reasonable tolerance
+        ignore += ['modelV2.acceleration.x',
+                   'modelV2.position.x',
+                   'modelV2.position.xStd',
+                   'modelV2.position.y',
+                   'modelV2.position.yStd',
+                   'modelV2.position.z',
+                   'modelV2.position.zStd',
+                   'drivingModelData.path.xCoefficients',]
+        for i in range(3):
+          for field in ('x', 'y', 'v', 'a'):
+            ignore.append(f'modelV2.leadsV3.{i}.{field}')
+            ignore.append(f'modelV2.leadsV3.{i}.{field}Std')
+        for i in range(4):
+          for field in ('x', 'y', 'z', 't'):
+            ignore.append(f'modelV2.laneLines.{i}.{field}')
+        for i in range(2):
+          for field in ('x', 'y', 'z', 't'):
+            ignore.append(f'modelV2.roadEdges.{i}.{field}')
+      tolerance = .3 if PC else None
+      results: Any = {TEST_ROUTE: {}}
+      log_paths: Any = {TEST_ROUTE: {"models": {'ref': log_fn, 'new': log_fn}}}
+      results[TEST_ROUTE]["models"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore)
+      diff_short, diff_long, failed = format_diff(results, log_paths, 'master')
+
+      if "CI" in os.environ:
+        failed = False
+        print(diff_long)
+      print('-------------\n'*5)
+      print(diff_short)
+      with open("model_diff.txt", "w") as f:
+        f.write(diff_long)
+    except Exception as e:
+      print(str(e))
+      failed = True
+
+  if update and not PC:
+    log_fn = get_log_fn(TEST_ROUTE)
+    save_log(log_fn, log_msgs)
+
+  sys.exit(int(failed))

iqpilot/selfdrive/test/process_replay/process_replay.py

@@ -0,0 +1,779 @@
+#!/usr/bin/env python3
+import os
+import time
+import copy
+import heapq
+import signal
+from collections import Counter
+from dataclasses import dataclass, field
+from itertools import islice
+from typing import Any
+from collections.abc import Callable, Iterable
+from tqdm import tqdm
+import capnp
+from iqpilot.system.hardware.hw import Paths
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import car
+from iqpilot.cereal.services import SERVICE_LIST
+from msgq.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name
+from iqdbc.car.can_definitions import CanData
+from iqdbc.car.car_helpers import get_car, interfaces
+from iqpilot.common.params import Params
+from iqpilot.common.prefix import OpenpilotPrefix
+from iqpilot.common.timeout import Timeout
+from iqpilot.common.realtime import DT_CTRL
+from iqpilot.selfdrive.car.card import convert_to_capnp
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera_state, available_streams
+from iqpilot.selfdrive.test.process_replay.migration import migrate_all
+from iqpilot.selfdrive.test.process_replay.capture import ProcessOutputCapture
+from iqpilot.tools.lib.logreader import LogIterable
+from iqpilot.tools.lib.framereader import FrameReader
+
+# Numpy gives different results based on CPU features after version 19
+NUMPY_TOLERANCE = 1e-2
+PROC_REPLAY_DIR = os.path.dirname(os.path.abspath(__file__))
+FAKEDATA = os.path.join(PROC_REPLAY_DIR, "fakedata/")
+
+
+class LauncherWithCapture:
+  def __init__(self, capture: ProcessOutputCapture, launcher: Callable):
+    self.capture = capture
+    self.launcher = launcher
+
+  def __call__(self, *args, **kwargs):
+    self.capture.link_with_current_proc()
+    self.launcher(*args, **kwargs)
+
+
+class ReplayContext:
+  def __init__(self, cfg):
+    self.proc_name = cfg.proc_name
+    self.pubs = cfg.pubs
+    self.main_pub = cfg.main_pub
+    self.main_pub_drained = cfg.main_pub_drained
+    assert len(self.pubs) != 0 or self.main_pub is not None
+
+  def __enter__(self):
+    self.open_context()
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    self.close_context()
+
+  def open_context(self):
+    messaging.toggle_fake_events(True)
+    messaging.set_fake_prefix(self.proc_name)
+
+    if self.main_pub is None:
+      self.events = {}
+      for pub in self.pubs:
+        self.events[pub] = messaging.fake_event_handle(pub, enable=True)
+    else:
+      self.events = {self.main_pub: messaging.fake_event_handle(self.main_pub, enable=True)}
+
+  def close_context(self):
+    del self.events
+
+    messaging.toggle_fake_events(False)
+    messaging.delete_fake_prefix()
+
+  @property
+  def all_recv_called_events(self):
+    return [man.recv_called_event for man in self.events.values()]
+
+  @property
+  def all_recv_ready_events(self):
+    return [man.recv_ready_event for man in self.events.values()]
+
+  def send_sync(self, pm, endpoint, dat):
+    self.events[endpoint].recv_called_event.wait()
+    self.events[endpoint].recv_called_event.clear()
+    pm.send(endpoint, dat)
+    self.events[endpoint].recv_ready_event.set()
+
+  def unlock_sockets(self):
+    expected_sets = len(self.events)
+    while expected_sets > 0:
+      index = messaging.wait_for_one_event(self.all_recv_called_events)
+      self.all_recv_called_events[index].clear()
+      self.all_recv_ready_events[index].set()
+      expected_sets -= 1
+
+  def wait_for_recv_called(self):
+    messaging.wait_for_one_event(self.all_recv_called_events)
+
+  def wait_for_next_recv(self, trigger_empty_recv):
+    index = messaging.wait_for_one_event(self.all_recv_called_events)
+    if self.main_pub is not None and self.main_pub_drained and trigger_empty_recv:
+      self.all_recv_called_events[index].clear()
+      self.all_recv_ready_events[index].set()
+      self.all_recv_called_events[index].wait()
+
+
+@dataclass
+class ProcessConfig:
+  proc_name: str
+  pubs: list[str]
+  subs: list[str]
+  ignore: list[str]
+  config_callback: Callable | None = None
+  init_callback: Callable | None = None
+  should_recv_callback: Callable | None = None
+  tolerance: float | None = None
+  processing_time: float = 0.001
+  timeout: int = 30
+  simulation: bool = True
+  # Set to service process receives on first
+  main_pub: str | None = None
+  main_pub_drained: bool = False
+  vision_pubs: list[str] = field(default_factory=list)
+  ignore_alive_pubs: list[str] = field(default_factory=list)
+
+  def __post_init__(self):
+    # If the process is polling a service, we can just lock that one to speed up replay
+    if self.main_pub is None and isinstance(self.should_recv_callback, MessageBasedRcvCallback):
+      self.main_pub = self.should_recv_callback.trigger_msg_type
+
+
+class ProcessContainer:
+  def __init__(self, cfg: ProcessConfig):
+    self.prefix = OpenpilotPrefix(create_dirs_on_enter=False, clean_dirs_on_exit=False)
+    self.cfg = copy.deepcopy(cfg)
+    self.process = copy.deepcopy(managed_processes[cfg.proc_name])
+    self.msg_queue: list[capnp._DynamicStructReader] = []
+    self.cnt = 0
+    self.pm: messaging.PubMaster | None = None
+    self.sockets: list[messaging.SubSocket] | None = None
+    self.rc: ReplayContext | None = None
+    self.vipc_server: VisionIpcServer | None = None
+    self.environ_config: dict[str, Any] | None = None
+    self.capture: ProcessOutputCapture | None = None
+
+  @property
+  def has_empty_queue(self) -> bool:
+    return len(self.msg_queue) == 0
+
+  @property
+  def pubs(self) -> list[str]:
+    return self.cfg.pubs
+
+  @property
+  def subs(self) -> list[str]:
+    return self.cfg.subs
+
+  def _clean_env(self):
+    for k in self.environ_config.keys():
+      if k in os.environ:
+        del os.environ[k]
+
+    for k in ["PROC_NAME", "SIMULATION"]:
+      if k in os.environ:
+        del os.environ[k]
+
+  def _setup_env(self, params_config: dict[str, Any], environ_config: dict[str, Any]):
+    for k, v in environ_config.items():
+      if len(v) != 0:
+        os.environ[k] = v
+      elif k in os.environ:
+        del os.environ[k]
+
+    os.environ["PROC_NAME"] = self.cfg.proc_name
+    if self.cfg.simulation:
+      os.environ["SIMULATION"] = "1"
+    elif "SIMULATION" in os.environ:
+      del os.environ["SIMULATION"]
+
+    params = Params()
+    for k, v in params_config.items():
+      if isinstance(v, bool):
+        params.put_bool(k, v)
+      else:
+        params.put(k, v)
+
+    self.environ_config = environ_config
+
+  def _setup_vision_ipc(self, all_msgs: LogIterable, frs: dict[str, Any]):
+    assert len(self.cfg.vision_pubs) != 0
+
+    vipc_server = VisionIpcServer("camerad")
+    streams_metas = available_streams(all_msgs)
+    for meta in streams_metas:
+      if meta.camera_state in self.cfg.vision_pubs:
+        assert frs[meta.camera_state].pix_fmt == 'nv12'
+        frame_size = (frs[meta.camera_state].w, frs[meta.camera_state].h)
+        vipc_server.create_buffers(meta.stream, 2, *frame_size)
+    vipc_server.start_listener()
+
+    self.vipc_server = vipc_server
+    self.cfg.vision_pubs = [meta.camera_state for meta in streams_metas if meta.camera_state in self.cfg.vision_pubs]
+
+  def _start_process(self):
+    if self.capture is not None:
+      self.process.launcher = LauncherWithCapture(self.capture, self.process.launcher)
+    self.process.prepare()
+    self.process.start()
+
+  def start(
+    self, params_config: dict[str, Any], environ_config: dict[str, Any],
+    all_msgs: LogIterable, frs: dict[str, FrameReader] | None,
+    fingerprint: str | None, capture_output: bool
+  ):
+    with self.prefix as p:
+      self.prefix.create_dirs()
+      self._setup_env(params_config, environ_config)
+
+      if self.cfg.config_callback is not None:
+        params = Params()
+        self.cfg.config_callback(params, self.cfg, all_msgs)
+
+      self.rc = ReplayContext(self.cfg)
+      self.rc.open_context()
+
+      self.pm = messaging.PubMaster(self.cfg.pubs)
+      self.sockets = [messaging.sub_sock(s, timeout=100) for s in self.cfg.subs]
+
+      if len(self.cfg.vision_pubs) != 0:
+        assert frs is not None
+        self._setup_vision_ipc(all_msgs, frs)
+        assert self.vipc_server is not None
+
+      if capture_output:
+        self.capture = ProcessOutputCapture(self.cfg.proc_name, p.prefix)
+
+      self._start_process()
+
+      if self.cfg.init_callback is not None:
+        self.cfg.init_callback(self.rc, self.pm, all_msgs, fingerprint)
+
+  def stop(self):
+    with self.prefix:
+      self.process.signal(signal.SIGKILL)
+      self.process.stop()
+      self.rc.close_context()
+      self.prefix.clean_dirs()
+      self._clean_env()
+
+  def get_output_msgs(self, start_time: int):
+    assert self.rc and self.sockets
+
+    output_msgs = []
+    self.rc.wait_for_recv_called()
+    for socket in self.sockets:
+      ms = messaging.drain_sock(socket)
+      for m in ms:
+        m = m.as_builder()
+        m.logMonoTime = start_time + int(self.cfg.processing_time * 1e9)
+        output_msgs.append(m.as_reader())
+    return output_msgs
+
+  def run_step(self, msg: capnp._DynamicStructReader, frs: dict[str, FrameReader] | None) -> list[capnp._DynamicStructReader]:
+    assert self.rc and self.pm and self.sockets and self.process.proc
+
+    output_msgs = []
+    end_of_cycle = True
+    if self.cfg.should_recv_callback is not None:
+      end_of_cycle = self.cfg.should_recv_callback(msg, self.cfg, self.cnt)
+
+    self.msg_queue.append(msg)
+    if end_of_cycle:
+      with self.prefix, Timeout(self.cfg.timeout, error_msg=f"timed out testing process {repr(self.cfg.proc_name)}"):
+        # call recv to let sub-sockets reconnect, after we know the process is ready
+        if self.cnt == 0:
+          for s in self.sockets:
+            messaging.recv_one_or_none(s)
+
+        # certain processes use drain_sock. need to cause empty recv to break from this loop
+        trigger_empty_recv = False
+        if self.cfg.main_pub and self.cfg.main_pub_drained:
+          trigger_empty_recv = any(m.which() == self.cfg.main_pub for m in self.msg_queue)
+
+        # get output msgs from previous inputs
+        output_msgs = self.get_output_msgs(msg.logMonoTime)
+
+        for m in self.msg_queue:
+          self.pm.send(m.which(), m.as_builder())
+          # send frames if needed
+          if self.vipc_server is not None and m.which() in self.cfg.vision_pubs:
+            camera_state = getattr(m, m.which())
+            camera_meta = meta_from_camera_state(m.which())
+            assert frs is not None
+            img = frs[m.which()].get(camera_state.frameId)
+            self.vipc_server.send(camera_meta.stream, img.flatten().tobytes(),
+                                  camera_state.frameId, camera_state.timestampSof, camera_state.timestampEof)
+        self.msg_queue = []
+
+        self.rc.unlock_sockets()
+        if trigger_empty_recv:
+          self.rc.unlock_sockets()
+        self.cnt += 1
+    assert self.process.proc.is_alive()
+
+    return output_msgs
+
+
+def card_fingerprint_callback(rc, pm, msgs, fingerprint):
+  print("start fingerprinting")
+  params = Params()
+  canmsgs = list(islice((m for m in msgs if m.which() == "can"), 300))
+
+  # card expects one arbitrary can and pandaState
+  rc.send_sync(pm, "can", messaging.new_message("can", 1))
+  pm.send("pandaStates", messaging.new_message("pandaStates", 1))
+  rc.send_sync(pm, "can", messaging.new_message("can", 1))
+  rc.wait_for_next_recv(True)
+
+  # fingerprinting is done, when CarParams is set
+  while params.get("CarParams") is None:
+    if len(canmsgs) == 0:
+      raise ValueError("Fingerprinting failed. Run out of can msgs")
+
+    m = canmsgs.pop(0)
+    rc.send_sync(pm, "can", m.as_builder().to_bytes())
+    rc.wait_for_next_recv(True)
+
+
+def get_car_params_callback(rc, pm, msgs, fingerprint):
+  params = Params()
+  if fingerprint:
+    CarInterface = interfaces[fingerprint]
+    CP = CarInterface.get_non_essential_params(fingerprint)
+    CP_IQ = CarInterface.get_non_essential_params_iq(CP, fingerprint)
+  else:
+    can_msgs = ([CanData(can.address, can.dat, can.src) for can in m.can] for m in msgs if m.which() == "can")
+    cached_params_raw = params.get("CarParamsCache")
+    assert next(can_msgs, None), "CAN messages are required for fingerprinting"
+    assert os.environ.get("SKIP_FW_QUERY", False) or cached_params_raw is not None, \
+            "CarParamsCache is required for fingerprinting. Make sure to keep carParams msgs in the logs."
+
+    def can_recv(wait_for_one: bool = False) -> list[list[CanData]]:
+      return [next(can_msgs, [])]
+
+    cached_params = None
+    if cached_params_raw is not None:
+      with car.CarParams.from_bytes(cached_params_raw) as _cached_params:
+        cached_params = _cached_params
+
+    _CI = get_car(can_recv, lambda _msgs: None, lambda obd: None, params.get_bool("AlphaLongitudinalEnabled"), False, cached_params=cached_params)
+    CP, CP_IQ = _CI.CP, _CI.CP_IQ
+
+  params.put("CarParams", CP.to_bytes())
+  params.put("IQCarParams", convert_to_capnp(CP_IQ).to_bytes())
+
+
+def card_rcv_callback(msg, cfg, frame):
+  # no sendcan until card is initialized
+  if msg.which() != "can":
+    return False
+
+  socks = [
+    s for s in cfg.subs if
+    frame % int(SERVICE_LIST[msg.which()].frequency / SERVICE_LIST[s].frequency) == 0
+  ]
+  if "sendcan" in socks and (frame - 1) < 2000:
+    socks.remove("sendcan")
+  return len(socks) > 0
+
+
+class ModeldCameraSyncRcvCallback:
+  def __init__(self):
+    self.road_present = False
+    self.wide_road_present = False
+    self.is_dual_camera = True
+
+  def __call__(self, msg, cfg, frame):
+    self.is_dual_camera = len(cfg.vision_pubs) == 2
+    if msg.which() == "roadCameraState":
+      self.road_present = True
+    elif msg.which() == "wideRoadCameraState":
+      self.wide_road_present = True
+
+    if self.road_present and self.wide_road_present:
+      self.road_present, self.wide_road_present = False, False
+      return True
+    elif self.road_present and not self.is_dual_camera:
+      self.road_present = False
+      return True
+    else:
+      return False
+
+
+class MessageBasedRcvCallback:
+  def __init__(self, trigger_msg_type: str, first_frame: bool = False):
+    self.trigger_msg_type = trigger_msg_type
+    self.first_frame = first_frame
+
+  def __call__(self, msg, cfg, frame):
+    # publish on first frame or trigger msg
+    return ((frame - 1) == 0 and self.first_frame) or msg.which() == self.trigger_msg_type
+
+
+def selfdrived_config_callback(params, cfg, lr):
+  ublox = params.get_bool("UbloxAvailable")
+  sub_keys = ({"gpsLocation", } if ublox else {"gpsLocationExternal", })
+
+  cfg.pubs = set(cfg.pubs) - sub_keys
+
+
+CONFIGS = [
+  ProcessConfig(
+    proc_name="selfdrived",
+    pubs=[
+      "carState", "deviceState", "pandaStates", "peripheralState", "extrinsicsCalibration", "driverMonitoringState",
+      "longitudinalPlan", "deviceMotion", "lateralDelay", "vehicleParameters", "radarState", "modelV2",
+      "driverCameraState", "roadCameraState", "wideRoadCameraState", "managerState", "lateralTorqueParameters",
+      "accelerometer", "gyroscope", "carOutput", "gpsLocationExternal", "gpsLocation", "controlsState",
+      "carControl", "driverAssistance", "alertDebug", "audioFeedback",
+    ],
+    subs=["selfdriveState", "onroadEvents"],
+    ignore=["logMonoTime", "selfdriveState.alertText1", "selfdriveState.alertText2", "selfdriveState.alertStatus"],
+    config_callback=selfdrived_config_callback,
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("carState", True),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.004,
+  ),
+  ProcessConfig(
+    proc_name="controlsd",
+    pubs=["vehicleParameters", "lateralTorqueParameters", "modelV2", "selfdriveState",
+          "extrinsicsCalibration", "deviceMotion", "longitudinalPlan", "carState", "carOutput",
+          "driverMonitoringState", "onroadEvents", "driverAssistance"],
+    subs=["carControl", "controlsState"],
+    ignore=["logMonoTime", ],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("selfdriveState"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="card",
+    pubs=["pandaStates", "carControl", "onroadEvents", "can"],
+    subs=["sendcan", "carState", "carParams", "carOutput", "radarTracks"],
+    ignore=["logMonoTime", "carState.cumLagMs"],
+    init_callback=card_fingerprint_callback,
+    should_recv_callback=card_rcv_callback,
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.004,
+    main_pub="can",
+    main_pub_drained=True,
+  ),
+  ProcessConfig(
+    proc_name="radard",
+    pubs=["radarTracks", "carState", "modelV2"],
+    subs=["radarState"],
+    ignore=["logMonoTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("modelV2"),
+  ),
+  ProcessConfig(
+    proc_name="plannerd",
+    pubs=["modelV2", "carControl", "carState", "controlsState", "vehicleParameters", "radarState", "selfdriveState"],
+    subs=["longitudinalPlan", "driverAssistance"],
+    ignore=["logMonoTime", "longitudinalPlan.processingDelay", "longitudinalPlan.solverExecutionTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("modelV2"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="calibrationd",
+    pubs=["carState", "cameraOdometry"],
+    subs=["extrinsicsCalibration"],
+    ignore=["logMonoTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("cameraOdometry", True),
+  ),
+  ProcessConfig(
+    proc_name="dmonitoringd",
+    pubs=["driverStateV2", "extrinsicsCalibration", "carState", "modelV2", "selfdriveState"],
+    subs=["driverMonitoringState"],
+    ignore=["logMonoTime"],
+    should_recv_callback=MessageBasedRcvCallback("driverStateV2"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="locationd",
+    pubs=[
+      "cameraOdometry", "accelerometer", "gyroscope", "extrinsicsCalibration", "carState"
+    ],
+    subs=["liveLocationKalman", "deviceMotion"],
+    ignore=["logMonoTime"],
+    should_recv_callback=MessageBasedRcvCallback("cameraOdometry"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="ubloxd",
+    pubs=["ubloxRaw"],
+    subs=["ubloxGnss", "gpsLocationExternal"],
+    ignore=["logMonoTime"],
+  ),
+  ProcessConfig(
+    proc_name="modeld",
+    pubs=["deviceState", "roadCameraState", "wideRoadCameraState", "extrinsicsCalibration", "lateralDelay", "driverMonitoringState", "carState", "carControl"],
+    subs=["modelV2", "drivingModelData", "cameraOdometry"],
+    ignore=["logMonoTime", "modelV2.frameDropPerc", "modelV2.modelExecutionTime", "drivingModelData.frameDropPerc", "drivingModelData.modelExecutionTime"],
+    should_recv_callback=ModeldCameraSyncRcvCallback(),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.020,
+    main_pub=vipc_get_endpoint_name("camerad", meta_from_camera_state("roadCameraState").stream),
+    vision_pubs=["roadCameraState", "wideRoadCameraState"],
+    ignore_alive_pubs=["wideRoadCameraState"],
+    init_callback=get_car_params_callback,
+  ),
+  ProcessConfig(
+    proc_name="dmonitoringmodeld",
+    pubs=["extrinsicsCalibration", "driverCameraState"],
+    subs=["driverStateV2"],
+    ignore=["logMonoTime", "driverStateV2.modelExecutionTime", "driverStateV2.gpuExecutionTime"],
+    should_recv_callback=MessageBasedRcvCallback("driverCameraState"),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.020,
+    main_pub=vipc_get_endpoint_name("camerad", meta_from_camera_state("driverCameraState").stream),
+    vision_pubs=["driverCameraState"],
+    ignore_alive_pubs=["driverCameraState"],
+  ),
+]
+
+
+def get_process_config(name: str) -> ProcessConfig:
+  try:
+    return copy.deepcopy(next(c for c in CONFIGS if c.proc_name == name))
+  except StopIteration as ex:
+    raise Exception(f"Cannot find process config with name: {name}") from ex
+
+
+def get_custom_params_from_lr(lr: LogIterable, initial_state: str = "first") -> dict[str, Any]:
+  """
+  Use this to get custom params dict based on provided logs.
+  Useful when replaying calibrationd.
+  The params may be based on first or last message of given type (carParams, extrinsicsCalibration, vehicleParameters, lateralTorqueParameters) in the logs.
+  """
+
+  car_params = [m for m in lr if m.which() == "carParams"]
+  live_calibration = [m for m in lr if m.which() == "extrinsicsCalibration"]
+  live_parameters = [m for m in lr if m.which() == "vehicleParameters"]
+  live_torque_parameters = [m for m in lr if m.which() == "lateralTorqueParameters"]
+
+  assert initial_state in ["first", "last"]
+  msg_index = 0 if initial_state == "first" else -1
+
+  assert len(car_params) > 0, "carParams required for initial state of vehicleParameters and CarParamsPrevRoute"
+  CP = car_params[msg_index].carParams
+
+  custom_params = {
+    "CarParamsPrevRoute": CP.as_builder().to_bytes()
+  }
+
+  if len(live_calibration) > 0:
+    custom_params["CalibrationParams"] = live_calibration[msg_index].as_builder().to_bytes()
+  if len(live_parameters) > 0:
+    custom_params["LiveParametersV2"] = live_parameters[msg_index].as_builder().to_bytes()
+  if len(live_torque_parameters) > 0:
+    custom_params["LiveTorqueParameters"] = live_torque_parameters[msg_index].as_builder().to_bytes()
+
+  return custom_params
+
+
+def replay_process_with_name(name: str | Iterable[str], lr: LogIterable, *args, **kwargs) -> list[capnp._DynamicStructReader]:
+  if isinstance(name, str):
+    cfgs = [get_process_config(name)]
+  elif isinstance(name, Iterable):
+    cfgs = [get_process_config(n) for n in name]
+  else:
+    raise ValueError("name must be str or collections of strings")
+
+  return replay_process(cfgs, lr, *args, **kwargs)
+
+
+def replay_process(
+  cfg: ProcessConfig | Iterable[ProcessConfig], lr: LogIterable, frs: dict[str, FrameReader] | None = None,
+  fingerprint: str | None = None, return_all_logs: bool = False, custom_params: dict[str, Any] | None = None,
+  captured_output_store: dict[str, dict[str, str]] | None = None, disable_progress: bool = False
+) -> list[capnp._DynamicStructReader]:
+  if isinstance(cfg, Iterable):
+    cfgs = list(cfg)
+  else:
+    cfgs = [cfg]
+
+  all_msgs = migrate_all(lr,
+                         manager_states=True,
+                         panda_states=any("pandaStates" in cfg.pubs for cfg in cfgs),
+                         camera_states=any(len(cfg.vision_pubs) != 0 for cfg in cfgs))
+  process_logs = _replay_multi_process(cfgs, all_msgs, frs, fingerprint, custom_params, captured_output_store, disable_progress)
+
+  if return_all_logs:
+    keys = {m.which() for m in process_logs}
+    modified_logs = [m for m in all_msgs if m.which() not in keys]
+    modified_logs.extend(process_logs)
+    modified_logs.sort(key=lambda m: int(m.logMonoTime))
+    log_msgs = modified_logs
+  else:
+    log_msgs = process_logs
+
+  return log_msgs
+
+
+def _replay_multi_process(
+  cfgs: list[ProcessConfig], lr: LogIterable, frs: dict[str, FrameReader] | None, fingerprint: str | None,
+  custom_params: dict[str, Any] | None, captured_output_store: dict[str, dict[str, str]] | None, disable_progress: bool
+) -> list[capnp._DynamicStructReader]:
+  if fingerprint is not None:
+    params_config = generate_params_config(lr=lr, fingerprint=fingerprint, custom_params=custom_params)
+    env_config = generate_environ_config(fingerprint=fingerprint)
+  else:
+    CP = next((m.carParams for m in lr if m.which() == "carParams"), None)
+    params_config = generate_params_config(lr=lr, CP=CP, custom_params=custom_params)
+    env_config = generate_environ_config(CP=CP)
+
+  # validate frs and vision pubs
+  all_vision_pubs = [pub for cfg in cfgs for pub in cfg.vision_pubs]
+  if len(all_vision_pubs) != 0:
+    assert frs is not None, "frs must be provided when replaying process using vision streams"
+    assert all(meta_from_camera_state(st) is not None for st in all_vision_pubs), \
+                                                          f"undefined vision stream spotted, probably misconfigured process: (vision pubs: {all_vision_pubs})"
+    required_vision_pubs = {m.camera_state for m in available_streams(lr)} & set(all_vision_pubs)
+    assert all(st in frs for st in required_vision_pubs), f"frs for this process must contain following vision streams: {required_vision_pubs}"
+
+  all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
+  log_msgs = []
+  containers = []
+  try:
+    for cfg in cfgs:
+      container = ProcessContainer(cfg)
+      containers.append(container)
+      container.start(params_config, env_config, all_msgs, frs, fingerprint, captured_output_store is not None)
+
+    all_pubs = {pub for container in containers for pub in container.pubs}
+    all_subs = {sub for container in containers for sub in container.subs}
+    lr_pubs = all_pubs - all_subs
+    pubs_to_containers = {pub: [container for container in containers if pub in container.pubs] for pub in all_pubs}
+
+    pub_msgs = [msg for msg in all_msgs if msg.which() in lr_pubs]
+    # external queue for messages taken from logs; internal queue for messages generated by processes, which will be republished
+    external_pub_queue: list[capnp._DynamicStructReader] = pub_msgs.copy()
+    internal_pub_queue: list[capnp._DynamicStructReader] = []
+    # heap for maintaining the order of messages generated by processes, where each element: (logMonoTime, index in internal_pub_queue)
+    internal_pub_index_heap: list[tuple[int, int]] = []
+
+    pbar = tqdm(total=len(external_pub_queue), disable=disable_progress)
+    while len(external_pub_queue) != 0 or (len(internal_pub_index_heap) != 0 and not all(c.has_empty_queue for c in containers)):
+      if len(internal_pub_index_heap) == 0 or (len(external_pub_queue) != 0 and external_pub_queue[0].logMonoTime < internal_pub_index_heap[0][0]):
+        msg = external_pub_queue.pop(0)
+        pbar.update(1)
+      else:
+        _, index = heapq.heappop(internal_pub_index_heap)
+        msg = internal_pub_queue[index]
+
+      target_containers = pubs_to_containers[msg.which()]
+      for container in target_containers:
+        output_msgs = container.run_step(msg, frs)
+        for m in output_msgs:
+          if m.which() in all_pubs:
+            internal_pub_queue.append(m)
+            heapq.heappush(internal_pub_index_heap, (m.logMonoTime, len(internal_pub_queue) - 1))
+        log_msgs.extend(output_msgs)
+
+    # flush last set of messages from each process
+    for container in containers:
+      last_time = log_msgs[-1].logMonoTime if len(log_msgs) > 0 else int(time.monotonic() * 1e9)
+      log_msgs.extend(container.get_output_msgs(last_time))
+  finally:
+    for container in containers:
+      container.stop()
+      if captured_output_store is not None:
+        assert container.capture is not None
+        out, err = container.capture.read_outerr()
+        captured_output_store[container.cfg.proc_name] = {"out": out, "err": err}
+
+  return log_msgs
+
+
+def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=None) -> dict[str, Any]:
+  params_dict = {
+    "OpenpilotEnabledToggle": True,
+    "DisengageOnAccelerator": True,
+    "DisableLogging": False,
+  }
+
+  if custom_params is not None:
+    params_dict.update(custom_params)
+  if lr is not None:
+    has_ublox = any(msg.which() == "ubloxGnss" for msg in lr)
+    params_dict["UbloxAvailable"] = has_ublox
+    is_rhd = next((msg.driverMonitoringState.isRHD for msg in lr if msg.which() == "driverMonitoringState"), False)
+    params_dict["IsRhdDetected"] = is_rhd
+
+  if CP is not None:
+    if fingerprint is None:
+      if CP.fingerprintSource == "fw":
+        params_dict["CarParamsCache"] = CP.as_builder().to_bytes()
+
+    if CP.openpilotLongitudinalControl:
+      params_dict["AlphaLongitudinalEnabled"] = True
+
+    if CP.notCar:
+      params_dict["JoystickDebugMode"] = True
+
+  return params_dict
+
+
+def generate_environ_config(CP=None, fingerprint=None, log_dir=None) -> dict[str, Any]:
+  environ_dict = {}
+  environ_dict["PARAMS_ROOT"] = f"{Paths.shm_path()}/params"
+  if log_dir is not None:
+    environ_dict["LOG_ROOT"] = log_dir
+
+  environ_dict["REPLAY"] = "1"
+
+  # Regen or python process
+  if CP is not None and fingerprint is None:
+    if CP.fingerprintSource == "fw":
+      environ_dict['SKIP_FW_QUERY'] = ""
+      environ_dict['FINGERPRINT'] = ""
+    else:
+      environ_dict['SKIP_FW_QUERY'] = "1"
+      environ_dict['FINGERPRINT'] = CP.carFingerprint
+  elif fingerprint is not None:
+    environ_dict['SKIP_FW_QUERY'] = "1"
+    environ_dict['FINGERPRINT'] = fingerprint
+  else:
+    environ_dict["SKIP_FW_QUERY"] = ""
+    environ_dict["FINGERPRINT"] = ""
+
+  return environ_dict
+
+
+def check_openpilot_enabled(msgs: LogIterable) -> bool:
+  cur_enabled_count = 0
+  max_enabled_count = 0
+  for msg in msgs:
+    if msg.which() == "carParams":
+      if msg.carParams.notCar:
+        return True
+    elif msg.which() == "selfdriveState":
+      if msg.selfdriveState.active:
+        cur_enabled_count += 1
+      else:
+        cur_enabled_count = 0
+      max_enabled_count = max(max_enabled_count, cur_enabled_count)
+
+  return max_enabled_count > int(10. / DT_CTRL)
+
+
+def check_most_messages_valid(msgs: LogIterable, threshold: float = 0.9) -> bool:
+  relevant_services = {sock for cfg in CONFIGS for sock in cfg.subs}
+  msgs_counts = Counter(msg.which() for msg in msgs)
+  msgs_valid_counts = Counter(msg.which() for msg in msgs if msg.valid)
+
+  most_valid_for_service = {}
+  for msg_type in msgs_counts.keys():
+    if msg_type not in relevant_services:
+      continue
+
+    valid_share = msgs_valid_counts.get(msg_type, 0) / msgs_counts[msg_type]
+    ok = valid_share >= threshold
+    if not ok:
+      print(f"WARNING: Service {msg_type} has {valid_share * 100:.2f}% valid messages, which is below threshold of {threshold * 100:.2f}%")
+    most_valid_for_service[msg_type] = ok
+
+  return all(most_valid_for_service.values())

iqpilot/selfdrive/test/process_replay/ref_commit

@@ -0,0 +1 @@
+67f3daf309dc6cbb6844fcbaeb83e6596637e551

iqpilot/selfdrive/test/process_replay/regen.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import os
+import argparse
+import time
+import capnp
+
+from typing import Any
+from collections.abc import Iterable
+
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \
+                                                                   check_openpilot_enabled, check_most_messages_valid, get_custom_params_from_lr
+from iqpilot.selfdrive.test.update_ci_routes import upload_route
+from iqpilot.tools.lib.framereader import FrameReader
+from iqpilot.tools.lib.logreader import LogReader, LogIterable, save_log
+from iqpilot.tools.lib.logreader import get_url
+
+
+def regen_segment(
+  lr: LogIterable, frs: dict[str, Any] | None = None,
+  processes: Iterable[ProcessConfig] = CONFIGS, disable_tqdm: bool = False
+) -> list[capnp._DynamicStructReader]:
+  all_msgs = sorted(lr, key=lambda m: m.logMonoTime)
+  custom_params = get_custom_params_from_lr(all_msgs)
+
+  print("Replayed processes:", [p.proc_name for p in processes])
+  print("\n\n", "*"*30, "\n\n", sep="")
+
+  output_logs = replay_process(processes, all_msgs, frs, return_all_logs=True, custom_params=custom_params, disable_progress=disable_tqdm)
+
+  return output_logs
+
+
+def setup_data_readers(
+    route: str, sidx: int, needs_driver_cam: bool = True, needs_road_cam: bool = True, dummy_driver_cam: bool = False
+) -> tuple[LogReader, dict[str, Any]]:
+  lr = LogReader(f"{route}/{sidx}/r")
+  frs = {}
+  if needs_road_cam:
+    frs['roadCameraState'] = FrameReader(get_url(route, str(sidx), "fcamera.hevc"))
+    if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+      frs['wideRoadCameraState'] = FrameReader(get_url(route, str(sidx), "ecamera.hevc"))
+  if needs_driver_cam:
+    if dummy_driver_cam:
+      frs['driverCameraState'] = FrameReader(get_url(route, str(sidx), "fcamera.hevc")) # Use fcam as dummy
+    else:
+      device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData")
+      assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera."
+      frs['driverCameraState'] = FrameReader(get_url(route, str(sidx), "dcamera.hevc"))
+
+  return lr, frs
+
+
+def regen_and_save(
+  route: str, sidx: int, processes: str | Iterable[str] = "all", outdir: str = FAKEDATA,
+  upload: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False
+) -> str:
+  if not isinstance(processes, str) and not hasattr(processes, "__iter__"):
+    raise ValueError("whitelist_proc must be a string or iterable")
+
+  if processes != "all":
+    if isinstance(processes, str):
+      raise ValueError(f"Invalid value for processes: {processes}")
+
+    replayed_processes = []
+    for d in processes:
+      cfg = get_process_config(d)
+      replayed_processes.append(cfg)
+  else:
+    replayed_processes = CONFIGS
+
+  all_vision_pubs = {pub for cfg in replayed_processes for pub in cfg.vision_pubs}
+  lr, frs = setup_data_readers(route, sidx,
+                               needs_driver_cam="driverCameraState" in all_vision_pubs,
+                               needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs,
+                               dummy_driver_cam=dummy_driver_cam)
+  output_logs = regen_segment(lr, frs, replayed_processes, disable_tqdm=disable_tqdm)
+
+  log_dir = os.path.join(outdir, time.strftime("%Y-%m-%d--%H-%M-%S--0", time.gmtime()))
+  rel_log_dir = os.path.relpath(log_dir)
+  rpath = os.path.join(log_dir, "rlog.zst")
+
+  os.makedirs(log_dir)
+  save_log(rpath, output_logs, compress=True)
+
+  print("\n\n", "*"*30, "\n\n", sep="")
+  print("New route:", rel_log_dir, "\n")
+
+  if not check_openpilot_enabled(output_logs):
+    raise Exception("Route did not engage for long enough")
+  if not check_most_messages_valid(output_logs):
+    raise Exception("Route has too many invalid messages")
+
+  if upload:
+    upload_route(rel_log_dir)
+
+  return rel_log_dir
+
+
+if __name__ == "__main__":
+  def comma_separated_list(string):
+    return string.split(",")
+
+  all_procs = [p.proc_name for p in CONFIGS]
+  parser = argparse.ArgumentParser(description="Generate new segments from old ones")
+  parser.add_argument("--upload", action="store_true", help="Upload the new segment to the CI bucket")
+  parser.add_argument("--outdir", help="log output dir", default=FAKEDATA)
+  parser.add_argument("--dummy-dcamera", action='store_true', help="Use dummy blank driver camera")
+  parser.add_argument("--whitelist-procs", type=comma_separated_list, default=all_procs,
+                      help="Comma-separated whitelist of processes to regen (e.g. controlsd,radard)")
+  parser.add_argument("--blacklist-procs", type=comma_separated_list, default=[],
+                      help="Comma-separated blacklist of processes to regen (e.g. controlsd,radard)")
+  parser.add_argument("route", type=str, help="The source route")
+  parser.add_argument("seg", type=int, help="Segment in source route")
+  args = parser.parse_args()
+
+  blacklist_set = set(args.blacklist_procs)
+  processes = [p for p in args.whitelist_procs if p not in blacklist_set]
+  regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera)

iqpilot/selfdrive/test/process_replay/regen_all.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import argparse
+import concurrent.futures
+import os
+import random
+import traceback
+from tqdm import tqdm
+
+from iqpilot.common.prefix import OpenpilotPrefix
+from iqpilot.selfdrive.test.process_replay.regen import regen_and_save
+from iqpilot.selfdrive.test.process_replay.test_processes import FAKEDATA, source_segments as segments
+from iqpilot.tools.lib.route import SegmentName
+
+
+def regen_job(segment, upload, disable_tqdm):
+  with OpenpilotPrefix():
+    sn = SegmentName(segment[1])
+    fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for _ in range(11))
+    try:
+      relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=upload,
+                            outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm, dummy_driver_cam=True)
+      relr = '|'.join(relr.split('/')[-2:])
+      return f'  ("{segment[0]}", "{relr}"), '
+    except Exception as e:
+      err = f"  {segment} failed: {str(e)}"
+      err += traceback.format_exc()
+      err += "\n\n"
+      return err
+
+
+if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+
+  parser = argparse.ArgumentParser(description="Generate new segments from old ones")
+  parser.add_argument("-j", "--jobs", type=int, default=1)
+  parser.add_argument("--no-upload", action="store_true")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
+  args = parser.parse_args()
+
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+  tested_segments = [(car, segment) for car, segment in segments if car in tested_cars]
+
+  with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
+    p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments))
+    msg = "Copy these new segments into test_processes.py:"
+    for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)):
+      msg += "\n" + str(seg)
+    print()
+    print()
+    print(msg)

iqpilot/selfdrive/test/process_replay/test_compare_logs.py

@@ -0,0 +1,16 @@
+import iqpilot.cereal.messaging as messaging
+from iqpilot.selfdrive.test.process_replay.compare_logs import remove_ignored_fields
+
+
+def test_remove_ignored_text_field():
+  msg = messaging.new_message("selfdriveState")
+  msg.selfdriveState.alertText1 = "IQ.Pilot"
+  cleared = remove_ignored_fields(msg.as_reader(), ["selfdriveState.alertText1"])
+  assert cleared.selfdriveState.alertText1 == ""
+
+
+def test_remove_ignored_enum_field():
+  msg = messaging.new_message("selfdriveState")
+  msg.selfdriveState.alertStatus = "userPrompt"
+  cleared = remove_ignored_fields(msg.as_reader(), ["selfdriveState.alertStatus"])
+  assert str(cleared.selfdriveState.alertStatus) == "normal"

iqpilot/selfdrive/test/process_replay/test_fuzzy.py

@@ -0,0 +1,34 @@
+import copy
+import os
+from hypothesis import given, HealthCheck, Phase, settings
+import hypothesis.strategies as st
+from parameterized import parameterized
+import pytest
+
+from iqpilot.cereal import log
+from iqdbc.car.toyota.values import CAR as TOYOTA
+from iqpilot.selfdrive.test.fuzzy_generation import FuzzyGenerator
+import iqpilot.selfdrive.test.process_replay.process_replay as pr
+
+pytestmark = [pytest.mark.linux, pytest.mark.slow]
+
+# These processes currently fail because of unrealistic data breaking assumptions
+# that openpilot makes causing error with NaN, inf, int size, array indexing ...
+# TODO: Make each one testable
+NOT_TESTED = ['selfdrived', 'controlsd', 'card', 'plannerd', 'calibrationd', 'dmonitoringd', 'estimatord', 'dmonitoringmodeld', 'modeld']
+
+TEST_CASES = [(cfg.proc_name, copy.deepcopy(cfg)) for cfg in pr.CONFIGS if cfg.proc_name not in NOT_TESTED]
+MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "10"))
+
+class TestFuzzProcesses:
+
+  # TODO: make this faster and increase examples
+  @parameterized.expand(TEST_CASES)
+  @given(st.data())
+  @settings(phases=[Phase.generate, Phase.target], max_examples=MAX_EXAMPLES, deadline=1000,
+            suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large])
+  def test_fuzz_process(self, proc_name, cfg, data):
+    msgs = FuzzyGenerator.get_random_event_msg(data.draw, events=cfg.pubs, real_floats=True)
+    lr = [log.Event.new_message(**m).as_reader() for m in msgs]
+    cfg.timeout = 5
+    pr.replay_process(cfg, lr, fingerprint=TOYOTA.TOYOTA_COROLLA_TSS2, disable_progress=True)

iqpilot/selfdrive/test/process_replay/test_migration.py

@@ -0,0 +1,19 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+from iqpilot.cereal import messaging
+from iqpilot.selfdrive.test.process_replay.migration import migrate_drivingModelData
+
+
+def test_driving_model_migration_ignores_incomplete_lane_metadata():
+  msg = messaging.new_message("modelV2")
+  msg.modelV2.init("laneLines", 4)
+  for lane_line in msg.modelV2.laneLines:
+    lane_line.y = [1.0]
+  msg.modelV2.laneLineProbs = [0.5]
+
+  _, added, _ = migrate_drivingModelData([(0, msg.as_reader())])
+
+  assert len(added) == 1
+  assert added[0].drivingModelData.laneLineMeta.leftProb == 0.0
+  assert added[0].drivingModelData.laneLineMeta.rightProb == 0.0

iqpilot/selfdrive/test/process_replay/test_processes.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+import argparse
+import concurrent.futures
+import os
+import sys
+from collections import defaultdict
+from tqdm import tqdm
+from typing import Any
+
+from iqdbc.car.car_helpers import interface_names
+from iqpilot.common.git import get_commit
+from iqpilot.tools.lib.logreader import get_url, upload_file
+from iqpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, replay_process, \
+                                                                   check_most_messages_valid
+from iqpilot.tools.lib.filereader import FileReader
+from iqpilot.tools.lib.logreader import LogReader, save_log
+
+IS_AZURE_TOKEN_DEFINED = os.getenv("AZURE_TOKEN")
+
+source_segments = [
+  ("HYUNDAI", "02c45f73a2e5c6e9|2021-01-01--19-08-22--1"),     # HYUNDAI.HYUNDAI_SONATA
+  ("HYUNDAI2", "d545129f3ca90f28|2022-11-07--20-43-08--3"),    # HYUNDAI.HYUNDAI_KIA_EV6 (+ QCOM GPS)
+  ("TOYOTA", "0982d79ebb0de295|2021-01-04--17-13-21--13"),     # TOYOTA.TOYOTA_PRIUS
+  ("TOYOTA2", "0982d79ebb0de295|2021-01-03--20-03-36--6"),     # TOYOTA.TOYOTA_RAV4
+  ("TOYOTA3", "8011d605be1cbb77|000000cc--8e8d8ec716--6"),     # TOYOTA.TOYOTA_COROLLA_TSS2
+  ("HONDA", "eb140f119469d9ab|2021-06-12--10-46-24--27"),      # HONDA.HONDA_CIVIC (NIDEC)
+  ("HONDA2", "7d2244f34d1bbcda|2021-06-25--12-25-37--26"),     # HONDA.HONDA_ACCORD (BOSCH)
+  ("CHRYSLER", "4deb27de11bee626|2021-02-20--11-28-55--8"),    # CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID
+  ("RAM", "17fc16d840fe9d21|2023-04-26--13-28-44--5"),         # CHRYSLER.RAM_1500_5TH_GEN
+  ("SUBARU", "341dccd5359e3c97|2022-09-12--10-35-33--3"),      # SUBARU.SUBARU_OUTBACK
+  ("GM", "376bf99325883932|2022-10-27--13-41-22--1"),         # GM.CHEVROLET_BOLT_EUV
+  ("NISSAN", "35336926920f3571|2021-02-12--18-38-48--46"),     # NISSAN.NISSAN_XTRAIL
+  ("VOLKSWAGEN", "de9592456ad7d144|2021-06-29--11-00-15--6"),  # VOLKSWAGEN.VOLKSWAGEN_GOLF
+  # FIXME the sensor timings are bad in mazda segment, we're not fully testing it, but it should be replaced
+  ("MAZDA", "bd6a637565e91581|2021-10-30--15-14-53--4"),       # MAZDA.MAZDA_CX9_2021
+  ("FORD", "54827bf84c38b14f|2023-01-26--21-59-07--4"),        # FORD.FORD_BRONCO_SPORT_MK1
+  ("RIVIAN", "bc095dc92e101734|000000db--ee9fe46e57--1"),      # RIVIAN.RIVIAN_R1_GEN1
+  ("TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4"),       # TESLA.TESLA_MODEL_Y
+
+  # Enable when port is tested and dashcamOnly is no longer set
+  #("VOLKSWAGEN2", "3cfdec54aa035f3f|2022-07-19--23-45-10--2"),  # VOLKSWAGEN.VOLKSWAGEN_PASSAT_NMS
+]
+
+segments = [
+  ("HYUNDAI", "regenAA0FC4ED71E|2025-04-08--22-57-50--0"),
+  ("HYUNDAI2", "regenAFB9780D823|2025-04-08--23-00-34--0"),
+  ("TOYOTA", "regen218A4DCFAA1|2025-04-08--22-57-51--0"),
+  # TODO: get new RAV4 route without enableDsu
+  # ("TOYOTA2", "regen107352E20EB|2025-04-08--22-57-46--0"),
+  ("TOYOTA3", "regen1455E3B4BDF|2025-04-09--03-26-06--0"),
+  ("HONDA", "regenB328FF8BA0A|2025-04-08--22-57-45--0"),
+  ("HONDA2", "regen6170C8C9A35|2025-04-08--22-57-46--0"),
+  ("CHRYSLER", "regen5B28FC2A437|2025-04-08--23-04-24--0"),
+  ("RAM", "regenBF81EA96E08|2025-04-08--23-06-54--0"),
+  ("SUBARU", "regen7366F13F6A1|2025-04-08--23-07-07--0"),
+  ("GM", "regen1271097D038|2025-04-09--03-26-00--0"),
+  ("NISSAN", "regen15D60604EAB|2025-04-08--23-06-59--0"),
+  ("VOLKSWAGEN", "regen0F2F06C9539|2025-04-08--23-06-56--0"),
+  ("MAZDA", "regenACF84CCF482|2024-08-30--03-21-55--0"),
+  ("FORD", "regen755D8CB1E1F|2025-04-08--23-13-43--0"),
+  ("RIVIAN", "regen5FCAC896BBE|2025-04-08--23-13-35--0"),
+  ("TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4"),
+]
+
+# dashcamOnly makes don't need to be tested until a full port is done
+excluded_interfaces = ["mock", "body", "psa"]
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit")
+EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"}
+
+
+def preserve_only_specified_files_from_ref_commit(*commits_to_keep):
+  """Keep only files in fakedata that contain any of the specified commit hashes."""
+  removed = 0
+  for f in os.listdir(FAKEDATA):
+    if not any(commit in f for commit in commits_to_keep):
+      os.remove(os.path.join(FAKEDATA, f))
+      removed += 1
+  if removed > 0:
+    print(f"Removed {removed} old files from {FAKEDATA}")
+
+
+def handle_output_file(cur_log_fn, local):
+  """Handle the output file based on whether we're using remote or local storage."""
+  assert os.path.exists(cur_log_fn), f"Cannot find log to upload: {cur_log_fn}"
+
+  if local:
+    os.system(f"git add '{os.path.realpath(cur_log_fn)}'")
+  else:
+    upload_file(cur_log_fn, os.path.basename(cur_log_fn))
+    os.remove(cur_log_fn)
+
+
+def run_test_process(data):
+  segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data
+  res = None
+  if not args.upload_only:
+    lr = LogReader.from_bytes(lr_dat)
+    res, log_msgs = test_process(cfg, lr, segment, ref_log_path, cur_log_fn, args.ignore_fields, args.ignore_msgs)
+    # save logs so we can upload when updating refs
+    save_log(cur_log_fn, log_msgs)
+
+  if args.update_refs or args.upload_only:
+    print(f'Processing: {os.path.basename(cur_log_fn)}')
+    handle_output_file(cur_log_fn, args.local)
+
+  return (segment, cfg.proc_name, res)
+
+
+def get_log_data(segment):
+  r, n = segment.rsplit("--", 1)
+  with FileReader(get_url(r, n, "rlog.zst")) as f:
+    return (segment, f.read())
+
+
+def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=None, ignore_msgs=None):
+  if ignore_fields is None:
+    ignore_fields = []
+  if ignore_msgs is None:
+    ignore_msgs = []
+
+  ref_log_msgs = list(LogReader(ref_log_path))
+
+  try:
+    log_msgs = replay_process(cfg, lr, disable_progress=True)
+  except Exception as e:
+    raise Exception("failed on segment: " + segment) from e
+
+  if not check_most_messages_valid(log_msgs):
+    return f"Route did not have enough valid messages: {new_log_path}", log_msgs
+
+  # skip this check if the segment is using qcom gps
+  if cfg.proc_name != 'ubloxd' or any(m.which() in cfg.pubs for m in lr):
+    seen_msgs = {m.which() for m in log_msgs}
+    expected_msgs = set(cfg.subs)
+    if seen_msgs != expected_msgs:
+      return f"Expected messages: {expected_msgs}, but got: {seen_msgs}", log_msgs
+
+  try:
+    return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs
+  except Exception as e:
+    return str(e), log_msgs
+
+
+def finalize_git_updates(cur_commit, ref_commit_fn):
+  """Finalize git updates and create commit."""
+  try:
+    # Add all new files first
+    os.system(f"git add {os.path.realpath(ref_commit_fn)}")
+    os.system(f"git add {os.path.realpath(FAKEDATA)}/*.zst")
+
+    # Clean up old files - keep only new ref files since they're becoming the reference
+    preserve_only_specified_files_from_ref_commit(cur_commit)
+
+    # Add the deletions to git
+    os.system(f"git add -u {os.path.realpath(FAKEDATA)}")
+
+    # Create the commit
+    commit_msg = f"test_processes: update ref logs to {cur_commit[:7]}"
+    os.system(f'git commit -m "{commit_msg}"')
+    print("Successfully committed reference log updates")
+  except Exception as e:
+    print(f"Failed to commit changes: {e}")
+
+
+if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+  all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS}
+
+  cpu_count = os.cpu_count() or 1
+
+  parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
+  parser.add_argument("--whitelist-procs", type=str, nargs="*", default=all_procs,
+                      help="Whitelist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
+                      help="Blacklist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--ignore-fields", type=str, nargs="*", default=[],
+                      help="Extra fields or msgs to ignore (e.g. driverMonitoringState.events)")
+  parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[],
+                      help="Msgs to ignore (e.g. carEvents)")
+  parser.add_argument("--update-refs", action="store_true",
+                      help="Updates reference logs using current commit")
+  parser.add_argument("--upload-only", action="store_true",
+                      help="Skips testing processes and uploads logs from previous test run")
+  parser.add_argument("--local", action="store_true",
+                      help="Use  local git/ storage instead of remote (Azure for Comma)")
+  parser.add_argument("-j", "--jobs", type=int, default=max(cpu_count - 2, 1),
+                      help="Max amount of parallel jobs")
+  args = parser.parse_args()
+
+  tested_procs = set(args.whitelist_procs) - set(args.blacklist_procs)
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+
+  full_test = (tested_procs == all_procs) and (tested_cars == all_cars) and all(len(x) == 0 for x in (args.ignore_fields, args.ignore_msgs))
+  upload = args.update_refs or args.upload_only
+  os.makedirs(os.path.dirname(FAKEDATA), exist_ok=True)
+
+  if upload:
+    assert full_test, "Need to run full test when updating refs"
+
+  try:
+    with open(REF_COMMIT_FN) as f:
+      ref_commit = f.read().strip()
+  except FileNotFoundError:
+    print("Couldn't find reference commit")
+    sys.exit(1)
+
+  cur_commit = get_commit()
+  if not cur_commit:
+    raise Exception("Couldn't get current commit")
+
+  # Could be set as default in args, but wanted to be more explicit on the flow.
+  if upload and not args.local and not IS_AZURE_TOKEN_DEFINED:
+    print("***** Warning: local/git run was used by default since AZURE_TOKEN was NOT found on the env variables! *****")
+    args.local = True
+
+  # Clean up old files before starting
+  if upload and args.local:
+    print("***** Cleaning up old fakedata for local/git tracked refs *****")
+    preserve_only_specified_files_from_ref_commit(cur_commit, ref_commit)
+
+  print(f"***** testing against commit {ref_commit} *****")
+
+  # check to make sure all car brands are tested
+  if full_test:
+    untested = (set(interface_names) - set(excluded_interfaces)) - {c.lower() for c in tested_cars}
+    assert len(untested) == 0, f"Cars missing routes: {str(untested)}"
+
+  log_paths: defaultdict[str, dict[str, dict[str, str]]] = defaultdict(lambda: defaultdict(dict))
+  with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
+    if not args.upload_only:
+      download_segments = [seg for car, seg in segments if car in tested_cars]
+      log_data: dict[str, LogReader] = {}
+      p1 = pool.map(get_log_data, download_segments)
+      for segment, lr in tqdm(p1, desc="Getting Logs", total=len(download_segments)):
+        log_data[segment] = lr
+
+    pool_args: Any = []
+    for car_brand, segment in segments:
+      if car_brand not in tested_cars:
+        continue
+
+      for cfg in CONFIGS:
+        if cfg.proc_name not in tested_procs:
+          continue
+
+        # to speed things up, we only test all segments on card
+        if cfg.proc_name not in ('card', 'controlsd') and car_brand not in ('HYUNDAI', 'TOYOTA'):
+          continue
+
+        cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst")
+        if args.update_refs:  # reference logs will not exist if routes were just regenerated
+          ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.zst")
+        else:
+          ref_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{ref_commit}.zst")
+          ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn)
+
+        dat = None if args.upload_only else log_data[segment]
+        pool_args.append((segment, cfg, args, cur_log_fn, ref_log_path, dat))
+
+        log_paths[segment][cfg.proc_name]['ref'] = ref_log_path
+        log_paths[segment][cfg.proc_name]['new'] = cur_log_fn
+
+    results: Any = defaultdict(dict)
+    p2 = pool.map(run_test_process, pool_args)
+    for (segment, proc, result) in tqdm(p2, desc="Running Tests", total=len(pool_args)):
+      if not args.upload_only:
+        results[segment][proc] = result
+
+  diff_short, diff_long, failed = format_diff(results, log_paths, ref_commit)
+  if not upload:
+    with open(os.path.join(PROC_REPLAY_DIR, "diff.txt"), "w") as f:
+      f.write(diff_long)
+    print(diff_short)
+
+    if failed:
+      print("TEST FAILED")
+      print("\n\nTo push the new reference logs for this commit run:")
+      print("./test_processes.py --upload-only")
+    else:
+      print("TEST SUCCEEDED")
+
+  else:
+    with open(REF_COMMIT_FN, "w") as f:
+      f.write(cur_commit)
+    print(f"\n\nUpdated reference logs for commit: {cur_commit}")
+
+    # Only do git operations if we're in local mode
+    if upload and args.local:
+      finalize_git_updates(cur_commit, REF_COMMIT_FN)
+
+  sys.exit(int(failed))

iqpilot/selfdrive/test/process_replay/test_regen.py

@@ -0,0 +1,37 @@
+from parameterized import parameterized
+
+from iqpilot.selfdrive.test.process_replay.regen import regen_segment
+from iqpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled
+from iqpilot.tools.lib.logreader import get_url
+from iqpilot.tools.lib.logreader import LogReader
+from iqpilot.tools.lib.framereader import FrameReader
+
+TESTED_SEGMENTS = [
+  ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA.TOYOTA_PRIUS:     NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
+  # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
+  # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"),  # MAZDA.CX9_2021:        TICI, incomplete managerState
+  # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"),   # FORD.BRONCO_SPORT_MK1: TICI
+]
+
+
+def ci_setup_data_readers(route, sidx):
+  lr = LogReader(get_url(route, sidx, "rlog.bz2"))
+  frs = {
+    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")),
+    'driverCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")),
+  }
+  if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera.hevc"))
+
+  return lr, frs
+
+
+class TestRegen:
+  @parameterized.expand(TESTED_SEGMENTS)
+  def test_engaged(self, case_name, segment):
+    route, sidx = segment.rsplit("--", 1)
+    lr, frs = ci_setup_data_readers(route, sidx)
+    output_logs = regen_segment(lr, frs, disable_tqdm=True)
+
+    engaged = check_openpilot_enabled(output_logs)
+    assert engaged, f"openpilot not engaged in {case_name}"

iqpilot/selfdrive/test/process_replay/vision_meta.py

@@ -0,0 +1,43 @@
+from collections import namedtuple
+from iqpilot.cereal.visionipc import VisionStreamType
+from iqpilot.common.realtime import DT_MDL, DT_DMON
+from iqpilot.common.transformations.camera import DEVICE_CAMERAS
+
+VideoStreamMeta = namedtuple("VideoStreamMeta", ["camera_state", "encode_index", "stream", "dt", "frame_sizes"])
+ROAD_CAMERA_FRAME_SIZES = {k: (v.dcam.width, v.dcam.height) for k, v in DEVICE_CAMERAS.items()}
+WIDE_ROAD_CAMERA_FRAME_SIZES = {k: (v.ecam.width, v.ecam.height) for k, v in DEVICE_CAMERAS.items() if v.ecam is not None}
+DRIVER_CAMERA_FRAME_SIZES = {k: (v.dcam.width, v.dcam.height) for k, v in DEVICE_CAMERAS.items()}
+VIPC_STREAM_METADATA = [
+  # metadata: (state_msg_type, encode_msg_type, stream_type, dt, frame_sizes)
+  ("roadCameraState", "roadEncodeIdx", VisionStreamType.VISION_STREAM_ROAD, DT_MDL, ROAD_CAMERA_FRAME_SIZES),
+  ("wideRoadCameraState", "wideRoadEncodeIdx", VisionStreamType.VISION_STREAM_WIDE_ROAD, DT_MDL, WIDE_ROAD_CAMERA_FRAME_SIZES),
+  ("driverCameraState", "driverEncodeIdx", VisionStreamType.VISION_STREAM_DRIVER, DT_DMON, DRIVER_CAMERA_FRAME_SIZES),
+]
+
+
+def meta_from_camera_state(state):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[0] == state), None)
+  return meta
+
+
+def meta_from_encode_index(encode_index):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[1] == encode_index), None)
+  return meta
+
+
+def meta_from_stream_type(stream_type):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[2] == stream_type), None)
+  return meta
+
+
+def available_streams(lr=None):
+  if lr is None:
+    return [VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA]
+
+  result = []
+  for meta in VIPC_STREAM_METADATA:
+    has_cam_state = next((True for m in lr if m.which() == meta[0]), False)
+    if has_cam_state:
+      result.append(VideoStreamMeta(*meta))
+
+  return result

iqpilot/selfdrive/test/scons_build_test.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -e
+
+SCRIPT_DIR=$(dirname "$0")
+BASEDIR=$(realpath "$SCRIPT_DIR/../../../")
+cd $BASEDIR
+
+# tests that our build system's dependencies are configured properly,
+# needs a machine with lots of cores
+
+# helpful commands:
+# scons -Q --tree=derived
+
+scons --clean
+scons --no-cache --random -j$(nproc)
+if ! scons -q; then
+  echo "FAILED: all build products not up to date after first pass."
+  exit 1
+fi

iqpilot/selfdrive/test/setup_device_ci.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+if [ -z "$SOURCE_DIR" ]; then
+  echo "SOURCE_DIR must be set"
+  exit 1
+fi
+
+if [ -z "$GIT_COMMIT" ]; then
+  echo "GIT_COMMIT must be set"
+  exit 1
+fi
+
+if [ -z "$TEST_DIR" ]; then
+  echo "TEST_DIR must be set"
+  exit 1
+fi
+
+# prevent storage from filling up
+rm -rf /data/media/0/realdata/*
+
+rm -rf /data/safe_staging/ || true
+if [ -d /data/safe_staging/ ]; then
+  sudo umount /data/safe_staging/merged/ || true
+  rm -rf /data/safe_staging/ || true
+fi
+
+CONTINUE_PATH="/data/continue.sh"
+tee $CONTINUE_PATH << EOF
+#!/usr/bin/env bash
+
+sudo abctl --set_success
+
+# patch sshd config
+sudo mount -o rw,remount /
+sudo sed -i "s,/data/params/d/GithubSshKeys,/usr/comma/setup_keys," /etc/ssh/sshd_config
+sudo systemctl daemon-reload
+sudo systemctl restart ssh
+sudo systemctl restart NetworkManager
+sudo systemctl disable ssh-param-watcher.path
+sudo systemctl disable ssh-param-watcher.service
+sudo mount -o ro,remount /
+sudo systemctl stop power_monitor
+
+while true; do
+  if ! sudo systemctl is-active -q ssh; then
+    sudo systemctl start ssh
+  fi
+
+  #if ! pgrep -f 'ciui.py' > /dev/null 2>&1; then
+  #  echo 'starting UI'
+  #  cp $SOURCE_DIR/selfdrive/test/ciui.py /data/
+  #  /data/ciui.py &
+  #fi
+
+  sleep 5s
+done
+
+sleep infinity
+EOF
+chmod +x $CONTINUE_PATH
+
+safe_checkout() {
+  # completely clean TEST_DIR
+
+  cd $SOURCE_DIR
+
+  # cleanup orphaned locks
+  find .git -type f -name "*.lock" -exec rm {} +
+
+  git reset --hard
+  git fetch --no-tags -j4 --verbose --depth 1 origin $GIT_COMMIT
+  find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
+  git reset --hard $GIT_COMMIT
+  git checkout $GIT_COMMIT
+  git clean -xdff
+  git lfs pull
+  (ulimit -n 65535 && git lfs prune)
+
+  echo "git checkout done, t=$SECONDS"
+  du -hs $SOURCE_DIR $SOURCE_DIR/.git
+
+  rsync -a --delete $SOURCE_DIR $TEST_DIR
+}
+
+unsafe_checkout() {( set -e
+  # checkout directly in test dir, leave old build products
+
+  cd $TEST_DIR
+
+  # cleanup orphaned locks
+  find .git -type f -name "*.lock" -exec rm {} +
+
+  git fetch --no-tags -j8 --verbose --depth 1 origin $GIT_COMMIT
+  git checkout --force $GIT_COMMIT
+  git reset --hard $GIT_COMMIT
+  git clean -dff
+  git lfs pull
+  (ulimit -n 65535 && git lfs prune)
+)}
+
+export GIT_PACK_THREADS=8
+
+# set up environment
+if [ ! -d "$SOURCE_DIR" ]; then
+  git clone https://github.com/commaai/openpilot.git $SOURCE_DIR
+fi
+
+if [ ! -z "$UNSAFE" ]; then
+  echo "trying unsafe checkout"
+  set +e
+  unsafe_checkout
+  if [[ "$?" -ne 0 ]]; then
+    safe_checkout
+  fi
+  set -e
+else
+  echo "doing safe checkout"
+  safe_checkout
+fi
+
+echo "$TEST_DIR synced with $GIT_COMMIT, t=$SECONDS"

iqpilot/selfdrive/test/setup_vsound.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+{
+  #start pulseaudio daemon
+  sudo pulseaudio -D
+
+  # create a virtual null audio and set it to default device
+  sudo pactl load-module module-null-sink sink_name=virtual_audio
+  sudo pactl set-default-sink virtual_audio
+} > /dev/null 2>&1

iqpilot/selfdrive/test/setup_xvfb.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Sets up a virtual display for running map renderer and simulator without an X11 display
+
+DISP_ID=99
+export DISPLAY=:$DISP_ID
+
+sudo Xvfb $DISPLAY -screen 0 2160x1080x24 2>/dev/null &
+
+# check for x11 socket for the specified display ID
+while [ ! -S /tmp/.X11-unix/X$DISP_ID ]
+do
+  echo "Waiting for Xvfb..."
+  sleep 1
+done
+
+touch ~/.Xauthority
+export XDG_SESSION_TYPE="x11"
+xset -q

iqpilot/selfdrive/test/test_onroad.py

@@ -0,0 +1,452 @@
+import math
+import json
+import os
+import pytest
+import shutil
+import subprocess
+import time
+import numpy as np
+from collections import Counter, defaultdict
+from pathlib import Path
+from tabulate import tabulate
+
+from iqpilot.cereal import log
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal.services import SERVICE_LIST
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.timeout import Timeout
+from iqpilot.common.params import Params
+from iqpilot.selfdrive.selfdrived.events import EVENTS, ET
+from iqpilot.selfdrive.test.helpers import set_params_enabled, release_only
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.tools.lib.logreader import LogReader
+from iqpilot.tools.lib.logreader import msgs_to_time_series
+
+"""
+CPU usage budget
+* each process is entitled to at least 8%
+* total CPU usage of openpilot (sum(PROCS.values())
+  should not exceed MAX_TOTAL_CPU
+"""
+
+TEST_DURATION = 25
+LOG_OFFSET = 8
+
+MAX_TOTAL_CPU = 350.  # total for all 8 cores
+PROCS = {
+  # Baseline CPU usage by process
+  "selfdrive.controls.controlsd": 16.0,
+  "selfdrive.selfdrived.selfdrived": 16.0,
+  "selfdrive.car.card": 26.0,
+  "./loggerd": 14.0,
+  "./encoderd": 13.0,
+  "./camerad": 10.0,
+  "selfdrive.controls.plannerd": 8.0,
+  "selfdrive.ui.ui": 40.0,
+  "system.sensord.sensord": 13.0,
+  "selfdrive.controls.radard": 2.0,
+  "selfdrive.iqmodeld.daemon": 22.0,
+  "selfdrive.dmonitoringmodeld.dmonitoringmodeld": 18.0,
+  "system.hardware.hardwared": 4.0,
+  "selfdrive.locationd.calibrationd": 2.0,
+  "selfdrive.locationd.locationd": 25.0,
+  "selfdrive.locationd.estimatord": 22.0,
+  "selfdrive.ui.soundd": 3.0,
+  "selfdrive.ui.feedback.feedbackd": 1.0,
+  "selfdrive.monitoring.dmonitoringd": 4.0,
+  "system.proclogd": 3.0,
+  "system.logmessaged": 1.0,
+  "system.tombstoned": 0,
+  "system.journald": 1.0,
+  "system.micd": 5.0,
+  "system.timed": 0,
+  "selfdrive.pandad.pandad": 0,
+  "iqpilot_private.konn3kt.uploaderd.iquploaderd": 15.0,
+  "system.loggerd.deleter": 1.0,
+  "./pandad": 19.0,
+  "system.qcomgpsd.qcomgpsd": 1.0,
+}
+
+TIMINGS = {
+  # rtols: max/min, rsd
+  "can": [2.5, 0.35],
+  "pandaStates": [2.5, 0.35],
+  "peripheralState": [2.5, 0.35],
+  "sendcan": [2.5, 0.35],
+  "carState": [2.5, 0.35],
+  "carControl": [2.5, 0.35],
+  "controlsState": [2.5, 0.35],
+  "longitudinalPlan": [2.5, 0.5],
+  "driverAssistance": [2.5, 0.5],
+  "roadCameraState": [2.5, 0.35],
+  "driverCameraState": [2.5, 0.35],
+  "modelV2": [2.5, 0.35],
+  "driverStateV2": [2.5, 0.40],
+  "deviceMotion": [2.5, 0.35],
+  "vehicleParameters": [2.5, 0.35],
+  "wideRoadCameraState": [1.5, 0.35],
+}
+
+LOGS_SIZE = {  # MB per segment
+  "qlog.zst": 0.5,
+  "rlog.zst": 8.1,
+  "qcamera.ts": 2.3,
+}
+LOGS_SIZE.update(dict.fromkeys(['ecamera.hevc', 'fcamera.hevc', 'dcamera.hevc'], 76.5))
+
+
+def cputime_total(ct):
+  return ct.cpuUser + ct.cpuSystem + ct.cpuChildrenUser + ct.cpuChildrenSystem
+
+
+@pytest.mark.tici
+@pytest.mark.skip_tici_setup
+class TestOnroad:
+
+  @classmethod
+  def setup_class(cls):
+    if "DEBUG" in os.environ:
+      segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.zst")), Path(Paths.log_root()).iterdir())
+      segs = sorted(segs, key=lambda x: x.stat().st_mtime)
+      cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.zst")))
+      cls.ts = msgs_to_time_series(cls.lr)
+      return
+
+    # setup env
+    params = Params()
+    params.remove("CurrentRoute")
+    params.put_bool("RecordFront", True)
+    set_params_enabled()
+    os.environ['REPLAY'] = '1'
+    os.environ['MSGQ_PREALLOC'] = '1'
+    os.environ['TESTING_CLOSET'] = '1'
+    if os.path.exists(Paths.log_root()):
+      shutil.rmtree(Paths.log_root())
+
+    # start manager and run openpilot for TEST_DURATION
+    proc = None
+    try:
+      manager_path = os.path.join(BASEDIR, "iqpilot/system/manager/manager.py")
+      cls.manager_st = time.monotonic()
+      proc = subprocess.Popen(["python", manager_path])
+
+      sm = messaging.SubMaster(['carState'])
+      with Timeout(30, "controls didn't start"):
+        while not sm.seen['carState']:
+          sm.update(1000)
+
+      route = params.get("CurrentRoute")
+      assert route is not None
+
+      segs = list(Path(Paths.log_root()).glob(f"{route}--*"))
+      assert len(segs) == 1
+
+      time.sleep(TEST_DURATION)
+    finally:
+      if proc is not None:
+        proc.terminate()
+        if proc.wait(60) is None:
+          proc.kill()
+
+    cls.lr = list(LogReader(os.path.join(str(segs[0]), "rlog.zst")))
+    st = time.monotonic()
+    cls.ts = msgs_to_time_series(cls.lr)
+    print("msgs to time series", time.monotonic() - st)
+    log_path = segs[0]
+
+    cls.log_sizes = {}
+    for f in log_path.iterdir():
+      assert f.is_file()
+      cls.log_sizes[f] = f.stat().st_size / 1e6
+
+    cls.msgs = defaultdict(list)
+    for m in cls.lr:
+      cls.msgs[m.which()].append(m)
+
+  def test_service_frequencies(self, subtests):
+    for s, msgs in self.msgs.items():
+      if s in ('initData', 'sentinel'):
+        continue
+
+      # skip gps services for now
+      if s in ('ubloxGnss', 'ubloxRaw', 'gnssMeasurements', 'gpsLocation', 'gpsLocationExternal', 'qcomGnss'):
+        continue
+
+      with subtests.test(service=s):
+        assert len(msgs) >= math.floor(SERVICE_LIST[s].frequency*int(TEST_DURATION*0.8))
+
+  def test_manager_starting_time(self):
+    st = self.ts['managerState']['t'][0]
+    assert (st - self.manager_st) < 15.0, f"manager.py took {st - self.manager_st}s to publish the first 'managerState' msg"
+
+  def test_cloudlog_size(self):
+    msgs = self.msgs['logMessage']
+
+    total_size = sum(len(m.as_builder().to_bytes()) for m in msgs)
+    assert total_size < 3.5e5
+
+    cnt = Counter(json.loads(m.logMessage)['filename'] for m in msgs)
+    big_logs = [f for f, n in cnt.most_common(3) if n / sum(cnt.values()) > 30.]
+    assert len(big_logs) == 0, f"Log spam: {big_logs}"
+
+  def test_log_sizes(self, subtests):
+    # TODO: this isn't super stable between different devices
+    for f, sz in self.log_sizes.items():
+      rate = LOGS_SIZE[f.name]/60.
+      minn = rate * TEST_DURATION * 0.5
+      maxx = rate * TEST_DURATION * 1.5
+      with subtests.test(file=f.name):
+        assert minn < sz <  maxx
+
+  def test_ui_timings(self):
+    result = "\n"
+    result += "------------------------------------------------\n"
+    result += "-------------- UI Draw Timing ------------------\n"
+    result += "------------------------------------------------\n"
+
+    # other processes preempt ui while starting up
+    offset = int(20 * LOG_OFFSET)
+    ts = self.ts['uiDebug']['drawTimeMillis'][offset:]
+    result += f"min  {min(ts):.2f}ms\n"
+    result += f"max  {max(ts):.2f}ms\n"
+    result += f"std  {np.std(ts):.2f}ms\n"
+    result += f"mean {np.mean(ts):.2f}ms\n"
+    result += "------------------------------------------------\n"
+    print(result)
+
+    assert max(ts) < 250.
+    assert np.mean(ts) < 20.  # TODO: ~6-11ms, increase consistency
+    #self.assertLess(np.std(ts), 5.)
+
+    # some slow frames are expected since camerad/modeld can preempt ui
+    veryslow = [x for x in ts if x > 40.]
+    assert len(veryslow) < 5, f"Too many slow frame draw times: {veryslow}"
+
+  def test_cpu_usage(self, subtests):
+    print("\n------------------------------------------------")
+    print("------------------ CPU Usage -------------------")
+    print("------------------------------------------------")
+
+    plogs_by_proc = defaultdict(list)
+    for pl in self.msgs['procLog']:
+      for x in pl.procLog.procs:
+        if len(x.cmdline) > 0:
+          n = list(x.cmdline)[0]
+          plogs_by_proc[n].append(x)
+
+    cpu_ok = True
+    dt = (self.msgs['procLog'][-1].logMonoTime - self.msgs['procLog'][0].logMonoTime) / 1e9
+    header = ['process', 'usage', 'expected', 'max allowed', 'test result']
+    rows = []
+    for proc_name, expected in PROCS.items():
+
+      error = ""
+      usage = 0.
+      x = plogs_by_proc[proc_name]
+      if len(x) > 2:
+        cpu_time = cputime_total(x[-1]) - cputime_total(x[0])
+        usage = cpu_time / dt * 100.
+
+        max_allowed = max(expected * 1.8, expected + 5.0)
+        if usage > max_allowed:
+          error = "❌ USING MORE CPU THAN EXPECTED ❌"
+          cpu_ok = False
+
+      else:
+        error = "❌ NO METRICS FOUND ❌"
+        cpu_ok = False
+
+      rows.append([proc_name, usage, expected, max_allowed, error or "✅"])
+    print(tabulate(rows, header, tablefmt="simple_grid", stralign="center", numalign="center", floatfmt=".2f"))
+
+    # Ensure there's no missing procs
+    all_procs = {p.name for p in self.msgs['managerState'][0].managerState.processes if p.shouldBeRunning}
+    for p in all_procs:
+      with subtests.test(proc=p):
+        assert any(p in pp for pp in PROCS.keys()), f"Expected CPU usage missing for {p}"
+
+    # total CPU check
+    procs_tot = sum([(max(x) if isinstance(x, tuple) else x) for x in PROCS.values()])
+    with subtests.test(name="total CPU"):
+      assert procs_tot < MAX_TOTAL_CPU, "Total CPU budget exceeded"
+    print("------------------------------------------------")
+    print(f"Total allocated CPU usage is {procs_tot}%, budget is {MAX_TOTAL_CPU}%, {MAX_TOTAL_CPU-procs_tot:.1f}% left")
+    print("------------------------------------------------")
+
+    assert cpu_ok
+
+  def test_memory_usage(self):
+    print("\n------------------------------------------------")
+    print("--------------- Memory Usage -------------------")
+    print("------------------------------------------------")
+    offset = int(SERVICE_LIST['deviceState'].frequency * LOG_OFFSET)
+    mems = [m.deviceState.memoryUsagePercent for m in self.msgs['deviceState'][offset:]]
+    print("Overall memory usage: ", mems)
+    print("MSGQ (/dev/shm/) usage: ", subprocess.check_output(["du", "-hs", "/dev/shm"]).split()[0].decode())
+
+    # check for big leaks. note that memory usage is
+    # expected to go up while the MSGQ buffers fill up
+    assert np.average(mems) <= 80, "Average memory usage too high"
+    assert np.max(np.diff(mems)) <= 4, "Max memory increase too high"
+    assert np.average(np.diff(mems)) <= 1, "Average memory increase too high"
+
+  def test_camera_frame_timings(self, subtests):
+    # test timing within a single camera
+    result = "\n"
+    result += "------------------------------------------------\n"
+    result += "-----------------  SOF Timing ------------------\n"
+    result += "------------------------------------------------\n"
+    for name in ['roadCameraState', 'wideRoadCameraState', 'driverCameraState']:
+      ts = self.ts[name]['timestampSof']
+      d_ms = np.diff(ts) / 1e6
+      d50 = np.abs(d_ms-50)
+      result += f"{name} sof delta vs 50ms: min  {min(d50):.2f}ms\n"
+      result += f"{name} sof delta vs 50ms: max  {max(d50):.2f}ms\n"
+      result += f"{name} sof delta vs 50ms: mean {d50.mean():.2f}ms\n"
+      with subtests.test(camera=name):
+        assert max(d50) < 5.0, f"high SOF delta vs 50ms: {max(d50)}"
+    result += "------------------------------------------------\n"
+    print(result)
+
+  def test_camera_sync(self, subtests):
+    cam_states = ['roadCameraState', 'wideRoadCameraState', 'driverCameraState']
+    encode_cams = ['roadEncodeIdx', 'wideRoadEncodeIdx', 'driverEncodeIdx']
+    for cams in (cam_states, encode_cams):
+      with subtests.test(cams=cams):
+        # sanity checks within a single cam
+        for cam in cams:
+          with subtests.test(test="frame_skips", camera=cam):
+            assert set(np.diff(self.ts[cam]['frameId'])) == {1, }, "Frame ID skips"
+
+            # EOF > SOF
+            eof_sof_diff = self.ts[cam]['timestampEof'] - self.ts[cam]['timestampSof']
+            assert np.all(eof_sof_diff > 0)
+            assert np.all(eof_sof_diff < 50*1e6)
+
+        first_fid = {min(self.ts[c]['frameId']) for c in cams}
+        assert len(first_fid) == 1, "Cameras don't start on same frame ID"
+        if cam.endswith('CameraState'):
+          # camerad guarantees that all cams start on frame ID 0
+          # (note loggerd also needs to start up fast enough to catch it)
+          assert next(iter(first_fid)) < 100, "Cameras start on frame ID too high"
+
+        # we don't do a full segment rotation, so these might not match exactly
+        last_fid = {max(self.ts[c]['frameId']) for c in cams}
+        assert max(last_fid) - min(last_fid) < 10
+
+        start, end = min(first_fid), min(last_fid)
+        for i in range(end-start):
+          ts = {c: round(self.ts[c]['timestampSof'][i]/1e6, 1) for c in cams}
+          diff = (max(ts.values()) - min(ts.values()))
+          assert diff < 2, f"Cameras not synced properly: frame_id={start+i}, {diff=:.1f}ms, {ts=}"
+
+  def test_camera_encoder_matches(self, subtests):
+    # sanity check that the frame metadata is consistent with the encoded frames
+    pairs = [('roadCameraState', 'roadEncodeIdx'),
+             ('wideRoadCameraState', 'wideRoadEncodeIdx'),
+             ('driverCameraState', 'driverEncodeIdx')]
+    for cam, enc in pairs:
+      with subtests.test(camera=cam, encoder=enc):
+        cam_frames = {fid: (sof, eof) for fid, sof, eof in zip(
+          self.ts[cam]['frameId'],
+          self.ts[cam]['timestampSof'],
+          self.ts[cam]['timestampEof'],
+          strict=True,
+        )}
+        for i, fid in enumerate(self.ts[enc]['frameId']):
+          cam_sof, cam_eof = cam_frames[fid]
+          enc_sof, enc_eof = self.ts[enc]['timestampSof'][i], self.ts[enc]['timestampEof'][i]
+          assert enc_sof == cam_sof, f"SOF mismatch: frameId={fid}, enc_sof={enc_sof}, cam_sof={cam_sof}"
+          assert enc_eof == cam_eof, f"EOF mismatch: frameId={fid}, enc_eof={enc_eof}, cam_eof={cam_eof}"
+
+  def test_mpc_execution_timings(self):
+    result = "\n"
+    result += "------------------------------------------------\n"
+    result += "-----------------  MPC Timing ------------------\n"
+    result += "------------------------------------------------\n"
+
+    cfgs = [("longitudinalPlan", 0.05, 0.05),]
+    for (s, instant_max, avg_max) in cfgs:
+      ts = [getattr(m, s).solverExecutionTime for m in self.msgs[s]]
+      assert max(ts) < instant_max, f"high '{s}' execution time: {max(ts)}"
+      assert np.mean(ts) < avg_max, f"high avg '{s}' execution time: {np.mean(ts)}"
+      result += f"'{s}' execution time: min  {min(ts):.5f}s\n"
+      result += f"'{s}' execution time: max  {max(ts):.5f}s\n"
+      result += f"'{s}' execution time: mean {np.mean(ts):.5f}s\n"
+    result += "------------------------------------------------\n"
+    print(result)
+
+  @pytest.mark.model_validation
+  def test_model_execution_timings(self, subtests):
+    result = "\n"
+    result += "------------------------------------------------\n"
+    result += "----------------- Model Timing -----------------\n"
+    result += "------------------------------------------------\n"
+    cfgs = [
+      # since multiple processes use the GPU and can preempt each other,
+      # these numbers are not fully self-contained.
+      ("modelV2", 0.06, 0.040),
+
+      # can miss cycles here and there, just important the avg frequency is 20Hz
+      ("driverStateV2", 0.3, 0.05),
+    ]
+    for (s, instant_max, avg_max) in cfgs:
+      ts = [getattr(m, s).modelExecutionTime for m in self.msgs[s]]
+      # TODO some init can happen in first iteration
+      ts = ts[1:]
+      result += f"'{s}' execution time: min  {min(ts):.5f}s\n"
+      result += f"'{s}' execution time: max {max(ts):.5f}s\n"
+      result += f"'{s}' execution time: mean {np.mean(ts):.5f}s\n"
+      with subtests.test(s):
+        assert max(ts) < instant_max, f"high '{s}' execution time: {max(ts)}"
+        assert np.mean(ts) < avg_max, f"high avg '{s}' execution time: {np.mean(ts)}"
+    result += "------------------------------------------------\n"
+    print(result)
+
+  def test_timings(self):
+    passed = True
+    print("\n------------------------------------------------")
+    print("----------------- Service Timings --------------")
+    print("------------------------------------------------")
+
+    header = ['service', 'max', 'min', 'mean', 'expected mean', 'rsd', 'max allowed rsd', 'test result']
+    rows = []
+    for s, (maxmin, rsd) in TIMINGS.items():
+      offset = int(SERVICE_LIST[s].frequency * LOG_OFFSET)
+      msgs = [m.logMonoTime for m in self.msgs[s][offset:]]
+      if not len(msgs):
+        raise Exception(f"missing {s}")
+
+      ts = np.diff(msgs) / 1e9
+      dt = 1 / SERVICE_LIST[s].frequency
+
+      errors = []
+      if not np.allclose(np.mean(ts), dt, rtol=0.03, atol=0):
+        errors.append("❌ FAILED MEAN TIMING CHECK ❌")
+      if not np.allclose([np.max(ts), np.min(ts)], dt, rtol=maxmin, atol=0):
+        errors.append("❌ FAILED MAX/MIN TIMING CHECK ❌")
+      if (np.std(ts)/dt) > rsd:
+        errors.append("❌ FAILED RSD TIMING CHECK ❌")
+      passed = not errors and passed
+      rows.append([s, *(np.array([np.max(ts), np.min(ts), np.mean(ts), dt])*1e3), np.std(ts)/dt, rsd, "\n".join(errors) or "✅"])
+
+    print(tabulate(rows, header, tablefmt="simple_grid", stralign="center", numalign="center", floatfmt=".2f"))
+    assert passed
+
+  @release_only
+  def test_startup(self):
+    startup_alert = self.ts['selfdriveState']['alertText1'][0]
+    expected = EVENTS[log.OnroadEvent.EventName.startup][ET.PERMANENT].alert_text_1
+    assert startup_alert == expected, "wrong startup alert"
+
+  def test_engagable(self):
+    no_entries = Counter()
+    for m in self.msgs['onroadEvents']:
+      for evt in m.onroadEvents:
+        if evt.noEntry:
+          no_entries[evt.name] += 1
+
+    offset = int(SERVICE_LIST['selfdriveState'].frequency * LOG_OFFSET)
+    eng = [m.selfdriveState.engageable for m in self.msgs['selfdriveState'][offset:]]
+    assert all(eng), \
+           f"Not engageable for whole segment:\n- selfdriveState.engageable: {Counter(eng)}\n- No entry events: {no_entries}"

iqpilot/selfdrive/test/update_ci_routes.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import os
+import re
+import subprocess
+import sys
+from collections.abc import Iterable
+from datetime import datetime, timedelta, UTC
+from functools import lru_cache
+from pathlib import Path
+from typing import IO
+
+from tqdm import tqdm
+
+from iqdbc.car.tests.routes import routes as test_car_models_routes
+from iqpilot.selfdrive.test.process_replay.test_processes import source_segments as replay_segments
+
+TOKEN_PATH = Path("/data/azure_token")
+
+
+@lru_cache
+def get_azure_credential():
+  if "AZURE_TOKEN" in os.environ:
+    return os.environ["AZURE_TOKEN"]
+  if TOKEN_PATH.is_file():
+    return TOKEN_PATH.read_text().strip()
+  from azure.identity import AzureCliCredential
+  return AzureCliCredential()
+
+
+@lru_cache
+def get_container_sas(account_name: str, container_name: str):
+  from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas
+  start_time = datetime.now(UTC).replace(tzinfo=None)
+  expiry_time = start_time + timedelta(hours=1)
+  blob_service = BlobServiceClient(account_url=f"https://{account_name}.blob.core.windows.net", credential=get_azure_credential())
+  return generate_container_sas(account_name, container_name,
+                                user_delegation_key=blob_service.get_user_delegation_key(start_time, expiry_time),
+                                permission=ContainerSasPermissions(read=True, write=True, list=True), expiry=expiry_time)
+
+
+class AzureContainer:
+  def __init__(self, account, container):
+    self.ACCOUNT = account
+    self.CONTAINER = container
+
+  @property
+  def ACCOUNT_URL(self) -> str:
+    return f"https://{self.ACCOUNT}.blob.core.windows.net"
+
+  @property
+  def BASE_URL(self) -> str:
+    return f"{self.ACCOUNT_URL}/{self.CONTAINER}/"
+
+  def get_client_and_key(self):
+    from azure.storage.blob import ContainerClient
+    return ContainerClient(self.ACCOUNT_URL, self.CONTAINER, credential=get_azure_credential()), get_container_sas(self.ACCOUNT, self.CONTAINER)
+
+  def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
+    from azure.storage.blob import BlobClient
+    client = BlobClient(account_url=self.ACCOUNT_URL, container_name=self.CONTAINER, blob_name=blob_name, credential=get_azure_credential())
+    client.upload_blob(data, overwrite=overwrite)
+    return self.BASE_URL + blob_name
+
+  def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str:
+    with open(path, "rb") as f:
+      return self.upload_bytes(f, blob_name, overwrite)
+
+
+DataCIContainer = AzureContainer("commadataci", "commadataci")
+DataProdContainer = AzureContainer("commadata2", "commadata2")
+OpenpilotCIContainer = AzureContainer("commadataci", "openpilotci")
+
+SOURCES: list[AzureContainer] = [
+  DataProdContainer,
+  DataCIContainer
+]
+
+DEST = OpenpilotCIContainer
+
+def upload_route(path: str, exclude_patterns: Iterable[str] | None = None) -> None:
+  if exclude_patterns is None:
+    exclude_patterns = [r'dcamera\.hevc']
+
+  r, n = path.rsplit("--", 1)
+  r = '/'.join(r.split('/')[-2:])  # strip out anything extra in the path
+  destpath = f"{r}/{n}"
+  for file in os.listdir(path):
+    if any(re.search(pattern, file) for pattern in exclude_patterns):
+      continue
+    DEST.upload_file(os.path.join(path, file), f"{destpath}/{file}")
+
+
+def sync_to_ci_public(route: str) -> bool:
+  dest_container, dest_key = DEST.get_client_and_key()
+  key_prefix = route.replace('|', '/')
+  dongle_id = key_prefix.split('/')[0]
+
+  if next(dest_container.list_blob_names(name_starts_with=key_prefix), None) is not None:
+    return True
+
+  print(f"Uploading {route}")
+  for source_container in SOURCES:
+    # assumes az login has been run
+    print(f"Trying {source_container.ACCOUNT}/{source_container.CONTAINER}")
+    _, source_key = source_container.get_client_and_key()
+    cmd = [
+      "azcopy",
+      "copy",
+      f"{source_container.BASE_URL}{key_prefix}?{source_key}",
+      f"{DEST.BASE_URL}{dongle_id}?{dest_key}",
+      "--recursive=true",
+      "--overwrite=false",
+      "--exclude-pattern=*/dcamera.hevc",
+    ]
+
+    try:
+      result = subprocess.call(cmd, stdout=subprocess.DEVNULL)
+      if result == 0:
+        print("Success")
+        return True
+    except subprocess.CalledProcessError:
+      print("Failed")
+
+  return False
+
+
+if __name__ == "__main__":
+  failed_routes = []
+
+  to_sync = sys.argv[1:]
+
+  if not len(to_sync):
+    # sync routes from the car tests routes and process replay
+    to_sync.extend([rt.route for rt in test_car_models_routes])
+    to_sync.extend([s[1].rsplit('--', 1)[0] for s in replay_segments])
+
+  for r in tqdm(to_sync):
+    if not sync_to_ci_public(r):
+      failed_routes.append(r)
+
+  if len(failed_routes):
+    print("failed routes:", failed_routes)