IQ.Pilot Release Commit @ b6534c0

iqpilot/selfdrive/test/process_replay/.gitignore

@@ -0,0 +1 @@
+fakedata/

iqpilot/selfdrive/test/process_replay/README.md

@@ -0,0 +1,122 @@
+# Process replay
+
+Process replay is a regression test designed to identify any changes in the output of a process. This test replays a segment through individual processes and compares the output to a known good replay. Each make is represented in the test with a segment.
+
+If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated.
+
+Use `test_processes.py` to run the test locally.
+Log files are cached by default. Use `DISABLE_FILEREADER_CACHE='1' test_processes.py` to disable caching.
+
+Currently the following processes are tested:
+
+* controlsd
+* radard
+* plannerd
+* calibrationd
+* dmonitoringd
+* locationd
+* ubloxd
+
+### Usage
+```
+Usage: test_processes.py [-h] [--whitelist-procs PROCS] [--whitelist-cars CARS] [--blacklist-procs PROCS]
+                         [--blacklist-cars CARS] [--ignore-fields FIELDS] [--ignore-msgs MSGS] [--update-refs] [--upload-only]
+Regression test to identify changes in a process's output
+optional arguments:
+  -h, --help            show this help message and exit
+  --whitelist-procs PROCS               Whitelist given processes from the test (e.g. controlsd)
+  --whitelist-cars WHITELIST_CARS       Whitelist given cars from the test (e.g. HONDA)
+  --blacklist-procs BLACKLIST_PROCS     Blacklist given processes from the test (e.g. controlsd)
+  --blacklist-cars BLACKLIST_CARS       Blacklist given cars from the test (e.g. HONDA)
+  --ignore-fields IGNORE_FIELDS         Extra fields or msgs to ignore (e.g. driverMonitoringState.events)
+  --ignore-msgs IGNORE_MSGS             Msgs to ignore (e.g. onroadEvents)
+  --update-refs                         Updates reference logs using current commit
+  --upload-only                         Skips testing processes and uploads logs from previous test run
+```
+
+## Forks
+
+openpilot forks can use this test with their own reference logs, by default `test_proccesses.py` saves logs locally.
+
+To generate new logs:
+
+`./test_processes.py`
+
+Then, check in the new logs using git-lfs. Make sure to also update the `ref_commit` file to the current commit.
+
+## API
+
+Process replay test suite exposes programmatic APIs for simultaneously running processes or groups of processes on provided logs.
+
+```py
+def replay_process_with_name(name: Union[str, Iterable[str]], lr: LogIterable, *args, **kwargs) -> List[capnp._DynamicStructReader]:
+
+def replay_process(
+  cfg: Union[ProcessConfig, Iterable[ProcessConfig]], lr: LogIterable, frs: Optional[Dict[str, Any]] = None,
+  fingerprint: Optional[str] = None, return_all_logs: bool = False, custom_params: Optional[Dict[str, Any]] = None, disable_progress: bool = False
+) -> List[capnp._DynamicStructReader]:
+```
+
+Example usage:
+```py
+from iqpilot.selfdrive.test.process_replay import replay_process_with_name
+from iqpilot.tools.lib.logreader import LogReader
+
+lr = LogReader(...)
+
+# provide a name of the process to replay
+output_logs = replay_process_with_name('locationd', lr)
+
+# or list of names
+output_logs = replay_process_with_name(['ubloxd', 'locationd'], lr)
+```
+
+Supported processes:
+* controlsd
+* radard
+* plannerd
+* calibrationd
+* dmonitoringd
+* locationd
+* ubloxd
+* modeld
+* dmonitoringmodeld
+
+Certain processes may require an initial state, which is usually supplied within `Params` and persisting from segment to segment (e.g CalibrationParams, LiveParameters). The `custom_params` is dictionary  used to prepopulate `Params` with arbitrary values. The `get_custom_params_from_lr` helper is provided to fetch meaningful values from log files.
+
+```py
+from iqpilot.selfdrive.test.process_replay import get_custom_params_from_lr
+
+previous_segment_lr = LogReader(...)
+current_segment_lr = LogReader(...)
+
+custom_params = get_custom_params_from_lr(previous_segment_lr, 'last')
+
+output_logs = replay_process_with_name('calibrationd', lr, custom_params=custom_params)
+```
+
+Replaying processes that use VisionIPC (e.g. modeld, dmonitoringmodeld) require additional `frs` dictionary with camera states as keys and `FrameReader` objects as values.
+
+```py
+from iqpilot.tools.lib.framereader import FrameReader
+
+frs = {
+  'roadCameraState': FrameReader(...),
+  'wideRoadCameraState': FrameReader(...),
+  'driverCameraState': FrameReader(...),
+}
+
+output_logs = replay_process_with_name(['modeld', 'dmonitoringmodeld'], lr, frs=frs)
+```
+
+To capture stdout/stderr of the replayed process, `captured_output_store` can be provided.
+
+```py
+output_store = dict()
+# pass dictionary by reference, it will be filled with standard outputs - even if process replay fails
+output_logs = replay_process_with_name(['radard', 'plannerd'], lr, captured_output_store=output_store)
+
+# entries with captured output in format { 'out': '...', 'err': '...' } will be added to provided dictionary for each replayed process
+print(output_store['radard']['out']) # radard stdout
+print(output_store['radard']['err']) # radard stderr
+```

iqpilot/selfdrive/test/process_replay/__init__.py

@@ -0,0 +1,2 @@
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, get_process_config, get_custom_params_from_lr, \
+                                                                  replay_process, replay_process_with_name  # noqa: F401

iqpilot/selfdrive/test/process_replay/capture.py

@@ -0,0 +1,59 @@
+import os
+import sys
+
+from typing import no_type_check
+
+class FdRedirect:
+  def __init__(self, file_prefix: str, fd: int):
+    fname = os.path.join("/tmp", f"{file_prefix}.{fd}")
+    if os.path.exists(fname):
+      os.unlink(fname)
+    self.dest_fd = os.open(fname, os.O_WRONLY | os.O_CREAT)
+    self.dest_fname = fname
+    self.source_fd = fd
+    os.set_inheritable(self.dest_fd, True)
+
+  def __del__(self):
+    os.close(self.dest_fd)
+
+  def purge(self) -> None:
+    os.unlink(self.dest_fname)
+
+  def read(self) -> bytes:
+    with open(self.dest_fname, "rb") as f:
+      return f.read() or b""
+
+  def link(self) -> None:
+    os.dup2(self.dest_fd, self.source_fd)
+
+
+class ProcessOutputCapture:
+  def __init__(self, proc_name: str, prefix: str):
+    prefix = f"{proc_name}_{prefix}"
+    self.stdout_redirect = FdRedirect(prefix, 1)
+    self.stderr_redirect = FdRedirect(prefix, 2)
+
+  def __del__(self):
+    self.stdout_redirect.purge()
+    self.stderr_redirect.purge()
+
+  @no_type_check # ipython classes have incompatible signatures
+  def link_with_current_proc(self) -> None:
+    try:
+      # prevent ipykernel from redirecting stdout/stderr of python subprocesses
+      from ipykernel.iostream import OutStream
+      if isinstance(sys.stdout, OutStream):
+        sys.stdout = sys.__stdout__
+      if isinstance(sys.stderr, OutStream):
+        sys.stderr = sys.__stderr__
+    except ImportError:
+      pass
+
+    # link stdout/stderr to the fifo
+    self.stdout_redirect.link()
+    self.stderr_redirect.link()
+
+  def read_outerr(self) -> tuple[str, str]:
+    out_str = self.stdout_redirect.read().decode()
+    err_str = self.stderr_redirect.read().decode()
+    return out_str, err_str

iqpilot/selfdrive/test/process_replay/compare_logs.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+import sys
+import math
+import capnp
+import numbers
+import dictdiffer
+from collections import Counter
+
+from iqpilot.tools.lib.logreader import LogReader
+
+EPSILON = sys.float_info.epsilon
+
+
+def remove_ignored_fields(msg, ignore):
+  msg = msg.as_builder()
+  for key in ignore:
+    attr = msg
+    keys = key.split(".")
+    if msg.which() != keys[0] and len(keys) > 1:
+      continue
+
+    for k in keys[:-1]:
+      # indexing into list
+      if k.isdigit():
+        attr = attr[int(k)]
+      else:
+        attr = getattr(attr, k)
+
+    v = getattr(attr, keys[-1])
+    if isinstance(v, bool):
+      val = False
+    elif isinstance(v, numbers.Number):
+      val = 0
+    elif isinstance(v, (list, capnp.lib.capnp._DynamicListBuilder)):
+      val = []
+    elif isinstance(v, str):
+      val = ""
+    elif isinstance(v, capnp.lib.capnp._DynamicEnum):
+      val = 0
+    else:
+      raise NotImplementedError(f"Unknown type: {type(v)}")
+    setattr(attr, keys[-1], val)
+  return msg
+
+
+def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None,):
+  if ignore_fields is None:
+    ignore_fields = []
+  if ignore_msgs is None:
+    ignore_msgs = []
+  tolerance = EPSILON if tolerance is None else tolerance
+
+  log1, log2 = (
+    [m for m in log if m.which() not in ignore_msgs]
+    for log in (log1, log2)
+  )
+
+  if len(log1) != len(log2):
+    cnt1 = Counter(m.which() for m in log1)
+    cnt2 = Counter(m.which() for m in log2)
+    raise Exception(f"logs are not same length: {len(log1)} VS {len(log2)}\n\t\t{cnt1}\n\t\t{cnt2}")
+
+  diff = []
+  for msg1, msg2 in zip(log1, log2, strict=True):
+    if msg1.which() != msg2.which():
+      raise Exception("msgs not aligned between logs")
+
+    msg1 = remove_ignored_fields(msg1, ignore_fields)
+    msg2 = remove_ignored_fields(msg2, ignore_fields)
+
+    if msg1.to_bytes() != msg2.to_bytes():
+      msg1_dict = msg1.as_reader().to_dict(verbose=True)
+      msg2_dict = msg2.as_reader().to_dict(verbose=True)
+
+      dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
+
+      # Dictdiffer only supports relative tolerance, we also want to check for absolute
+      # TODO: add this to dictdiffer
+      def outside_tolerance(diff):
+        try:
+          if diff[0] == "change":
+            a, b = diff[2]
+            finite = math.isfinite(a) and math.isfinite(b)
+            if finite and isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
+              return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
+        except TypeError:
+          pass
+        return True
+
+      dd = list(filter(outside_tolerance, dd))
+
+      diff.extend(dd)
+  return diff
+
+
+def format_process_diff(diff):
+  diff_short, diff_long = "", ""
+
+  if isinstance(diff, str):
+    diff_short += f"        {diff}\n"
+    diff_long += f"\t{diff}\n"
+  else:
+    cnt: dict[str, int] = {}
+    for d in diff:
+      diff_long += f"\t{str(d)}\n"
+
+      k = str(d[1])
+      cnt[k] = 1 if k not in cnt else cnt[k] + 1
+
+    for k, v in sorted(cnt.items()):
+      diff_short += f"        {k}: {v}\n"
+
+  return diff_short, diff_long
+
+
+def format_diff(results, log_paths, ref_commit):
+  diff_short, diff_long = "", ""
+  diff_long += f"***** tested against commit {ref_commit} *****\n"
+
+  failed = False
+  for segment, result in list(results.items()):
+    diff_short += f"***** results for segment {segment} *****\n"
+    diff_long += f"***** differences for segment {segment} *****\n"
+
+    for proc, diff in list(result.items()):
+      diff_long += f"*** process: {proc} ***\n"
+      diff_long += f"\tref: {log_paths[segment][proc]['ref']}\n"
+      diff_long += f"\tnew: {log_paths[segment][proc]['new']}\n\n"
+
+      diff_short += f"    {proc}\n"
+
+      if isinstance(diff, str) or len(diff):
+        diff_short += f"        ref: {log_paths[segment][proc]['ref']}\n"
+        diff_short += f"        new: {log_paths[segment][proc]['new']}\n\n"
+        failed = True
+
+        proc_diff_short, proc_diff_long = format_process_diff(diff)
+
+        diff_long += proc_diff_long
+        diff_short += proc_diff_short
+
+  return diff_short, diff_long, failed
+
+
+if __name__ == "__main__":
+  log1 = list(LogReader(sys.argv[1]))
+  log2 = list(LogReader(sys.argv[2]))
+  ignore_fields = sys.argv[3:] or ["logMonoTime"]
+  results = {"segment": {"proc": compare_logs(log1, log2, ignore_fields)}}
+  log_paths = {"segment": {"proc": {"ref": sys.argv[1], "new": sys.argv[2]}}}
+  diff_short, diff_long, failed = format_diff(results, log_paths, None)
+
+  print(diff_long)
+  print(diff_short)

iqpilot/selfdrive/test/process_replay/migration.py

@@ -0,0 +1,477 @@
+from collections import defaultdict
+from collections.abc import Callable
+from typing import cast
+import capnp
+import functools
+import traceback
+
+from iqpilot.cereal import messaging, car, log
+from iqdbc.car.fingerprints import MIGRATION
+from iqdbc.car.toyota.values import EPS_SCALE, ToyotaSafetyFlags
+from iqdbc.car.ford.values import CAR as FORD, FordFlags, FordSafetyFlags
+from iqdbc.car.hyundai.values import HyundaiSafetyFlags
+from iqdbc.car.gm.values import GMSafetyFlags
+from iqpilot.selfdrive.iqmodeld.config import ModelConstants
+from iqpilot.selfdrive.iqmodeld.messaging import fill_xyz_poly, fill_lane_line_meta
+from iqpilot.selfdrive.test.process_replay.vision_meta import meta_from_encode_index
+from iqpilot.selfdrive.controls.lib.longitudinal_planner import get_accel_from_plan, CONTROL_N_T_IDX
+from iqpilot.tools.lib.logreader import LogIterable
+
+MessageWithIndex = tuple[int, capnp.lib.capnp._DynamicStructReader]
+MigrationOps = tuple[list[tuple[int, capnp.lib.capnp._DynamicStructReader]], list[capnp.lib.capnp._DynamicStructReader], list[int]]
+MigrationFunc = Callable[[list[MessageWithIndex]], MigrationOps]
+
+
+# rules for migration functions
+# 1. must use the decorator @migration(inputs=[...], product="...") and MigrationFunc signature
+# 2. it only gets the messages that are in the inputs list
+# 3. product is the message type created by the migration function, and the function will be skipped if product type already exists in lr
+# 4. it must return a list of operations to be applied to the logreader (replace, add, delete)
+# 5. all migration functions must be independent of each other
+def migrate_all(lr: LogIterable, manager_states: bool = False, panda_states: bool = False, camera_states: bool = False,
+                live_location_kalman: bool = True):
+  migrations = [
+    migrate_sensorEvents,
+    migrate_carParams,
+    migrate_gpsLocation,
+    migrate_deviceState,
+    migrate_carOutput,
+    migrate_controlsState,
+    migrate_carState,
+    migrate_radarTracks,
+    migrate_driverAssistance,
+    migrate_drivingModelData,
+    migrate_onroadEvents,
+    migrate_driverMonitoringState,
+    migrate_longitudinalPlan,
+  ]
+  if manager_states:
+    migrations.append(migrate_managerState)
+  if panda_states:
+    migrations.extend([migrate_pandaStates, migrate_peripheralState])
+  if camera_states:
+    migrations.append(migrate_cameraStates)
+  if live_location_kalman:
+    migrations.append(migrate_liveLocationKalman)
+
+  return migrate(lr, migrations)
+
+
+def migrate(lr: LogIterable, migration_funcs: list[MigrationFunc]):
+  lr = list(lr)
+  grouped = defaultdict(list)
+  for i, msg in enumerate(lr):
+    grouped[msg.which()].append(i)
+
+  replace_ops, add_ops, del_ops = [], [], []
+  for migration in migration_funcs:
+    assert hasattr(migration, "inputs") and hasattr(migration, "product"), "Migration functions must use @migration decorator"
+    if migration.product in grouped: # skip if product already exists
+      continue
+
+    sorted_indices = sorted(ii for i in cast(list[str], migration.inputs) for ii in grouped.get(i, []))
+    msg_gen = [(i, lr[i]) for i in sorted_indices]
+    r_ops, a_ops, d_ops = migration(msg_gen)
+    replace_ops.extend(r_ops)
+    add_ops.extend(a_ops)
+    del_ops.extend(d_ops)
+
+  for index, msg in replace_ops:
+    lr[index] = msg
+  for index in sorted(del_ops, reverse=True):
+    del lr[index]
+  for msg in add_ops:
+    lr.append(msg)
+  lr = sorted(lr, key=lambda x: x.logMonoTime)
+
+  return lr
+
+
+def migration(inputs: list[str], product: str|None=None):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      return func(*args, **kwargs)
+    wrapper.inputs = inputs
+    wrapper.product = product
+    return wrapper
+  return decorator
+
+
+@migration(inputs=["longitudinalPlan", "carParams"])
+def migrate_longitudinalPlan(msgs):
+  ops = []
+
+  needs_migration = all(msg.longitudinalPlan.aTarget == 0.0 for _, msg in msgs if msg.which() == 'longitudinalPlan')
+  CP = next((m.carParams for _, m in msgs if m.which() == 'carParams'), None)
+  if not needs_migration or CP is None:
+    return [], [], []
+
+  for index, msg in msgs:
+    if msg.which() != 'longitudinalPlan':
+      continue
+    new_msg = msg.as_builder()
+    a_target, should_stop = get_accel_from_plan(msg.longitudinalPlan.speeds, msg.longitudinalPlan.accels, CONTROL_N_T_IDX)
+    new_msg.longitudinalPlan.aTarget, new_msg.longitudinalPlan.shouldStop = float(a_target), bool(should_stop)
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["longitudinalPlan"], product="driverAssistance")
+def migrate_driverAssistance(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    new_msg = messaging.new_message('driverAssistance', valid=True, logMonoTime=msg.logMonoTime)
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["modelV2"], product="drivingModelData")
+def migrate_drivingModelData(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    dmd = messaging.new_message('drivingModelData', valid=msg.valid, logMonoTime=msg.logMonoTime)
+    for field in ["frameId", "frameIdExtra", "frameDropPerc", "modelExecutionTime", "action"]:
+      setattr(dmd.drivingModelData, field, getattr(msg.modelV2, field))
+    for meta_field in ["laneChangeState", "laneChangeState"]:
+      setattr(dmd.drivingModelData.meta, meta_field, getattr(msg.modelV2.meta, meta_field))
+    lane_lines = msg.modelV2.laneLines
+    lane_probs = msg.modelV2.laneLineProbs
+    if len(lane_lines) > 2 and len(lane_probs) > 2 and len(lane_lines[1].y) and len(lane_lines[2].y):
+      fill_lane_line_meta(dmd.drivingModelData.laneLineMeta, msg.modelV2.laneLines, msg.modelV2.laneLineProbs)
+    if all(len(a) for a in [msg.modelV2.position.x, msg.modelV2.position.y, msg.modelV2.position.z]):
+      fill_xyz_poly(dmd.drivingModelData.path, ModelConstants.POLY_PATH_DEGREE, msg.modelV2.position.x, msg.modelV2.position.y, msg.modelV2.position.z)
+    add_ops.append( dmd.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["radarTracksDEPRECATED"], product="radarTracks")
+def migrate_radarTracks(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = messaging.new_message('radarTracks')
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+
+    pts = []
+    for track in msg.radarTracksDEPRECATED:
+      pt = car.RadarData.RadarPoint()
+      pt.trackId = track.trackId
+
+      pt.dRel = track.dRel
+      pt.yRel = track.yRel
+      pt.vRel = track.vRel
+      pt.aRel = track.aRel
+      pt.measured = True
+      pts.append(pt)
+
+    new_msg.radarTracks.points = pts
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["liveLocationKalmanDEPRECATED"], product="deviceMotion")
+def migrate_liveLocationKalman(msgs):
+  nans = [float('nan')] * 3
+  ops = []
+  for index, msg in msgs:
+    m = messaging.new_message('deviceMotion')
+    m.valid = msg.valid
+    m.logMonoTime = msg.logMonoTime
+    for field in ["orientationNED", "velocityDevice", "accelerationDevice", "angularVelocityDevice"]:
+      lp_field, llk_field = getattr(m.deviceMotion, field), getattr(msg.liveLocationKalmanDEPRECATED, field)
+      lp_field.x, lp_field.y, lp_field.z = llk_field.value or nans
+      lp_field.xStd, lp_field.yStd, lp_field.zStd = llk_field.std or nans
+      lp_field.valid = llk_field.valid
+    for flag in ["inputsOK", "posenetOK", "sensorsOK"]:
+      setattr(m.deviceMotion, flag, getattr(msg.liveLocationKalmanDEPRECATED, flag))
+    ops.append((index, m.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["controlsState"], product="selfdriveState")
+def migrate_controlsState(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    m = messaging.new_message('selfdriveState')
+    m.valid = msg.valid
+    m.logMonoTime = msg.logMonoTime
+    ss = m.selfdriveState
+    for field in ("enabled", "active", "state", "engageable", "alertText1", "alertText2",
+                  "alertStatus", "alertSize", "alertType", "experimentalMode",
+                  "personality"):
+      setattr(ss, field, getattr(msg.controlsState, field+"DEPRECATED"))
+    add_ops.append(m.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["carState", "controlsState"])
+def migrate_carState(msgs):
+  ops = []
+  last_cs = None
+  for index, msg in msgs:
+    if msg.which() == 'controlsState':
+      last_cs = msg
+    elif msg.which() == 'carState' and last_cs is not None:
+      if last_cs.controlsState.vCruiseDEPRECATED - msg.carState.vCruise > 0.1:
+        msg = msg.as_builder()
+        msg.carState.vCruise = last_cs.controlsState.vCruiseDEPRECATED
+        msg.carState.vCruiseCluster = last_cs.controlsState.vCruiseClusterDEPRECATED
+        ops.append((index, msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["managerState"])
+def migrate_managerState(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = msg.as_builder()
+    for process in new_msg.managerState.processes:
+      process.running = True
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["gpsLocation", "gpsLocationExternal"])
+def migrate_gpsLocation(msgs):
+  ops = []
+  for index, msg in msgs:
+    new_msg = msg.as_builder()
+    g = getattr(new_msg, new_msg.which())
+    # hasFix is a newer field
+    if not g.hasFix and g.flags == 1:
+      g.hasFix = True
+    ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["deviceState", "initData"])
+def migrate_deviceState(msgs):
+  init_data = next((m.initData for _, m in msgs if m.which() == 'initData'), None)
+  device_state = next((m.deviceState for _, m in msgs if m.which() == 'deviceState'), None)
+  if init_data is None or device_state is None:
+    return [], [], []
+
+  ops = []
+  for i, msg in msgs:
+    if msg.which() == 'deviceState':
+      n = msg.as_builder()
+      n.deviceState.deviceType = init_data.deviceType
+      ops.append((i, n.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["carControl"], product="carOutput")
+def migrate_carOutput(msgs):
+  add_ops = []
+  for _, msg in msgs:
+    co = messaging.new_message('carOutput')
+    co.valid = msg.valid
+    co.logMonoTime = msg.logMonoTime
+    co.carOutput.actuatorsOutput = msg.carControl.actuatorsOutputDEPRECATED
+    add_ops.append(co.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["pandaStates", "pandaStateDEPRECATED", "carParams"])
+def migrate_pandaStates(msgs):
+  # TODO: safety param migration should be handled automatically
+  safety_param_migration = {
+    "TOYOTA_PRIUS": EPS_SCALE["TOYOTA_PRIUS"] | ToyotaSafetyFlags.STOCK_LONGITUDINAL,
+    "TOYOTA_RAV4": EPS_SCALE["TOYOTA_RAV4"] | ToyotaSafetyFlags.ALT_BRAKE,
+    "KIA_EV6": HyundaiSafetyFlags.EV_GAS | HyundaiSafetyFlags.CANFD_LKA_STEERING,
+    "CHEVROLET_VOLT": GMSafetyFlags.EV,
+    "CHEVROLET_BOLT_EUV": GMSafetyFlags.EV | GMSafetyFlags.HW_CAM,
+  }
+  # TODO: get new Ford route
+  safety_param_migration |= dict.fromkeys((set(FORD) - FORD.with_flags(FordFlags.CANFD)), FordSafetyFlags.LONG_CONTROL)
+
+  # Migrate safety param base on carParams
+  CP = next((m.carParams for _, m in msgs if m.which() == 'carParams'), None)
+  assert CP is not None, "carParams message not found"
+  fingerprint = MIGRATION.get(CP.carFingerprint, CP.carFingerprint)
+  if fingerprint in safety_param_migration:
+    safety_param = safety_param_migration[fingerprint].value
+  elif len(CP.safetyConfigs):
+    safety_param = CP.safetyConfigs[0].safetyParam
+    if CP.safetyConfigs[0].safetyParamDEPRECATED != 0:
+      safety_param = CP.safetyConfigs[0].safetyParamDEPRECATED
+  else:
+    safety_param = CP.safetyParamDEPRECATED
+
+  ops = []
+  for index, msg in msgs:
+    if msg.which() == 'pandaStateDEPRECATED':
+      new_msg = messaging.new_message('pandaStates', 1)
+      new_msg.valid = msg.valid
+      new_msg.logMonoTime = msg.logMonoTime
+      new_msg.pandaStates[0] = msg.pandaStateDEPRECATED
+      new_msg.pandaStates[0].safetyParam = safety_param
+      ops.append((index, new_msg.as_reader()))
+    elif msg.which() == 'pandaStates':
+      new_msg = msg.as_builder()
+      new_msg.pandaStates[-1].safetyParam = safety_param
+      # Clear DISABLE_DISENGAGE_ON_GAS bit to fix controls mismatch
+      new_msg.pandaStates[-1].alternativeExperience &= ~1
+      ops.append((index, new_msg.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["pandaStates", "pandaStateDEPRECATED"], product="peripheralState")
+def migrate_peripheralState(msgs):
+  add_ops = []
+
+  which = "pandaStates" if any(msg.which() == "pandaStates" for _, msg in msgs) else "pandaStateDEPRECATED"
+  for _, msg in msgs:
+    if msg.which() != which:
+      continue
+    new_msg = messaging.new_message("peripheralState")
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, []
+
+
+@migration(inputs=["roadEncodeIdx", "wideRoadEncodeIdx", "driverEncodeIdx", "roadCameraState", "wideRoadCameraState", "driverCameraState"])
+def migrate_cameraStates(msgs):
+  add_ops, del_ops = [], []
+  frame_to_encode_id = defaultdict(dict)
+  # just for encodeId fallback mechanism
+  min_frame_id = defaultdict(lambda: float('inf'))
+
+  for _, msg in msgs:
+    if msg.which() not in ["roadEncodeIdx", "wideRoadEncodeIdx", "driverEncodeIdx"]:
+      continue
+
+    encode_index = getattr(msg, msg.which())
+    meta = meta_from_encode_index(msg.which())
+
+    assert encode_index.segmentId < 1200, f"Encoder index segmentId greater that 1200: {msg.which()} {encode_index.segmentId}"
+    frame_to_encode_id[meta.camera_state][encode_index.frameId] = encode_index.segmentId
+
+  for index, msg in msgs:
+    if msg.which() not in ["roadCameraState", "wideRoadCameraState", "driverCameraState"]:
+      continue
+
+    camera_state = getattr(msg, msg.which())
+    min_frame_id[msg.which()] = min(min_frame_id[msg.which()], camera_state.frameId)
+
+    encode_id = frame_to_encode_id[msg.which()].get(camera_state.frameId)
+    if encode_id is None:
+      print(f"Missing encoded frame for camera feed {msg.which()} with frameId: {camera_state.frameId}")
+      if len(frame_to_encode_id[msg.which()]) != 0:
+        del_ops.append(index)
+        continue
+
+      # fallback mechanism for logs without encodeIdx (e.g. logs from before 2022 with dcamera recording disabled)
+      # try to fake encode_id by subtracting lowest frameId
+      encode_id = camera_state.frameId - min_frame_id[msg.which()]
+      print(f"Faking encodeId to {encode_id} for camera feed {msg.which()} with frameId: {camera_state.frameId}")
+
+    new_msg = messaging.new_message(msg.which())
+    new_camera_state = getattr(new_msg, new_msg.which())
+    new_camera_state.sensor = camera_state.sensor
+    new_camera_state.frameId = encode_id
+    new_camera_state.encodeId = encode_id
+    # timestampSof was added later so it might be missing on some old segments
+    if camera_state.timestampSof == 0 and camera_state.timestampEof > 25000000:
+      new_camera_state.timestampSof = camera_state.timestampEof - 18000000
+    else:
+      new_camera_state.timestampSof = camera_state.timestampSof
+    new_camera_state.timestampEof = camera_state.timestampEof
+    new_msg.logMonoTime = msg.logMonoTime
+    new_msg.valid = msg.valid
+
+    del_ops.append(index)
+    add_ops.append(new_msg.as_reader())
+  return [], add_ops, del_ops
+
+
+@migration(inputs=["carParams"])
+def migrate_carParams(msgs):
+  ops = []
+  for index, msg in msgs:
+    CP = msg.as_builder()
+    CP.carParams.carFingerprint = MIGRATION.get(CP.carParams.carFingerprint, CP.carParams.carFingerprint)
+    for car_fw in CP.carParams.carFw:
+      car_fw.brand = CP.carParams.brand
+    ops.append((index, CP.as_reader()))
+  return ops, [], []
+
+
+@migration(inputs=["sensorEventsDEPRECATED"], product="sensorEvents")
+def migrate_sensorEvents(msgs):
+  add_ops, del_ops = [], []
+  for index, msg in msgs:
+    # migrate to split sensor events
+    for evt in msg.sensorEventsDEPRECATED:
+      # build new message for each sensor type
+      sensor_service = ''
+      if evt.which() == 'acceleration':
+        sensor_service = 'accelerometer'
+      elif evt.which() == 'gyro' or evt.which() == 'gyroUncalibrated':
+        sensor_service = 'gyroscope'
+      elif evt.which() == 'light' or evt.which() == 'proximity':
+        sensor_service = 'lightSensor'
+      elif evt.which() == 'magnetic' or evt.which() == 'magneticUncalibrated':
+        sensor_service = 'magnetometer'
+      elif evt.which() == 'temperature':
+        sensor_service = 'temperatureSensor'
+
+      m = messaging.new_message(sensor_service)
+      m.valid = True
+      m.logMonoTime = msg.logMonoTime
+
+      m_dat = getattr(m, sensor_service)
+      m_dat.version = evt.version
+      m_dat.sensor = evt.sensor
+      m_dat.type = evt.type
+      m_dat.source = evt.source
+      m_dat.timestamp = evt.timestamp
+      setattr(m_dat, evt.which(), getattr(evt, evt.which()))
+
+      add_ops.append(m.as_reader())
+    del_ops.append(index)
+  return [], add_ops, del_ops
+
+
+@migration(inputs=["onroadEventsDEPRECATED"], product="onroadEvents")
+def migrate_onroadEvents(msgs):
+  ops = []
+  for index, msg in msgs:
+    onroadEvents = []
+    for event in msg.onroadEventsDEPRECATED:
+      try:
+        if not str(event.name).endswith('DEPRECATED'):
+          # dict converts name enum into string representation
+          onroadEvents.append(log.OnroadEvent(**event.to_dict()))
+      except RuntimeError:  # Member was null
+        traceback.print_exc()
+
+    new_msg = messaging.new_message('onroadEvents', len(msg.onroadEventsDEPRECATED))
+    new_msg.valid = msg.valid
+    new_msg.logMonoTime = msg.logMonoTime
+    new_msg.onroadEvents = onroadEvents
+    ops.append((index, new_msg.as_reader()))
+
+  return ops, [], []
+
+
+@migration(inputs=["driverMonitoringState"])
+def migrate_driverMonitoringState(msgs):
+  ops = []
+  for index, msg in msgs:
+    msg = msg.as_builder()
+    events = []
+    for event in msg.driverMonitoringState.eventsDEPRECATED:
+      try:
+        if not str(event.name).endswith('DEPRECATED'):
+          # dict converts name enum into string representation
+          events.append(log.OnroadEvent(**event.to_dict()))
+      except RuntimeError:  # Member was null
+        traceback.print_exc()
+
+    msg.driverMonitoringState.events = events
+    ops.append((index, msg.as_reader()))
+
+  return ops, [], []

iqpilot/selfdrive/test/process_replay/model_replay.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+import os
+import pickle
+import sys
+from collections import defaultdict
+from typing import Any
+
+import numpy as np
+from tabulate import tabulate
+
+from iqpilot.system.hardware import PC
+from iqpilot.tools.lib.logreader import get_url
+from iqpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
+from iqpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
+from iqpilot.tools.lib.framereader import FrameReader
+from iqpilot.tools.lib.logreader import LogReader, save_log
+
+TEST_ROUTE = "8494c69d3c710e81|000001d4--2648a9a404"
+SEGMENT = 4
+START_FRAME = 0
+END_FRAME = 60
+
+SEND_EXTRA_INPUTS = bool(int(os.getenv("SEND_EXTRA_INPUTS", "0")))
+
+MODEL_REPLAY_BUCKET="model_replay_master"
+
+EXEC_TIMINGS = [
+  # model, instant max, average max
+  ("modelV2", 0.035, 0.025),
+  ("driverStateV2", 0.02, 0.015),
+]
+
+def get_log_fn(test_route, ref="master"):
+  return f"{test_route}_model_tici_{ref}.zst"
+
+def get_model_replay_url(filename):
+  return f"https://raw.githubusercontent.com/commaai/ci-artifacts/refs/heads/{MODEL_REPLAY_BUCKET}/{filename}"
+
+def trim_logs(logs, start_frame, end_frame, frs_types, include_all_types):
+  all_msgs = []
+  cam_state_counts = defaultdict(int)
+  for msg in sorted(logs, key=lambda m: m.logMonoTime):
+    if msg.which() in frs_types:
+      cam_state_counts[msg.which()] += 1
+    if any(cam_state_counts[state]  >= start_frame for state in frs_types):
+      all_msgs.append(msg)
+    if all(cam_state_counts[state] == end_frame for state in frs_types):
+      break
+
+  if len(include_all_types) != 0:
+    other_msgs = [m for m in logs if m.which() in include_all_types]
+    all_msgs.extend(other_msgs)
+
+  return all_msgs
+
+
+def model_replay(lr, frs):
+  # modeld is using frame pairs
+  modeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"roadCameraState", "wideRoadCameraState"},
+                                                                         {"roadEncodeIdx", "wideRoadEncodeIdx", "carParams", "carState", "carControl", "can"})
+  dmodeld_logs = trim_logs(lr, START_FRAME, END_FRAME, {"driverCameraState"}, {"driverEncodeIdx", "carParams", "can"})
+
+  if not SEND_EXTRA_INPUTS:
+    modeld_logs = [msg for msg in modeld_logs if msg.which() != 'extrinsicsCalibration']
+    dmodeld_logs = [msg for msg in dmodeld_logs if msg.which() != 'extrinsicsCalibration']
+
+  # initial setup
+  for s in ('extrinsicsCalibration', 'deviceState'):
+    msg = next(msg for msg in lr if msg.which() == s).as_builder()
+    msg.logMonoTime = lr[0].logMonoTime
+    modeld_logs.insert(1, msg.as_reader())
+    dmodeld_logs.insert(1, msg.as_reader())
+
+  modeld = get_process_config("modeld")
+  dmonitoringmodeld = get_process_config("dmonitoringmodeld")
+
+  modeld_msgs = replay_process(modeld, modeld_logs, frs)
+  dmonitoringmodeld_msgs = replay_process(dmonitoringmodeld, dmodeld_logs, frs)
+
+  msgs = modeld_msgs + dmonitoringmodeld_msgs
+
+  header = ['model', 'max instant', 'max instant allowed', 'average', 'max average allowed', 'test result']
+  rows = []
+  timings_ok = True
+  for (s, instant_max, avg_max) in EXEC_TIMINGS:
+    ts = [getattr(m, s).modelExecutionTime for m in msgs if m.which() == s]
+    # TODO some init can happen in first iteration
+    ts = ts[1:]
+
+    errors = []
+    if np.max(ts) > instant_max:
+      errors.append("❌ FAILED MAX TIMING CHECK ❌")
+    if np.mean(ts) > avg_max:
+      errors.append("❌ FAILED AVG TIMING CHECK ❌")
+
+    timings_ok = not errors and timings_ok
+    rows.append([s, np.max(ts), instant_max, np.mean(ts), avg_max, "\n".join(errors) or "✅"])
+
+  print("------------------------------------------------")
+  print("----------------- Model Timing -----------------")
+  print("------------------------------------------------")
+  print(tabulate(rows, header, tablefmt="simple_grid", stralign="center", numalign="center", floatfmt=".4f"))
+  assert timings_ok or PC
+
+  return msgs
+
+
+def get_frames():
+  regen_cache = "--regen-cache" in sys.argv
+  frames_cache = '/tmp/model_replay_cache' if PC else '/data/model_replay_cache'
+  os.makedirs(frames_cache, exist_ok=True)
+
+  cache_name = f'{frames_cache}/{TEST_ROUTE}_{SEGMENT}_{START_FRAME}_{END_FRAME}.pkl'
+  if os.path.isfile(cache_name) and not regen_cache:
+    try:
+      print(f"Loading frames from cache {cache_name}")
+      return pickle.load(open(cache_name, "rb"))
+    except Exception as e:
+      print(f"Failed to load frames from cache {cache_name}: {e}")
+
+  frs = {
+    'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+    'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+    'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), pix_fmt='nv12', cache_size=END_FRAME - START_FRAME),
+  }
+  for fr in frs.values():
+    for fidx in range(START_FRAME, END_FRAME):
+      fr.get(fidx)
+    fr.it = None
+  print(f"Dumping frame cache {cache_name}")
+  pickle.dump(frs, open(cache_name, "wb"))
+  return frs
+
+if __name__ == "__main__":
+  update = "--update" in sys.argv or (os.getenv("GIT_BRANCH", "") == 'master')
+  replay_dir = os.path.dirname(os.path.abspath(__file__))
+
+  # load logs
+  lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.zst")))
+  frs = get_frames()
+
+  log_msgs = []
+  # run replays
+  log_msgs += model_replay(lr, frs)
+
+  # get diff
+  failed = False
+  if not update:
+    log_fn = get_log_fn(TEST_ROUTE)
+    try:
+      all_logs = list(LogReader(get_model_replay_url(log_fn)))
+      cmp_log = []
+      model_start_index = next(i for i, m in enumerate(all_logs) if m.which() in ("modelV2", "drivingModelData", "cameraOdometry"))
+      cmp_log += all_logs[model_start_index+START_FRAME*3:model_start_index + END_FRAME*3]
+      dmon_start_index = next(i for i, m in enumerate(all_logs) if m.which() == "driverStateV2")
+      cmp_log += all_logs[dmon_start_index+START_FRAME:dmon_start_index + END_FRAME]
+
+      ignore = [
+        'logMonoTime',
+        'drivingModelData.frameDropPerc',
+        'drivingModelData.modelExecutionTime',
+        'modelV2.frameDropPerc',
+        'modelV2.modelExecutionTime',
+        'driverStateV2.modelExecutionTime',
+        'driverStateV2.gpuExecutionTime'
+      ]
+      if PC:
+        # TODO We ignore whole bunch so we can compare important stuff
+        # like posenet with reasonable tolerance
+        ignore += ['modelV2.acceleration.x',
+                   'modelV2.position.x',
+                   'modelV2.position.xStd',
+                   'modelV2.position.y',
+                   'modelV2.position.yStd',
+                   'modelV2.position.z',
+                   'modelV2.position.zStd',
+                   'drivingModelData.path.xCoefficients',]
+        for i in range(3):
+          for field in ('x', 'y', 'v', 'a'):
+            ignore.append(f'modelV2.leadsV3.{i}.{field}')
+            ignore.append(f'modelV2.leadsV3.{i}.{field}Std')
+        for i in range(4):
+          for field in ('x', 'y', 'z', 't'):
+            ignore.append(f'modelV2.laneLines.{i}.{field}')
+        for i in range(2):
+          for field in ('x', 'y', 'z', 't'):
+            ignore.append(f'modelV2.roadEdges.{i}.{field}')
+      tolerance = .3 if PC else None
+      results: Any = {TEST_ROUTE: {}}
+      log_paths: Any = {TEST_ROUTE: {"models": {'ref': log_fn, 'new': log_fn}}}
+      results[TEST_ROUTE]["models"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore)
+      diff_short, diff_long, failed = format_diff(results, log_paths, 'master')
+
+      if "CI" in os.environ:
+        failed = False
+        print(diff_long)
+      print('-------------\n'*5)
+      print(diff_short)
+      with open("model_diff.txt", "w") as f:
+        f.write(diff_long)
+    except Exception as e:
+      print(str(e))
+      failed = True
+
+  if update and not PC:
+    log_fn = get_log_fn(TEST_ROUTE)
+    save_log(log_fn, log_msgs)
+
+  sys.exit(int(failed))

iqpilot/selfdrive/test/process_replay/process_replay.py

@@ -0,0 +1,779 @@
+#!/usr/bin/env python3
+import os
+import time
+import copy
+import heapq
+import signal
+from collections import Counter
+from dataclasses import dataclass, field
+from itertools import islice
+from typing import Any
+from collections.abc import Callable, Iterable
+from tqdm import tqdm
+import capnp
+from iqpilot.system.hardware.hw import Paths
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import car
+from iqpilot.cereal.services import SERVICE_LIST
+from msgq.visionipc import VisionIpcServer, get_endpoint_name as vipc_get_endpoint_name
+from iqdbc.car.can_definitions import CanData
+from iqdbc.car.car_helpers import get_car, interfaces
+from iqpilot.common.params import Params
+from iqpilot.common.prefix import OpenpilotPrefix
+from iqpilot.common.timeout import Timeout
+from iqpilot.common.realtime import DT_CTRL
+from iqpilot.selfdrive.car.card import convert_to_capnp
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.selfdrive.test.process_replay.vision_meta import meta_from_camera_state, available_streams
+from iqpilot.selfdrive.test.process_replay.migration import migrate_all
+from iqpilot.selfdrive.test.process_replay.capture import ProcessOutputCapture
+from iqpilot.tools.lib.logreader import LogIterable
+from iqpilot.tools.lib.framereader import FrameReader
+
+# Numpy gives different results based on CPU features after version 19
+NUMPY_TOLERANCE = 1e-2
+PROC_REPLAY_DIR = os.path.dirname(os.path.abspath(__file__))
+FAKEDATA = os.path.join(PROC_REPLAY_DIR, "fakedata/")
+
+
+class LauncherWithCapture:
+  def __init__(self, capture: ProcessOutputCapture, launcher: Callable):
+    self.capture = capture
+    self.launcher = launcher
+
+  def __call__(self, *args, **kwargs):
+    self.capture.link_with_current_proc()
+    self.launcher(*args, **kwargs)
+
+
+class ReplayContext:
+  def __init__(self, cfg):
+    self.proc_name = cfg.proc_name
+    self.pubs = cfg.pubs
+    self.main_pub = cfg.main_pub
+    self.main_pub_drained = cfg.main_pub_drained
+    assert len(self.pubs) != 0 or self.main_pub is not None
+
+  def __enter__(self):
+    self.open_context()
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    self.close_context()
+
+  def open_context(self):
+    messaging.toggle_fake_events(True)
+    messaging.set_fake_prefix(self.proc_name)
+
+    if self.main_pub is None:
+      self.events = {}
+      for pub in self.pubs:
+        self.events[pub] = messaging.fake_event_handle(pub, enable=True)
+    else:
+      self.events = {self.main_pub: messaging.fake_event_handle(self.main_pub, enable=True)}
+
+  def close_context(self):
+    del self.events
+
+    messaging.toggle_fake_events(False)
+    messaging.delete_fake_prefix()
+
+  @property
+  def all_recv_called_events(self):
+    return [man.recv_called_event for man in self.events.values()]
+
+  @property
+  def all_recv_ready_events(self):
+    return [man.recv_ready_event for man in self.events.values()]
+
+  def send_sync(self, pm, endpoint, dat):
+    self.events[endpoint].recv_called_event.wait()
+    self.events[endpoint].recv_called_event.clear()
+    pm.send(endpoint, dat)
+    self.events[endpoint].recv_ready_event.set()
+
+  def unlock_sockets(self):
+    expected_sets = len(self.events)
+    while expected_sets > 0:
+      index = messaging.wait_for_one_event(self.all_recv_called_events)
+      self.all_recv_called_events[index].clear()
+      self.all_recv_ready_events[index].set()
+      expected_sets -= 1
+
+  def wait_for_recv_called(self):
+    messaging.wait_for_one_event(self.all_recv_called_events)
+
+  def wait_for_next_recv(self, trigger_empty_recv):
+    index = messaging.wait_for_one_event(self.all_recv_called_events)
+    if self.main_pub is not None and self.main_pub_drained and trigger_empty_recv:
+      self.all_recv_called_events[index].clear()
+      self.all_recv_ready_events[index].set()
+      self.all_recv_called_events[index].wait()
+
+
+@dataclass
+class ProcessConfig:
+  proc_name: str
+  pubs: list[str]
+  subs: list[str]
+  ignore: list[str]
+  config_callback: Callable | None = None
+  init_callback: Callable | None = None
+  should_recv_callback: Callable | None = None
+  tolerance: float | None = None
+  processing_time: float = 0.001
+  timeout: int = 30
+  simulation: bool = True
+  # Set to service process receives on first
+  main_pub: str | None = None
+  main_pub_drained: bool = False
+  vision_pubs: list[str] = field(default_factory=list)
+  ignore_alive_pubs: list[str] = field(default_factory=list)
+
+  def __post_init__(self):
+    # If the process is polling a service, we can just lock that one to speed up replay
+    if self.main_pub is None and isinstance(self.should_recv_callback, MessageBasedRcvCallback):
+      self.main_pub = self.should_recv_callback.trigger_msg_type
+
+
+class ProcessContainer:
+  def __init__(self, cfg: ProcessConfig):
+    self.prefix = OpenpilotPrefix(create_dirs_on_enter=False, clean_dirs_on_exit=False)
+    self.cfg = copy.deepcopy(cfg)
+    self.process = copy.deepcopy(managed_processes[cfg.proc_name])
+    self.msg_queue: list[capnp._DynamicStructReader] = []
+    self.cnt = 0
+    self.pm: messaging.PubMaster | None = None
+    self.sockets: list[messaging.SubSocket] | None = None
+    self.rc: ReplayContext | None = None
+    self.vipc_server: VisionIpcServer | None = None
+    self.environ_config: dict[str, Any] | None = None
+    self.capture: ProcessOutputCapture | None = None
+
+  @property
+  def has_empty_queue(self) -> bool:
+    return len(self.msg_queue) == 0
+
+  @property
+  def pubs(self) -> list[str]:
+    return self.cfg.pubs
+
+  @property
+  def subs(self) -> list[str]:
+    return self.cfg.subs
+
+  def _clean_env(self):
+    for k in self.environ_config.keys():
+      if k in os.environ:
+        del os.environ[k]
+
+    for k in ["PROC_NAME", "SIMULATION"]:
+      if k in os.environ:
+        del os.environ[k]
+
+  def _setup_env(self, params_config: dict[str, Any], environ_config: dict[str, Any]):
+    for k, v in environ_config.items():
+      if len(v) != 0:
+        os.environ[k] = v
+      elif k in os.environ:
+        del os.environ[k]
+
+    os.environ["PROC_NAME"] = self.cfg.proc_name
+    if self.cfg.simulation:
+      os.environ["SIMULATION"] = "1"
+    elif "SIMULATION" in os.environ:
+      del os.environ["SIMULATION"]
+
+    params = Params()
+    for k, v in params_config.items():
+      if isinstance(v, bool):
+        params.put_bool(k, v)
+      else:
+        params.put(k, v)
+
+    self.environ_config = environ_config
+
+  def _setup_vision_ipc(self, all_msgs: LogIterable, frs: dict[str, Any]):
+    assert len(self.cfg.vision_pubs) != 0
+
+    vipc_server = VisionIpcServer("camerad")
+    streams_metas = available_streams(all_msgs)
+    for meta in streams_metas:
+      if meta.camera_state in self.cfg.vision_pubs:
+        assert frs[meta.camera_state].pix_fmt == 'nv12'
+        frame_size = (frs[meta.camera_state].w, frs[meta.camera_state].h)
+        vipc_server.create_buffers(meta.stream, 2, *frame_size)
+    vipc_server.start_listener()
+
+    self.vipc_server = vipc_server
+    self.cfg.vision_pubs = [meta.camera_state for meta in streams_metas if meta.camera_state in self.cfg.vision_pubs]
+
+  def _start_process(self):
+    if self.capture is not None:
+      self.process.launcher = LauncherWithCapture(self.capture, self.process.launcher)
+    self.process.prepare()
+    self.process.start()
+
+  def start(
+    self, params_config: dict[str, Any], environ_config: dict[str, Any],
+    all_msgs: LogIterable, frs: dict[str, FrameReader] | None,
+    fingerprint: str | None, capture_output: bool
+  ):
+    with self.prefix as p:
+      self.prefix.create_dirs()
+      self._setup_env(params_config, environ_config)
+
+      if self.cfg.config_callback is not None:
+        params = Params()
+        self.cfg.config_callback(params, self.cfg, all_msgs)
+
+      self.rc = ReplayContext(self.cfg)
+      self.rc.open_context()
+
+      self.pm = messaging.PubMaster(self.cfg.pubs)
+      self.sockets = [messaging.sub_sock(s, timeout=100) for s in self.cfg.subs]
+
+      if len(self.cfg.vision_pubs) != 0:
+        assert frs is not None
+        self._setup_vision_ipc(all_msgs, frs)
+        assert self.vipc_server is not None
+
+      if capture_output:
+        self.capture = ProcessOutputCapture(self.cfg.proc_name, p.prefix)
+
+      self._start_process()
+
+      if self.cfg.init_callback is not None:
+        self.cfg.init_callback(self.rc, self.pm, all_msgs, fingerprint)
+
+  def stop(self):
+    with self.prefix:
+      self.process.signal(signal.SIGKILL)
+      self.process.stop()
+      self.rc.close_context()
+      self.prefix.clean_dirs()
+      self._clean_env()
+
+  def get_output_msgs(self, start_time: int):
+    assert self.rc and self.sockets
+
+    output_msgs = []
+    self.rc.wait_for_recv_called()
+    for socket in self.sockets:
+      ms = messaging.drain_sock(socket)
+      for m in ms:
+        m = m.as_builder()
+        m.logMonoTime = start_time + int(self.cfg.processing_time * 1e9)
+        output_msgs.append(m.as_reader())
+    return output_msgs
+
+  def run_step(self, msg: capnp._DynamicStructReader, frs: dict[str, FrameReader] | None) -> list[capnp._DynamicStructReader]:
+    assert self.rc and self.pm and self.sockets and self.process.proc
+
+    output_msgs = []
+    end_of_cycle = True
+    if self.cfg.should_recv_callback is not None:
+      end_of_cycle = self.cfg.should_recv_callback(msg, self.cfg, self.cnt)
+
+    self.msg_queue.append(msg)
+    if end_of_cycle:
+      with self.prefix, Timeout(self.cfg.timeout, error_msg=f"timed out testing process {repr(self.cfg.proc_name)}"):
+        # call recv to let sub-sockets reconnect, after we know the process is ready
+        if self.cnt == 0:
+          for s in self.sockets:
+            messaging.recv_one_or_none(s)
+
+        # certain processes use drain_sock. need to cause empty recv to break from this loop
+        trigger_empty_recv = False
+        if self.cfg.main_pub and self.cfg.main_pub_drained:
+          trigger_empty_recv = any(m.which() == self.cfg.main_pub for m in self.msg_queue)
+
+        # get output msgs from previous inputs
+        output_msgs = self.get_output_msgs(msg.logMonoTime)
+
+        for m in self.msg_queue:
+          self.pm.send(m.which(), m.as_builder())
+          # send frames if needed
+          if self.vipc_server is not None and m.which() in self.cfg.vision_pubs:
+            camera_state = getattr(m, m.which())
+            camera_meta = meta_from_camera_state(m.which())
+            assert frs is not None
+            img = frs[m.which()].get(camera_state.frameId)
+            self.vipc_server.send(camera_meta.stream, img.flatten().tobytes(),
+                                  camera_state.frameId, camera_state.timestampSof, camera_state.timestampEof)
+        self.msg_queue = []
+
+        self.rc.unlock_sockets()
+        if trigger_empty_recv:
+          self.rc.unlock_sockets()
+        self.cnt += 1
+    assert self.process.proc.is_alive()
+
+    return output_msgs
+
+
+def card_fingerprint_callback(rc, pm, msgs, fingerprint):
+  print("start fingerprinting")
+  params = Params()
+  canmsgs = list(islice((m for m in msgs if m.which() == "can"), 300))
+
+  # card expects one arbitrary can and pandaState
+  rc.send_sync(pm, "can", messaging.new_message("can", 1))
+  pm.send("pandaStates", messaging.new_message("pandaStates", 1))
+  rc.send_sync(pm, "can", messaging.new_message("can", 1))
+  rc.wait_for_next_recv(True)
+
+  # fingerprinting is done, when CarParams is set
+  while params.get("CarParams") is None:
+    if len(canmsgs) == 0:
+      raise ValueError("Fingerprinting failed. Run out of can msgs")
+
+    m = canmsgs.pop(0)
+    rc.send_sync(pm, "can", m.as_builder().to_bytes())
+    rc.wait_for_next_recv(True)
+
+
+def get_car_params_callback(rc, pm, msgs, fingerprint):
+  params = Params()
+  if fingerprint:
+    CarInterface = interfaces[fingerprint]
+    CP = CarInterface.get_non_essential_params(fingerprint)
+    CP_IQ = CarInterface.get_non_essential_params_iq(CP, fingerprint)
+  else:
+    can_msgs = ([CanData(can.address, can.dat, can.src) for can in m.can] for m in msgs if m.which() == "can")
+    cached_params_raw = params.get("CarParamsCache")
+    assert next(can_msgs, None), "CAN messages are required for fingerprinting"
+    assert os.environ.get("SKIP_FW_QUERY", False) or cached_params_raw is not None, \
+            "CarParamsCache is required for fingerprinting. Make sure to keep carParams msgs in the logs."
+
+    def can_recv(wait_for_one: bool = False) -> list[list[CanData]]:
+      return [next(can_msgs, [])]
+
+    cached_params = None
+    if cached_params_raw is not None:
+      with car.CarParams.from_bytes(cached_params_raw) as _cached_params:
+        cached_params = _cached_params
+
+    _CI = get_car(can_recv, lambda _msgs: None, lambda obd: None, params.get_bool("AlphaLongitudinalEnabled"), False, cached_params=cached_params)
+    CP, CP_IQ = _CI.CP, _CI.CP_IQ
+
+  params.put("CarParams", CP.to_bytes())
+  params.put("IQCarParams", convert_to_capnp(CP_IQ).to_bytes())
+
+
+def card_rcv_callback(msg, cfg, frame):
+  # no sendcan until card is initialized
+  if msg.which() != "can":
+    return False
+
+  socks = [
+    s for s in cfg.subs if
+    frame % int(SERVICE_LIST[msg.which()].frequency / SERVICE_LIST[s].frequency) == 0
+  ]
+  if "sendcan" in socks and (frame - 1) < 2000:
+    socks.remove("sendcan")
+  return len(socks) > 0
+
+
+class ModeldCameraSyncRcvCallback:
+  def __init__(self):
+    self.road_present = False
+    self.wide_road_present = False
+    self.is_dual_camera = True
+
+  def __call__(self, msg, cfg, frame):
+    self.is_dual_camera = len(cfg.vision_pubs) == 2
+    if msg.which() == "roadCameraState":
+      self.road_present = True
+    elif msg.which() == "wideRoadCameraState":
+      self.wide_road_present = True
+
+    if self.road_present and self.wide_road_present:
+      self.road_present, self.wide_road_present = False, False
+      return True
+    elif self.road_present and not self.is_dual_camera:
+      self.road_present = False
+      return True
+    else:
+      return False
+
+
+class MessageBasedRcvCallback:
+  def __init__(self, trigger_msg_type: str, first_frame: bool = False):
+    self.trigger_msg_type = trigger_msg_type
+    self.first_frame = first_frame
+
+  def __call__(self, msg, cfg, frame):
+    # publish on first frame or trigger msg
+    return ((frame - 1) == 0 and self.first_frame) or msg.which() == self.trigger_msg_type
+
+
+def selfdrived_config_callback(params, cfg, lr):
+  ublox = params.get_bool("UbloxAvailable")
+  sub_keys = ({"gpsLocation", } if ublox else {"gpsLocationExternal", })
+
+  cfg.pubs = set(cfg.pubs) - sub_keys
+
+
+CONFIGS = [
+  ProcessConfig(
+    proc_name="selfdrived",
+    pubs=[
+      "carState", "deviceState", "pandaStates", "peripheralState", "extrinsicsCalibration", "driverMonitoringState",
+      "longitudinalPlan", "deviceMotion", "lateralDelay", "vehicleParameters", "radarState", "modelV2",
+      "driverCameraState", "roadCameraState", "wideRoadCameraState", "managerState", "lateralTorqueParameters",
+      "accelerometer", "gyroscope", "carOutput", "gpsLocationExternal", "gpsLocation", "controlsState",
+      "carControl", "driverAssistance", "alertDebug", "audioFeedback",
+    ],
+    subs=["selfdriveState", "onroadEvents"],
+    ignore=["logMonoTime", "selfdriveState.alertText1", "selfdriveState.alertText2", "selfdriveState.alertStatus"],
+    config_callback=selfdrived_config_callback,
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("carState", True),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.004,
+  ),
+  ProcessConfig(
+    proc_name="controlsd",
+    pubs=["vehicleParameters", "lateralTorqueParameters", "modelV2", "selfdriveState",
+          "extrinsicsCalibration", "deviceMotion", "longitudinalPlan", "carState", "carOutput",
+          "driverMonitoringState", "onroadEvents", "driverAssistance"],
+    subs=["carControl", "controlsState"],
+    ignore=["logMonoTime", ],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("selfdriveState"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="card",
+    pubs=["pandaStates", "carControl", "onroadEvents", "can"],
+    subs=["sendcan", "carState", "carParams", "carOutput", "radarTracks"],
+    ignore=["logMonoTime", "carState.cumLagMs"],
+    init_callback=card_fingerprint_callback,
+    should_recv_callback=card_rcv_callback,
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.004,
+    main_pub="can",
+    main_pub_drained=True,
+  ),
+  ProcessConfig(
+    proc_name="radard",
+    pubs=["radarTracks", "carState", "modelV2"],
+    subs=["radarState"],
+    ignore=["logMonoTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("modelV2"),
+  ),
+  ProcessConfig(
+    proc_name="plannerd",
+    pubs=["modelV2", "carControl", "carState", "controlsState", "vehicleParameters", "radarState", "selfdriveState"],
+    subs=["longitudinalPlan", "driverAssistance"],
+    ignore=["logMonoTime", "longitudinalPlan.processingDelay", "longitudinalPlan.solverExecutionTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("modelV2"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="calibrationd",
+    pubs=["carState", "cameraOdometry"],
+    subs=["extrinsicsCalibration"],
+    ignore=["logMonoTime"],
+    init_callback=get_car_params_callback,
+    should_recv_callback=MessageBasedRcvCallback("cameraOdometry", True),
+  ),
+  ProcessConfig(
+    proc_name="dmonitoringd",
+    pubs=["driverStateV2", "extrinsicsCalibration", "carState", "modelV2", "selfdriveState"],
+    subs=["driverMonitoringState"],
+    ignore=["logMonoTime"],
+    should_recv_callback=MessageBasedRcvCallback("driverStateV2"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="locationd",
+    pubs=[
+      "cameraOdometry", "accelerometer", "gyroscope", "extrinsicsCalibration", "carState"
+    ],
+    subs=["liveLocationKalman", "deviceMotion"],
+    ignore=["logMonoTime"],
+    should_recv_callback=MessageBasedRcvCallback("cameraOdometry"),
+    tolerance=NUMPY_TOLERANCE,
+  ),
+  ProcessConfig(
+    proc_name="ubloxd",
+    pubs=["ubloxRaw"],
+    subs=["ubloxGnss", "gpsLocationExternal"],
+    ignore=["logMonoTime"],
+  ),
+  ProcessConfig(
+    proc_name="modeld",
+    pubs=["deviceState", "roadCameraState", "wideRoadCameraState", "extrinsicsCalibration", "lateralDelay", "driverMonitoringState", "carState", "carControl"],
+    subs=["modelV2", "drivingModelData", "cameraOdometry"],
+    ignore=["logMonoTime", "modelV2.frameDropPerc", "modelV2.modelExecutionTime", "drivingModelData.frameDropPerc", "drivingModelData.modelExecutionTime"],
+    should_recv_callback=ModeldCameraSyncRcvCallback(),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.020,
+    main_pub=vipc_get_endpoint_name("camerad", meta_from_camera_state("roadCameraState").stream),
+    vision_pubs=["roadCameraState", "wideRoadCameraState"],
+    ignore_alive_pubs=["wideRoadCameraState"],
+    init_callback=get_car_params_callback,
+  ),
+  ProcessConfig(
+    proc_name="dmonitoringmodeld",
+    pubs=["extrinsicsCalibration", "driverCameraState"],
+    subs=["driverStateV2"],
+    ignore=["logMonoTime", "driverStateV2.modelExecutionTime", "driverStateV2.gpuExecutionTime"],
+    should_recv_callback=MessageBasedRcvCallback("driverCameraState"),
+    tolerance=NUMPY_TOLERANCE,
+    processing_time=0.020,
+    main_pub=vipc_get_endpoint_name("camerad", meta_from_camera_state("driverCameraState").stream),
+    vision_pubs=["driverCameraState"],
+    ignore_alive_pubs=["driverCameraState"],
+  ),
+]
+
+
+def get_process_config(name: str) -> ProcessConfig:
+  try:
+    return copy.deepcopy(next(c for c in CONFIGS if c.proc_name == name))
+  except StopIteration as ex:
+    raise Exception(f"Cannot find process config with name: {name}") from ex
+
+
+def get_custom_params_from_lr(lr: LogIterable, initial_state: str = "first") -> dict[str, Any]:
+  """
+  Use this to get custom params dict based on provided logs.
+  Useful when replaying calibrationd.
+  The params may be based on first or last message of given type (carParams, extrinsicsCalibration, vehicleParameters, lateralTorqueParameters) in the logs.
+  """
+
+  car_params = [m for m in lr if m.which() == "carParams"]
+  live_calibration = [m for m in lr if m.which() == "extrinsicsCalibration"]
+  live_parameters = [m for m in lr if m.which() == "vehicleParameters"]
+  live_torque_parameters = [m for m in lr if m.which() == "lateralTorqueParameters"]
+
+  assert initial_state in ["first", "last"]
+  msg_index = 0 if initial_state == "first" else -1
+
+  assert len(car_params) > 0, "carParams required for initial state of vehicleParameters and CarParamsPrevRoute"
+  CP = car_params[msg_index].carParams
+
+  custom_params = {
+    "CarParamsPrevRoute": CP.as_builder().to_bytes()
+  }
+
+  if len(live_calibration) > 0:
+    custom_params["CalibrationParams"] = live_calibration[msg_index].as_builder().to_bytes()
+  if len(live_parameters) > 0:
+    custom_params["LiveParametersV2"] = live_parameters[msg_index].as_builder().to_bytes()
+  if len(live_torque_parameters) > 0:
+    custom_params["LiveTorqueParameters"] = live_torque_parameters[msg_index].as_builder().to_bytes()
+
+  return custom_params
+
+
+def replay_process_with_name(name: str | Iterable[str], lr: LogIterable, *args, **kwargs) -> list[capnp._DynamicStructReader]:
+  if isinstance(name, str):
+    cfgs = [get_process_config(name)]
+  elif isinstance(name, Iterable):
+    cfgs = [get_process_config(n) for n in name]
+  else:
+    raise ValueError("name must be str or collections of strings")
+
+  return replay_process(cfgs, lr, *args, **kwargs)
+
+
+def replay_process(
+  cfg: ProcessConfig | Iterable[ProcessConfig], lr: LogIterable, frs: dict[str, FrameReader] | None = None,
+  fingerprint: str | None = None, return_all_logs: bool = False, custom_params: dict[str, Any] | None = None,
+  captured_output_store: dict[str, dict[str, str]] | None = None, disable_progress: bool = False
+) -> list[capnp._DynamicStructReader]:
+  if isinstance(cfg, Iterable):
+    cfgs = list(cfg)
+  else:
+    cfgs = [cfg]
+
+  all_msgs = migrate_all(lr,
+                         manager_states=True,
+                         panda_states=any("pandaStates" in cfg.pubs for cfg in cfgs),
+                         camera_states=any(len(cfg.vision_pubs) != 0 for cfg in cfgs))
+  process_logs = _replay_multi_process(cfgs, all_msgs, frs, fingerprint, custom_params, captured_output_store, disable_progress)
+
+  if return_all_logs:
+    keys = {m.which() for m in process_logs}
+    modified_logs = [m for m in all_msgs if m.which() not in keys]
+    modified_logs.extend(process_logs)
+    modified_logs.sort(key=lambda m: int(m.logMonoTime))
+    log_msgs = modified_logs
+  else:
+    log_msgs = process_logs
+
+  return log_msgs
+
+
+def _replay_multi_process(
+  cfgs: list[ProcessConfig], lr: LogIterable, frs: dict[str, FrameReader] | None, fingerprint: str | None,
+  custom_params: dict[str, Any] | None, captured_output_store: dict[str, dict[str, str]] | None, disable_progress: bool
+) -> list[capnp._DynamicStructReader]:
+  if fingerprint is not None:
+    params_config = generate_params_config(lr=lr, fingerprint=fingerprint, custom_params=custom_params)
+    env_config = generate_environ_config(fingerprint=fingerprint)
+  else:
+    CP = next((m.carParams for m in lr if m.which() == "carParams"), None)
+    params_config = generate_params_config(lr=lr, CP=CP, custom_params=custom_params)
+    env_config = generate_environ_config(CP=CP)
+
+  # validate frs and vision pubs
+  all_vision_pubs = [pub for cfg in cfgs for pub in cfg.vision_pubs]
+  if len(all_vision_pubs) != 0:
+    assert frs is not None, "frs must be provided when replaying process using vision streams"
+    assert all(meta_from_camera_state(st) is not None for st in all_vision_pubs), \
+                                                          f"undefined vision stream spotted, probably misconfigured process: (vision pubs: {all_vision_pubs})"
+    required_vision_pubs = {m.camera_state for m in available_streams(lr)} & set(all_vision_pubs)
+    assert all(st in frs for st in required_vision_pubs), f"frs for this process must contain following vision streams: {required_vision_pubs}"
+
+  all_msgs = sorted(lr, key=lambda msg: msg.logMonoTime)
+  log_msgs = []
+  containers = []
+  try:
+    for cfg in cfgs:
+      container = ProcessContainer(cfg)
+      containers.append(container)
+      container.start(params_config, env_config, all_msgs, frs, fingerprint, captured_output_store is not None)
+
+    all_pubs = {pub for container in containers for pub in container.pubs}
+    all_subs = {sub for container in containers for sub in container.subs}
+    lr_pubs = all_pubs - all_subs
+    pubs_to_containers = {pub: [container for container in containers if pub in container.pubs] for pub in all_pubs}
+
+    pub_msgs = [msg for msg in all_msgs if msg.which() in lr_pubs]
+    # external queue for messages taken from logs; internal queue for messages generated by processes, which will be republished
+    external_pub_queue: list[capnp._DynamicStructReader] = pub_msgs.copy()
+    internal_pub_queue: list[capnp._DynamicStructReader] = []
+    # heap for maintaining the order of messages generated by processes, where each element: (logMonoTime, index in internal_pub_queue)
+    internal_pub_index_heap: list[tuple[int, int]] = []
+
+    pbar = tqdm(total=len(external_pub_queue), disable=disable_progress)
+    while len(external_pub_queue) != 0 or (len(internal_pub_index_heap) != 0 and not all(c.has_empty_queue for c in containers)):
+      if len(internal_pub_index_heap) == 0 or (len(external_pub_queue) != 0 and external_pub_queue[0].logMonoTime < internal_pub_index_heap[0][0]):
+        msg = external_pub_queue.pop(0)
+        pbar.update(1)
+      else:
+        _, index = heapq.heappop(internal_pub_index_heap)
+        msg = internal_pub_queue[index]
+
+      target_containers = pubs_to_containers[msg.which()]
+      for container in target_containers:
+        output_msgs = container.run_step(msg, frs)
+        for m in output_msgs:
+          if m.which() in all_pubs:
+            internal_pub_queue.append(m)
+            heapq.heappush(internal_pub_index_heap, (m.logMonoTime, len(internal_pub_queue) - 1))
+        log_msgs.extend(output_msgs)
+
+    # flush last set of messages from each process
+    for container in containers:
+      last_time = log_msgs[-1].logMonoTime if len(log_msgs) > 0 else int(time.monotonic() * 1e9)
+      log_msgs.extend(container.get_output_msgs(last_time))
+  finally:
+    for container in containers:
+      container.stop()
+      if captured_output_store is not None:
+        assert container.capture is not None
+        out, err = container.capture.read_outerr()
+        captured_output_store[container.cfg.proc_name] = {"out": out, "err": err}
+
+  return log_msgs
+
+
+def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=None) -> dict[str, Any]:
+  params_dict = {
+    "OpenpilotEnabledToggle": True,
+    "DisengageOnAccelerator": True,
+    "DisableLogging": False,
+  }
+
+  if custom_params is not None:
+    params_dict.update(custom_params)
+  if lr is not None:
+    has_ublox = any(msg.which() == "ubloxGnss" for msg in lr)
+    params_dict["UbloxAvailable"] = has_ublox
+    is_rhd = next((msg.driverMonitoringState.isRHD for msg in lr if msg.which() == "driverMonitoringState"), False)
+    params_dict["IsRhdDetected"] = is_rhd
+
+  if CP is not None:
+    if fingerprint is None:
+      if CP.fingerprintSource == "fw":
+        params_dict["CarParamsCache"] = CP.as_builder().to_bytes()
+
+    if CP.openpilotLongitudinalControl:
+      params_dict["AlphaLongitudinalEnabled"] = True
+
+    if CP.notCar:
+      params_dict["JoystickDebugMode"] = True
+
+  return params_dict
+
+
+def generate_environ_config(CP=None, fingerprint=None, log_dir=None) -> dict[str, Any]:
+  environ_dict = {}
+  environ_dict["PARAMS_ROOT"] = f"{Paths.shm_path()}/params"
+  if log_dir is not None:
+    environ_dict["LOG_ROOT"] = log_dir
+
+  environ_dict["REPLAY"] = "1"
+
+  # Regen or python process
+  if CP is not None and fingerprint is None:
+    if CP.fingerprintSource == "fw":
+      environ_dict['SKIP_FW_QUERY'] = ""
+      environ_dict['FINGERPRINT'] = ""
+    else:
+      environ_dict['SKIP_FW_QUERY'] = "1"
+      environ_dict['FINGERPRINT'] = CP.carFingerprint
+  elif fingerprint is not None:
+    environ_dict['SKIP_FW_QUERY'] = "1"
+    environ_dict['FINGERPRINT'] = fingerprint
+  else:
+    environ_dict["SKIP_FW_QUERY"] = ""
+    environ_dict["FINGERPRINT"] = ""
+
+  return environ_dict
+
+
+def check_openpilot_enabled(msgs: LogIterable) -> bool:
+  cur_enabled_count = 0
+  max_enabled_count = 0
+  for msg in msgs:
+    if msg.which() == "carParams":
+      if msg.carParams.notCar:
+        return True
+    elif msg.which() == "selfdriveState":
+      if msg.selfdriveState.active:
+        cur_enabled_count += 1
+      else:
+        cur_enabled_count = 0
+      max_enabled_count = max(max_enabled_count, cur_enabled_count)
+
+  return max_enabled_count > int(10. / DT_CTRL)
+
+
+def check_most_messages_valid(msgs: LogIterable, threshold: float = 0.9) -> bool:
+  relevant_services = {sock for cfg in CONFIGS for sock in cfg.subs}
+  msgs_counts = Counter(msg.which() for msg in msgs)
+  msgs_valid_counts = Counter(msg.which() for msg in msgs if msg.valid)
+
+  most_valid_for_service = {}
+  for msg_type in msgs_counts.keys():
+    if msg_type not in relevant_services:
+      continue
+
+    valid_share = msgs_valid_counts.get(msg_type, 0) / msgs_counts[msg_type]
+    ok = valid_share >= threshold
+    if not ok:
+      print(f"WARNING: Service {msg_type} has {valid_share * 100:.2f}% valid messages, which is below threshold of {threshold * 100:.2f}%")
+    most_valid_for_service[msg_type] = ok
+
+  return all(most_valid_for_service.values())

iqpilot/selfdrive/test/process_replay/ref_commit

@@ -0,0 +1 @@
+67f3daf309dc6cbb6844fcbaeb83e6596637e551

iqpilot/selfdrive/test/process_replay/regen.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import os
+import argparse
+import time
+import capnp
+
+from typing import Any
+from collections.abc import Iterable
+
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, FAKEDATA, ProcessConfig, replay_process, get_process_config, \
+                                                                   check_openpilot_enabled, check_most_messages_valid, get_custom_params_from_lr
+from iqpilot.selfdrive.test.update_ci_routes import upload_route
+from iqpilot.tools.lib.framereader import FrameReader
+from iqpilot.tools.lib.logreader import LogReader, LogIterable, save_log
+from iqpilot.tools.lib.logreader import get_url
+
+
+def regen_segment(
+  lr: LogIterable, frs: dict[str, Any] | None = None,
+  processes: Iterable[ProcessConfig] = CONFIGS, disable_tqdm: bool = False
+) -> list[capnp._DynamicStructReader]:
+  all_msgs = sorted(lr, key=lambda m: m.logMonoTime)
+  custom_params = get_custom_params_from_lr(all_msgs)
+
+  print("Replayed processes:", [p.proc_name for p in processes])
+  print("\n\n", "*"*30, "\n\n", sep="")
+
+  output_logs = replay_process(processes, all_msgs, frs, return_all_logs=True, custom_params=custom_params, disable_progress=disable_tqdm)
+
+  return output_logs
+
+
+def setup_data_readers(
+    route: str, sidx: int, needs_driver_cam: bool = True, needs_road_cam: bool = True, dummy_driver_cam: bool = False
+) -> tuple[LogReader, dict[str, Any]]:
+  lr = LogReader(f"{route}/{sidx}/r")
+  frs = {}
+  if needs_road_cam:
+    frs['roadCameraState'] = FrameReader(get_url(route, str(sidx), "fcamera.hevc"))
+    if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+      frs['wideRoadCameraState'] = FrameReader(get_url(route, str(sidx), "ecamera.hevc"))
+  if needs_driver_cam:
+    if dummy_driver_cam:
+      frs['driverCameraState'] = FrameReader(get_url(route, str(sidx), "fcamera.hevc")) # Use fcam as dummy
+    else:
+      device_type = next(str(msg.initData.deviceType) for msg in lr if msg.which() == "initData")
+      assert device_type != "neo", "Driver camera not supported on neo segments. Use dummy dcamera."
+      frs['driverCameraState'] = FrameReader(get_url(route, str(sidx), "dcamera.hevc"))
+
+  return lr, frs
+
+
+def regen_and_save(
+  route: str, sidx: int, processes: str | Iterable[str] = "all", outdir: str = FAKEDATA,
+  upload: bool = False, disable_tqdm: bool = False, dummy_driver_cam: bool = False
+) -> str:
+  if not isinstance(processes, str) and not hasattr(processes, "__iter__"):
+    raise ValueError("whitelist_proc must be a string or iterable")
+
+  if processes != "all":
+    if isinstance(processes, str):
+      raise ValueError(f"Invalid value for processes: {processes}")
+
+    replayed_processes = []
+    for d in processes:
+      cfg = get_process_config(d)
+      replayed_processes.append(cfg)
+  else:
+    replayed_processes = CONFIGS
+
+  all_vision_pubs = {pub for cfg in replayed_processes for pub in cfg.vision_pubs}
+  lr, frs = setup_data_readers(route, sidx,
+                               needs_driver_cam="driverCameraState" in all_vision_pubs,
+                               needs_road_cam="roadCameraState" in all_vision_pubs or "wideRoadCameraState" in all_vision_pubs,
+                               dummy_driver_cam=dummy_driver_cam)
+  output_logs = regen_segment(lr, frs, replayed_processes, disable_tqdm=disable_tqdm)
+
+  log_dir = os.path.join(outdir, time.strftime("%Y-%m-%d--%H-%M-%S--0", time.gmtime()))
+  rel_log_dir = os.path.relpath(log_dir)
+  rpath = os.path.join(log_dir, "rlog.zst")
+
+  os.makedirs(log_dir)
+  save_log(rpath, output_logs, compress=True)
+
+  print("\n\n", "*"*30, "\n\n", sep="")
+  print("New route:", rel_log_dir, "\n")
+
+  if not check_openpilot_enabled(output_logs):
+    raise Exception("Route did not engage for long enough")
+  if not check_most_messages_valid(output_logs):
+    raise Exception("Route has too many invalid messages")
+
+  if upload:
+    upload_route(rel_log_dir)
+
+  return rel_log_dir
+
+
+if __name__ == "__main__":
+  def comma_separated_list(string):
+    return string.split(",")
+
+  all_procs = [p.proc_name for p in CONFIGS]
+  parser = argparse.ArgumentParser(description="Generate new segments from old ones")
+  parser.add_argument("--upload", action="store_true", help="Upload the new segment to the CI bucket")
+  parser.add_argument("--outdir", help="log output dir", default=FAKEDATA)
+  parser.add_argument("--dummy-dcamera", action='store_true', help="Use dummy blank driver camera")
+  parser.add_argument("--whitelist-procs", type=comma_separated_list, default=all_procs,
+                      help="Comma-separated whitelist of processes to regen (e.g. controlsd,radard)")
+  parser.add_argument("--blacklist-procs", type=comma_separated_list, default=[],
+                      help="Comma-separated blacklist of processes to regen (e.g. controlsd,radard)")
+  parser.add_argument("route", type=str, help="The source route")
+  parser.add_argument("seg", type=int, help="Segment in source route")
+  args = parser.parse_args()
+
+  blacklist_set = set(args.blacklist_procs)
+  processes = [p for p in args.whitelist_procs if p not in blacklist_set]
+  regen_and_save(args.route, args.seg, processes=processes, upload=args.upload, outdir=args.outdir, dummy_driver_cam=args.dummy_dcamera)

iqpilot/selfdrive/test/process_replay/regen_all.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import argparse
+import concurrent.futures
+import os
+import random
+import traceback
+from tqdm import tqdm
+
+from iqpilot.common.prefix import OpenpilotPrefix
+from iqpilot.selfdrive.test.process_replay.regen import regen_and_save
+from iqpilot.selfdrive.test.process_replay.test_processes import FAKEDATA, source_segments as segments
+from iqpilot.tools.lib.route import SegmentName
+
+
+def regen_job(segment, upload, disable_tqdm):
+  with OpenpilotPrefix():
+    sn = SegmentName(segment[1])
+    fake_dongle_id = 'regen' + ''.join(random.choice('0123456789ABCDEF') for _ in range(11))
+    try:
+      relr = regen_and_save(sn.route_name.canonical_name, sn.segment_num, upload=upload,
+                            outdir=os.path.join(FAKEDATA, fake_dongle_id), disable_tqdm=disable_tqdm, dummy_driver_cam=True)
+      relr = '|'.join(relr.split('/')[-2:])
+      return f'  ("{segment[0]}", "{relr}"), '
+    except Exception as e:
+      err = f"  {segment} failed: {str(e)}"
+      err += traceback.format_exc()
+      err += "\n\n"
+      return err
+
+
+if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+
+  parser = argparse.ArgumentParser(description="Generate new segments from old ones")
+  parser.add_argument("-j", "--jobs", type=int, default=1)
+  parser.add_argument("--no-upload", action="store_true")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
+  args = parser.parse_args()
+
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+  tested_segments = [(car, segment) for car, segment in segments if car in tested_cars]
+
+  with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
+    p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments))
+    msg = "Copy these new segments into test_processes.py:"
+    for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)):
+      msg += "\n" + str(seg)
+    print()
+    print()
+    print(msg)

iqpilot/selfdrive/test/process_replay/test_compare_logs.py

@@ -0,0 +1,16 @@
+import iqpilot.cereal.messaging as messaging
+from iqpilot.selfdrive.test.process_replay.compare_logs import remove_ignored_fields
+
+
+def test_remove_ignored_text_field():
+  msg = messaging.new_message("selfdriveState")
+  msg.selfdriveState.alertText1 = "IQ.Pilot"
+  cleared = remove_ignored_fields(msg.as_reader(), ["selfdriveState.alertText1"])
+  assert cleared.selfdriveState.alertText1 == ""
+
+
+def test_remove_ignored_enum_field():
+  msg = messaging.new_message("selfdriveState")
+  msg.selfdriveState.alertStatus = "userPrompt"
+  cleared = remove_ignored_fields(msg.as_reader(), ["selfdriveState.alertStatus"])
+  assert str(cleared.selfdriveState.alertStatus) == "normal"

iqpilot/selfdrive/test/process_replay/test_fuzzy.py

@@ -0,0 +1,34 @@
+import copy
+import os
+from hypothesis import given, HealthCheck, Phase, settings
+import hypothesis.strategies as st
+from parameterized import parameterized
+import pytest
+
+from iqpilot.cereal import log
+from iqdbc.car.toyota.values import CAR as TOYOTA
+from iqpilot.selfdrive.test.fuzzy_generation import FuzzyGenerator
+import iqpilot.selfdrive.test.process_replay.process_replay as pr
+
+pytestmark = [pytest.mark.linux, pytest.mark.slow]
+
+# These processes currently fail because of unrealistic data breaking assumptions
+# that openpilot makes causing error with NaN, inf, int size, array indexing ...
+# TODO: Make each one testable
+NOT_TESTED = ['selfdrived', 'controlsd', 'card', 'plannerd', 'calibrationd', 'dmonitoringd', 'estimatord', 'dmonitoringmodeld', 'modeld']
+
+TEST_CASES = [(cfg.proc_name, copy.deepcopy(cfg)) for cfg in pr.CONFIGS if cfg.proc_name not in NOT_TESTED]
+MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "10"))
+
+class TestFuzzProcesses:
+
+  # TODO: make this faster and increase examples
+  @parameterized.expand(TEST_CASES)
+  @given(st.data())
+  @settings(phases=[Phase.generate, Phase.target], max_examples=MAX_EXAMPLES, deadline=1000,
+            suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large])
+  def test_fuzz_process(self, proc_name, cfg, data):
+    msgs = FuzzyGenerator.get_random_event_msg(data.draw, events=cfg.pubs, real_floats=True)
+    lr = [log.Event.new_message(**m).as_reader() for m in msgs]
+    cfg.timeout = 5
+    pr.replay_process(cfg, lr, fingerprint=TOYOTA.TOYOTA_COROLLA_TSS2, disable_progress=True)

iqpilot/selfdrive/test/process_replay/test_migration.py

@@ -0,0 +1,19 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+"""
+from iqpilot.cereal import messaging
+from iqpilot.selfdrive.test.process_replay.migration import migrate_drivingModelData
+
+
+def test_driving_model_migration_ignores_incomplete_lane_metadata():
+  msg = messaging.new_message("modelV2")
+  msg.modelV2.init("laneLines", 4)
+  for lane_line in msg.modelV2.laneLines:
+    lane_line.y = [1.0]
+  msg.modelV2.laneLineProbs = [0.5]
+
+  _, added, _ = migrate_drivingModelData([(0, msg.as_reader())])
+
+  assert len(added) == 1
+  assert added[0].drivingModelData.laneLineMeta.leftProb == 0.0
+  assert added[0].drivingModelData.laneLineMeta.rightProb == 0.0

iqpilot/selfdrive/test/process_replay/test_processes.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+import argparse
+import concurrent.futures
+import os
+import sys
+from collections import defaultdict
+from tqdm import tqdm
+from typing import Any
+
+from iqdbc.car.car_helpers import interface_names
+from iqpilot.common.git import get_commit
+from iqpilot.tools.lib.logreader import get_url, upload_file
+from iqpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
+from iqpilot.selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, replay_process, \
+                                                                   check_most_messages_valid
+from iqpilot.tools.lib.filereader import FileReader
+from iqpilot.tools.lib.logreader import LogReader, save_log
+
+IS_AZURE_TOKEN_DEFINED = os.getenv("AZURE_TOKEN")
+
+source_segments = [
+  ("HYUNDAI", "02c45f73a2e5c6e9|2021-01-01--19-08-22--1"),     # HYUNDAI.HYUNDAI_SONATA
+  ("HYUNDAI2", "d545129f3ca90f28|2022-11-07--20-43-08--3"),    # HYUNDAI.HYUNDAI_KIA_EV6 (+ QCOM GPS)
+  ("TOYOTA", "0982d79ebb0de295|2021-01-04--17-13-21--13"),     # TOYOTA.TOYOTA_PRIUS
+  ("TOYOTA2", "0982d79ebb0de295|2021-01-03--20-03-36--6"),     # TOYOTA.TOYOTA_RAV4
+  ("TOYOTA3", "8011d605be1cbb77|000000cc--8e8d8ec716--6"),     # TOYOTA.TOYOTA_COROLLA_TSS2
+  ("HONDA", "eb140f119469d9ab|2021-06-12--10-46-24--27"),      # HONDA.HONDA_CIVIC (NIDEC)
+  ("HONDA2", "7d2244f34d1bbcda|2021-06-25--12-25-37--26"),     # HONDA.HONDA_ACCORD (BOSCH)
+  ("CHRYSLER", "4deb27de11bee626|2021-02-20--11-28-55--8"),    # CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID
+  ("RAM", "17fc16d840fe9d21|2023-04-26--13-28-44--5"),         # CHRYSLER.RAM_1500_5TH_GEN
+  ("SUBARU", "341dccd5359e3c97|2022-09-12--10-35-33--3"),      # SUBARU.SUBARU_OUTBACK
+  ("GM", "376bf99325883932|2022-10-27--13-41-22--1"),         # GM.CHEVROLET_BOLT_EUV
+  ("NISSAN", "35336926920f3571|2021-02-12--18-38-48--46"),     # NISSAN.NISSAN_XTRAIL
+  ("VOLKSWAGEN", "de9592456ad7d144|2021-06-29--11-00-15--6"),  # VOLKSWAGEN.VOLKSWAGEN_GOLF
+  # FIXME the sensor timings are bad in mazda segment, we're not fully testing it, but it should be replaced
+  ("MAZDA", "bd6a637565e91581|2021-10-30--15-14-53--4"),       # MAZDA.MAZDA_CX9_2021
+  ("FORD", "54827bf84c38b14f|2023-01-26--21-59-07--4"),        # FORD.FORD_BRONCO_SPORT_MK1
+  ("RIVIAN", "bc095dc92e101734|000000db--ee9fe46e57--1"),      # RIVIAN.RIVIAN_R1_GEN1
+  ("TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4"),       # TESLA.TESLA_MODEL_Y
+
+  # Enable when port is tested and dashcamOnly is no longer set
+  #("VOLKSWAGEN2", "3cfdec54aa035f3f|2022-07-19--23-45-10--2"),  # VOLKSWAGEN.VOLKSWAGEN_PASSAT_NMS
+]
+
+segments = [
+  ("HYUNDAI", "regenAA0FC4ED71E|2025-04-08--22-57-50--0"),
+  ("HYUNDAI2", "regenAFB9780D823|2025-04-08--23-00-34--0"),
+  ("TOYOTA", "regen218A4DCFAA1|2025-04-08--22-57-51--0"),
+  # TODO: get new RAV4 route without enableDsu
+  # ("TOYOTA2", "regen107352E20EB|2025-04-08--22-57-46--0"),
+  ("TOYOTA3", "regen1455E3B4BDF|2025-04-09--03-26-06--0"),
+  ("HONDA", "regenB328FF8BA0A|2025-04-08--22-57-45--0"),
+  ("HONDA2", "regen6170C8C9A35|2025-04-08--22-57-46--0"),
+  ("CHRYSLER", "regen5B28FC2A437|2025-04-08--23-04-24--0"),
+  ("RAM", "regenBF81EA96E08|2025-04-08--23-06-54--0"),
+  ("SUBARU", "regen7366F13F6A1|2025-04-08--23-07-07--0"),
+  ("GM", "regen1271097D038|2025-04-09--03-26-00--0"),
+  ("NISSAN", "regen15D60604EAB|2025-04-08--23-06-59--0"),
+  ("VOLKSWAGEN", "regen0F2F06C9539|2025-04-08--23-06-56--0"),
+  ("MAZDA", "regenACF84CCF482|2024-08-30--03-21-55--0"),
+  ("FORD", "regen755D8CB1E1F|2025-04-08--23-13-43--0"),
+  ("RIVIAN", "regen5FCAC896BBE|2025-04-08--23-13-35--0"),
+  ("TESLA", "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4"),
+]
+
+# dashcamOnly makes don't need to be tested until a full port is done
+excluded_interfaces = ["mock", "body", "psa"]
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+REF_COMMIT_FN = os.path.join(PROC_REPLAY_DIR, "ref_commit")
+EXCLUDED_PROCS = {"modeld", "dmonitoringmodeld"}
+
+
+def preserve_only_specified_files_from_ref_commit(*commits_to_keep):
+  """Keep only files in fakedata that contain any of the specified commit hashes."""
+  removed = 0
+  for f in os.listdir(FAKEDATA):
+    if not any(commit in f for commit in commits_to_keep):
+      os.remove(os.path.join(FAKEDATA, f))
+      removed += 1
+  if removed > 0:
+    print(f"Removed {removed} old files from {FAKEDATA}")
+
+
+def handle_output_file(cur_log_fn, local):
+  """Handle the output file based on whether we're using remote or local storage."""
+  assert os.path.exists(cur_log_fn), f"Cannot find log to upload: {cur_log_fn}"
+
+  if local:
+    os.system(f"git add '{os.path.realpath(cur_log_fn)}'")
+  else:
+    upload_file(cur_log_fn, os.path.basename(cur_log_fn))
+    os.remove(cur_log_fn)
+
+
+def run_test_process(data):
+  segment, cfg, args, cur_log_fn, ref_log_path, lr_dat = data
+  res = None
+  if not args.upload_only:
+    lr = LogReader.from_bytes(lr_dat)
+    res, log_msgs = test_process(cfg, lr, segment, ref_log_path, cur_log_fn, args.ignore_fields, args.ignore_msgs)
+    # save logs so we can upload when updating refs
+    save_log(cur_log_fn, log_msgs)
+
+  if args.update_refs or args.upload_only:
+    print(f'Processing: {os.path.basename(cur_log_fn)}')
+    handle_output_file(cur_log_fn, args.local)
+
+  return (segment, cfg.proc_name, res)
+
+
+def get_log_data(segment):
+  r, n = segment.rsplit("--", 1)
+  with FileReader(get_url(r, n, "rlog.zst")) as f:
+    return (segment, f.read())
+
+
+def test_process(cfg, lr, segment, ref_log_path, new_log_path, ignore_fields=None, ignore_msgs=None):
+  if ignore_fields is None:
+    ignore_fields = []
+  if ignore_msgs is None:
+    ignore_msgs = []
+
+  ref_log_msgs = list(LogReader(ref_log_path))
+
+  try:
+    log_msgs = replay_process(cfg, lr, disable_progress=True)
+  except Exception as e:
+    raise Exception("failed on segment: " + segment) from e
+
+  if not check_most_messages_valid(log_msgs):
+    return f"Route did not have enough valid messages: {new_log_path}", log_msgs
+
+  # skip this check if the segment is using qcom gps
+  if cfg.proc_name != 'ubloxd' or any(m.which() in cfg.pubs for m in lr):
+    seen_msgs = {m.which() for m in log_msgs}
+    expected_msgs = set(cfg.subs)
+    if seen_msgs != expected_msgs:
+      return f"Expected messages: {expected_msgs}, but got: {seen_msgs}", log_msgs
+
+  try:
+    return compare_logs(ref_log_msgs, log_msgs, ignore_fields + cfg.ignore, ignore_msgs, cfg.tolerance), log_msgs
+  except Exception as e:
+    return str(e), log_msgs
+
+
+def finalize_git_updates(cur_commit, ref_commit_fn):
+  """Finalize git updates and create commit."""
+  try:
+    # Add all new files first
+    os.system(f"git add {os.path.realpath(ref_commit_fn)}")
+    os.system(f"git add {os.path.realpath(FAKEDATA)}/*.zst")
+
+    # Clean up old files - keep only new ref files since they're becoming the reference
+    preserve_only_specified_files_from_ref_commit(cur_commit)
+
+    # Add the deletions to git
+    os.system(f"git add -u {os.path.realpath(FAKEDATA)}")
+
+    # Create the commit
+    commit_msg = f"test_processes: update ref logs to {cur_commit[:7]}"
+    os.system(f'git commit -m "{commit_msg}"')
+    print("Successfully committed reference log updates")
+  except Exception as e:
+    print(f"Failed to commit changes: {e}")
+
+
+if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+  all_procs = {cfg.proc_name for cfg in CONFIGS if cfg.proc_name not in EXCLUDED_PROCS}
+
+  cpu_count = os.cpu_count() or 1
+
+  parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
+  parser.add_argument("--whitelist-procs", type=str, nargs="*", default=all_procs,
+                      help="Whitelist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
+                      help="Blacklist given processes from the test (e.g. controlsd)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--ignore-fields", type=str, nargs="*", default=[],
+                      help="Extra fields or msgs to ignore (e.g. driverMonitoringState.events)")
+  parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[],
+                      help="Msgs to ignore (e.g. carEvents)")
+  parser.add_argument("--update-refs", action="store_true",
+                      help="Updates reference logs using current commit")
+  parser.add_argument("--upload-only", action="store_true",
+                      help="Skips testing processes and uploads logs from previous test run")
+  parser.add_argument("--local", action="store_true",
+                      help="Use  local git/ storage instead of remote (Azure for Comma)")
+  parser.add_argument("-j", "--jobs", type=int, default=max(cpu_count - 2, 1),
+                      help="Max amount of parallel jobs")
+  args = parser.parse_args()
+
+  tested_procs = set(args.whitelist_procs) - set(args.blacklist_procs)
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+
+  full_test = (tested_procs == all_procs) and (tested_cars == all_cars) and all(len(x) == 0 for x in (args.ignore_fields, args.ignore_msgs))
+  upload = args.update_refs or args.upload_only
+  os.makedirs(os.path.dirname(FAKEDATA), exist_ok=True)
+
+  if upload:
+    assert full_test, "Need to run full test when updating refs"
+
+  try:
+    with open(REF_COMMIT_FN) as f:
+      ref_commit = f.read().strip()
+  except FileNotFoundError:
+    print("Couldn't find reference commit")
+    sys.exit(1)
+
+  cur_commit = get_commit()
+  if not cur_commit:
+    raise Exception("Couldn't get current commit")
+
+  # Could be set as default in args, but wanted to be more explicit on the flow.
+  if upload and not args.local and not IS_AZURE_TOKEN_DEFINED:
+    print("***** Warning: local/git run was used by default since AZURE_TOKEN was NOT found on the env variables! *****")
+    args.local = True
+
+  # Clean up old files before starting
+  if upload and args.local:
+    print("***** Cleaning up old fakedata for local/git tracked refs *****")
+    preserve_only_specified_files_from_ref_commit(cur_commit, ref_commit)
+
+  print(f"***** testing against commit {ref_commit} *****")
+
+  # check to make sure all car brands are tested
+  if full_test:
+    untested = (set(interface_names) - set(excluded_interfaces)) - {c.lower() for c in tested_cars}
+    assert len(untested) == 0, f"Cars missing routes: {str(untested)}"
+
+  log_paths: defaultdict[str, dict[str, dict[str, str]]] = defaultdict(lambda: defaultdict(dict))
+  with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
+    if not args.upload_only:
+      download_segments = [seg for car, seg in segments if car in tested_cars]
+      log_data: dict[str, LogReader] = {}
+      p1 = pool.map(get_log_data, download_segments)
+      for segment, lr in tqdm(p1, desc="Getting Logs", total=len(download_segments)):
+        log_data[segment] = lr
+
+    pool_args: Any = []
+    for car_brand, segment in segments:
+      if car_brand not in tested_cars:
+        continue
+
+      for cfg in CONFIGS:
+        if cfg.proc_name not in tested_procs:
+          continue
+
+        # to speed things up, we only test all segments on card
+        if cfg.proc_name not in ('card', 'controlsd') and car_brand not in ('HYUNDAI', 'TOYOTA'):
+          continue
+
+        cur_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{cur_commit}.zst")
+        if args.update_refs:  # reference logs will not exist if routes were just regenerated
+          ref_log_path = get_url(*segment.rsplit("--", 1,), "rlog.zst")
+        else:
+          ref_log_fn = os.path.join(FAKEDATA, f"{segment}_{cfg.proc_name}_{ref_commit}.zst")
+          ref_log_path = ref_log_fn if os.path.exists(ref_log_fn) else BASE_URL + os.path.basename(ref_log_fn)
+
+        dat = None if args.upload_only else log_data[segment]
+        pool_args.append((segment, cfg, args, cur_log_fn, ref_log_path, dat))
+
+        log_paths[segment][cfg.proc_name]['ref'] = ref_log_path
+        log_paths[segment][cfg.proc_name]['new'] = cur_log_fn
+
+    results: Any = defaultdict(dict)
+    p2 = pool.map(run_test_process, pool_args)
+    for (segment, proc, result) in tqdm(p2, desc="Running Tests", total=len(pool_args)):
+      if not args.upload_only:
+        results[segment][proc] = result
+
+  diff_short, diff_long, failed = format_diff(results, log_paths, ref_commit)
+  if not upload:
+    with open(os.path.join(PROC_REPLAY_DIR, "diff.txt"), "w") as f:
+      f.write(diff_long)
+    print(diff_short)
+
+    if failed:
+      print("TEST FAILED")
+      print("\n\nTo push the new reference logs for this commit run:")
+      print("./test_processes.py --upload-only")
+    else:
+      print("TEST SUCCEEDED")
+
+  else:
+    with open(REF_COMMIT_FN, "w") as f:
+      f.write(cur_commit)
+    print(f"\n\nUpdated reference logs for commit: {cur_commit}")
+
+    # Only do git operations if we're in local mode
+    if upload and args.local:
+      finalize_git_updates(cur_commit, REF_COMMIT_FN)
+
+  sys.exit(int(failed))

iqpilot/selfdrive/test/process_replay/test_regen.py

@@ -0,0 +1,37 @@
+from parameterized import parameterized
+
+from iqpilot.selfdrive.test.process_replay.regen import regen_segment
+from iqpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled
+from iqpilot.tools.lib.logreader import get_url
+from iqpilot.tools.lib.logreader import LogReader
+from iqpilot.tools.lib.framereader import FrameReader
+
+TESTED_SEGMENTS = [
+  ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA.TOYOTA_PRIUS:     NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
+  # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
+  # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"),  # MAZDA.CX9_2021:        TICI, incomplete managerState
+  # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"),   # FORD.BRONCO_SPORT_MK1: TICI
+]
+
+
+def ci_setup_data_readers(route, sidx):
+  lr = LogReader(get_url(route, sidx, "rlog.bz2"))
+  frs = {
+    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")),
+    'driverCameraState': FrameReader(get_url(route, sidx, "fcamera.hevc")),
+  }
+  if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera.hevc"))
+
+  return lr, frs
+
+
+class TestRegen:
+  @parameterized.expand(TESTED_SEGMENTS)
+  def test_engaged(self, case_name, segment):
+    route, sidx = segment.rsplit("--", 1)
+    lr, frs = ci_setup_data_readers(route, sidx)
+    output_logs = regen_segment(lr, frs, disable_tqdm=True)
+
+    engaged = check_openpilot_enabled(output_logs)
+    assert engaged, f"openpilot not engaged in {case_name}"

iqpilot/selfdrive/test/process_replay/vision_meta.py

@@ -0,0 +1,43 @@
+from collections import namedtuple
+from iqpilot.cereal.visionipc import VisionStreamType
+from iqpilot.common.realtime import DT_MDL, DT_DMON
+from iqpilot.common.transformations.camera import DEVICE_CAMERAS
+
+VideoStreamMeta = namedtuple("VideoStreamMeta", ["camera_state", "encode_index", "stream", "dt", "frame_sizes"])
+ROAD_CAMERA_FRAME_SIZES = {k: (v.dcam.width, v.dcam.height) for k, v in DEVICE_CAMERAS.items()}
+WIDE_ROAD_CAMERA_FRAME_SIZES = {k: (v.ecam.width, v.ecam.height) for k, v in DEVICE_CAMERAS.items() if v.ecam is not None}
+DRIVER_CAMERA_FRAME_SIZES = {k: (v.dcam.width, v.dcam.height) for k, v in DEVICE_CAMERAS.items()}
+VIPC_STREAM_METADATA = [
+  # metadata: (state_msg_type, encode_msg_type, stream_type, dt, frame_sizes)
+  ("roadCameraState", "roadEncodeIdx", VisionStreamType.VISION_STREAM_ROAD, DT_MDL, ROAD_CAMERA_FRAME_SIZES),
+  ("wideRoadCameraState", "wideRoadEncodeIdx", VisionStreamType.VISION_STREAM_WIDE_ROAD, DT_MDL, WIDE_ROAD_CAMERA_FRAME_SIZES),
+  ("driverCameraState", "driverEncodeIdx", VisionStreamType.VISION_STREAM_DRIVER, DT_DMON, DRIVER_CAMERA_FRAME_SIZES),
+]
+
+
+def meta_from_camera_state(state):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[0] == state), None)
+  return meta
+
+
+def meta_from_encode_index(encode_index):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[1] == encode_index), None)
+  return meta
+
+
+def meta_from_stream_type(stream_type):
+  meta = next((VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA if meta[2] == stream_type), None)
+  return meta
+
+
+def available_streams(lr=None):
+  if lr is None:
+    return [VideoStreamMeta(*meta) for meta in VIPC_STREAM_METADATA]
+
+  result = []
+  for meta in VIPC_STREAM_METADATA:
+    has_cam_state = next((True for m in lr if m.which() == meta[0]), False)
+    if has_cam_state:
+      result.append(VideoStreamMeta(*meta))
+
+  return result