IQ.Pilot Release Commit @ d2ce8a8

iqpilot/selfdrive/iqmodeld/big_catalog.py

@@ -0,0 +1,9 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
+
+try:
+  load_private_module(__name__, "iqpilot_private.models.big_catalog")
+except ProprietaryModuleMissing:
+  from iqpilot.models_private_src.big_catalog import *

iqpilot/selfdrive/iqmodeld/daemon.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 from __future__ import annotations
 
+import sys
 import time
 from dataclasses import dataclass
 from typing import Any
@@ -471,7 +472,7 @@ class FrameDropMeter:
 
 
 class InferenceDaemon:
-  def __init__(self, demo: bool = False):
+  def __init__(self, demo: bool = False, channel_path: str | None = None):
     cloudlog.warning("iqmodeld init")
     sentry.set_tag("daemon", PROCESS_NAME)
     cloudlog.bind(daemon=PROCESS_NAME)
@@ -485,8 +486,15 @@ class InferenceDaemon:
     self._meta_layout = select_meta_layout()
     cloudlog.warning("models loaded, iqmodeld starting")
 
+    self._channel = None
+    if channel_path is not None:
+      from iqpilot.selfdrive.iqmodeld.model_channel import ModelChannel
+      self._channel = ModelChannel(channel_path, create=True)
+
     self._cameras = CameraIngress(self._gpu)
-    self._pub = PubMaster(["modelV2", "drivingModelData", "cameraOdometry", "iqDriveModelData", "iqPerfTrace"])
+    pub_services = ["iqPerfTrace"] if self._channel is not None else [
+      "modelV2", "drivingModelData", "cameraOdometry", "iqDriveModelData", "iqPerfTrace"]
+    self._pub = PubMaster(pub_services)
     self._sub = SubMaster([
       "deviceState", "carState", "roadCameraState", "extrinsicsCalibration",
       "driverMonitoringState", "carControl", "lateralDelay", "iqNavState", "radarState",
@@ -513,6 +521,12 @@ class InferenceDaemon:
   def _refresh_tunables(self, tick: int) -> None:
     if tick % 60 != 0:
       return
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    big_enabled = self._params.get_bool("IQEmacEnabled") or egpu_selected(self._params)
+    if big_enabled != (self._channel is not None):
+      # publish mode is fixed at startup: staying up would fight the selector for modelV2
+      cloudlog.warning("iqmodeld: big backend toggled, restarting to switch publish mode")
+      sys.exit(0)
     self._runtime.lat_delay = lateral_action_delay(self._params, self._car_params, self._sub["lateralDelay"].lateralDelay)
     self._runtime.PLANPLUS_CONTROL = self._params.get("PlanplusControl", return_default=True)
     self._runtime.model_smoothing_max_extra_sec = _model_lat_smooth_max_sec(self._params)
@@ -600,6 +614,22 @@ class InferenceDaemon:
       live_calib_seen,
     )
 
+    if self._channel is not None:
+      self._channel.write(main_stamp.frame_id, {
+        "source": "small",
+        "frame_id": main_stamp.frame_id,
+        "timestamp_sof": int(main_stamp.timestamp_sof),
+        "live_calib_seen": bool(live_calib_seen),
+        "model_execution_time": float(execution_time),
+        "msgs": {
+          "modelV2": model_msg.to_bytes(),
+          "drivingModelData": driving_msg.to_bytes(),
+          "cameraOdometry": pose_msg.to_bytes(),
+          "iqDriveModelData": iq_msg.to_bytes(),
+        },
+      })
+      return
+
     self._pub.send("modelV2", model_msg)
     self._pub.send("drivingModelData", driving_msg)
     self._pub.send("cameraOdometry", pose_msg)
@@ -695,8 +725,15 @@ class InferenceDaemon:
       tick += 1
 
 
-def main(demo: bool = False):
-  InferenceDaemon(demo=demo).serve()
+def main(demo: bool = False, channel_path: str | None = "auto"):
+  if channel_path == "auto":
+    channel_path = None
+    params = Params()
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    if params.get_bool("IQEmacEnabled") or egpu_selected(params):
+      from iqpilot.selfdrive.iqmodeld.model_channel import SMALL_CHANNEL
+      channel_path = SMALL_CHANNEL
+  InferenceDaemon(demo=demo, channel_path=channel_path).serve()
 
 
 __all__ = [

iqpilot/selfdrive/iqmodeld/driving_action.py

@@ -0,0 +1,46 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import numpy as np
+
+from iqpilot.cereal import log
+
+from iqpilot.selfdrive.controls.lib.drive_helpers import smooth_value
+
+LAT_SMOOTH_SECONDS = 0.0
+LONG_SMOOTH_SECONDS = 0.3
+MIN_LAT_CONTROL_SPEED = 0.3
+DESIRE_LEN = 8
+
+
+def get_action_from_model(outputs: dict[str, np.ndarray], prev_action: log.ModelDataV2.Action,
+                          v_ego: float, lat_action_t: float, long_action_t: float,
+                          lat_smooth_seconds: float | None = None) -> log.ModelDataV2.Action:
+  if "action" in outputs:
+    desired_accel = float(outputs["action"][0, 1])
+    desired_curvature = float(outputs["action"][0, 0]) / (max(1.0, v_ego)) ** 2
+    should_stop = bool(v_ego < 0.3 and desired_accel < 0.1)
+  else:
+    from iqpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, get_curvature_from_plan
+    from iqpilot.selfdrive.iqmodeld.config import ModelConstants, Plan
+    plan = outputs["plan"][0]
+    desired_accel, should_stop = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0],
+                                                     plan[:, Plan.ACCELERATION][:, 0],
+                                                     ModelConstants.T_IDXS,
+                                                     action_t=long_action_t)
+    desired_curvature = get_curvature_from_plan(plan[:, Plan.T_FROM_CURRENT_EULER][:, 2],
+                                                plan[:, Plan.ORIENTATION_RATE][:, 2],
+                                                ModelConstants.T_IDXS, v_ego, lat_action_t)
+    desired_accel, should_stop = float(desired_accel), bool(should_stop)
+    desired_curvature = float(desired_curvature)
+  desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, LONG_SMOOTH_SECONDS)
+  if v_ego > MIN_LAT_CONTROL_SPEED:
+    lat_smooth = LAT_SMOOTH_SECONDS if lat_smooth_seconds is None else lat_smooth_seconds
+    desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, lat_smooth)
+  else:
+    desired_curvature = prev_action.desiredCurvature
+  return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature),
+                                desiredAcceleration=float(desired_accel),
+                                shouldStop=should_stop)

iqpilot/selfdrive/iqmodeld/egpu_helpers.py

@@ -0,0 +1,158 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import urllib.request
+from pathlib import Path
+
+from iqpilot.system.hardware.usb import egpu_dock_ready
+
+USB_SYSFS_ROOT = "/sys/bus/usb/devices"
+
+COMMA_LFS_BATCH_URL = "https://gitlab.com/commaai/openpilot-lfs.git/info/lfs/objects/batch"
+
+DOWNLOAD_CHUNK = 4 * 1024 * 1024
+
+
+def usbgpu_present(sysfs_root: str = USB_SYSFS_ROOT) -> bool:
+  return egpu_dock_ready(Path(sysfs_root))
+
+
+def egpu_present_consented(params, sysfs_root: str = USB_SYSFS_ROOT) -> bool:
+  try:
+    if params is not None and params.get_bool("IQEgpuDisabled"):
+      return False
+  except Exception:
+    pass
+  return usbgpu_present(sysfs_root)
+
+
+def egpu_selected(params, sysfs_root: str = USB_SYSFS_ROOT) -> bool:
+  try:
+    if params is not None and params.get_bool("IQEgpuDisabled"):
+      return False
+    if params is not None and params.get_bool("IQEgpuEnabled"):
+      return True
+  except Exception:
+    pass
+  return usbgpu_present(sysfs_root)
+
+
+def resolve_backend(emac_enabled: bool, egpu_enabled: bool, egpu_present: bool = False) -> str | None:
+  if egpu_present:
+    return "egpu"
+  if emac_enabled:
+    return "emac"
+  if egpu_enabled:
+    return "egpu"
+  return None
+
+
+def egpu_pkl_path(meta: dict) -> str:
+  from iqpilot.system.hardware.hw import Paths
+  return os.path.join(Paths.model_root(), f"egpu_{meta['key']}_{meta['sha256'][:8]}_amd_tinygrad.pkl")
+
+
+def onnx_cache_path(meta: dict) -> str:
+  from iqpilot.system.hardware.hw import Paths
+  return os.path.join(Paths.model_root(), f"{meta['model_name']}_{meta['sha256'][:8]}.onnx")
+
+
+def _sha256_file(path: str) -> str:
+  digest = hashlib.sha256()
+  with open(path, "rb") as f:
+    while chunk := f.read(DOWNLOAD_CHUNK):
+      digest.update(chunk)
+  return digest.hexdigest()
+
+
+def quarantine_artifact(path: str, why: str) -> None:
+  try:
+    if os.path.isfile(path):
+      os.replace(path, path + ".unusable")
+  except OSError:
+    try:
+      os.remove(path)
+    except OSError:
+      pass
+
+
+def local_onnx(meta: dict) -> str | None:
+  path = onnx_cache_path(meta)
+  if not os.path.isfile(path):
+    return None
+  size = int(meta.get("download", {}).get("size", 0))
+  if size and os.path.getsize(path) != size:
+    quarantine_artifact(path, "onnx size mismatch")
+    return None
+  if _sha256_file(path) != meta["sha256"]:
+    quarantine_artifact(path, "onnx sha256 mismatch")
+    return None
+  return path
+
+
+def resolve_download_url(download_url: str, sha256: str, size: int, timeout: float = 30.0) -> str:
+  if download_url.startswith("commalfs:"):
+    oid = download_url.split(":", 1)[1]
+    body = json.dumps({"operation": "download", "transfers": ["basic"],
+                       "objects": [{"oid": oid, "size": size}]}).encode()
+    req = urllib.request.Request(COMMA_LFS_BATCH_URL, data=body, headers={
+      "Accept": "application/vnd.git-lfs+json", "Content-Type": "application/vnd.git-lfs+json"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+      d = json.load(r)
+    return d["objects"][0]["actions"]["download"]["href"]
+  return download_url
+
+
+def download_onnx(meta: dict, progress_cb=None) -> str:
+  from iqpilot.selfdrive.iqmodeld.egpu_model import download_descriptor
+  download_url, size = download_descriptor(meta)
+  if not download_url:
+    raise RuntimeError(f"model {meta['key']} has no download source; stage the onnx at {onnx_cache_path(meta)}")
+
+  url = resolve_download_url(download_url, meta["sha256"], size)
+  path = onnx_cache_path(meta)
+  os.makedirs(os.path.dirname(path), exist_ok=True)
+  tmp = path + ".part"
+  digest = hashlib.sha256()
+  got = 0
+  with urllib.request.urlopen(url, timeout=60) as r, open(tmp, "wb") as f:
+    while chunk := r.read(DOWNLOAD_CHUNK):
+      f.write(chunk)
+      digest.update(chunk)
+      got += len(chunk)
+      if progress_cb is not None and size:
+        progress_cb(got / size)
+  if size and got != size:
+    os.remove(tmp)
+    raise RuntimeError(f"onnx download truncated: {got}/{size} bytes")
+  if digest.hexdigest() != meta["sha256"]:
+    os.remove(tmp)
+    raise RuntimeError(f"onnx sha256 mismatch for {meta['key']}")
+  os.replace(tmp, path)
+  return path
+
+
+def patch_tinygrad_fetch_fw() -> None:
+  import pathlib
+
+  import zstandard
+  from tinygrad import helpers
+  if getattr(helpers.fetch_fw, "_iq_patched", False):
+    return
+  _orig = helpers.fetch_fw
+
+  def fetch_fw(path, name, sha256):
+    p = pathlib.Path(f"/lib/firmware/{path}/{name}.zst")
+    if p.is_file():
+      blob = zstandard.ZstdDecompressor().stream_reader(p.read_bytes()).read()
+      if hashlib.sha256(blob).hexdigest() == sha256:
+        return blob
+    return _orig(path, name, sha256)
+
+  fetch_fw._iq_patched = True
+  helpers.fetch_fw = fetch_fw

iqpilot/selfdrive/iqmodeld/egpu_model.py

@@ -0,0 +1,9 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
+
+try:
+  load_private_module(__name__, "iqpilot_private.models.egpu_model")
+except ProprietaryModuleMissing:
+  from iqpilot.models_private_src.egpu_model import *

iqpilot/selfdrive/iqmodeld/egpu_pipeline.py

@@ -0,0 +1,47 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import numpy as np
+
+from iqpilot.selfdrive.iqmodeld.temporal_state import MODEL_INPUT_SPEC, TemporalInputState, spec_from_meta
+
+
+class EgpuPipelineError(RuntimeError):
+  pass
+
+
+class EgpuPipeline:
+
+  def __init__(self, meta: dict, infer_fn):
+    if meta.get("split"):
+      raise EgpuPipelineError(f"model {meta['key']} is a split model; eGPU v1 runs fused models only")
+    self.meta = meta
+    self.infer_fn = infer_fn
+    self.state = TemporalInputState(meta["frame_skip"], spec_from_meta(meta) or MODEL_INPUT_SPEC)
+    self.hidden_slice = meta["output_slices"]["hidden_state"]
+    self.output_len = int(meta["output_len"])
+
+  def run(self, warped: np.ndarray, desire_vec: np.ndarray, traffic_convention: np.ndarray,
+          action_t: np.ndarray) -> np.ndarray:
+    inputs = self.state.push_and_materialize(warped, desire_vec, traffic_convention, action_t)
+    out = np.asarray(self.infer_fn(inputs), dtype=np.float32).reshape(-1)
+    if out.shape[0] != self.output_len:
+      raise EgpuPipelineError(f"eGPU output length {out.shape[0]} != {self.output_len}")
+    if not np.isfinite(out).all():
+      raise EgpuPipelineError("eGPU output contains non-finite values")
+    self.state.note_hidden_state(out, self.hidden_slice)
+    return out
+
+
+def make_big_channel_payload(frame_id: int, live_calib_seen: bool, execution_time: float,
+                             egpu_exec_ms: float, msgs: dict[str, bytes]) -> dict:
+  return {
+    "source": "egpu_big",
+    "frame_id": int(frame_id),
+    "live_calib_seen": bool(live_calib_seen),
+    "model_execution_time": float(execution_time),
+    "egpu_exec_ms": float(egpu_exec_ms),
+    "msgs": msgs,
+  }

iqpilot/selfdrive/iqmodeld/egpu_telemetry.py

@@ -0,0 +1,106 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import struct
+
+from iqpilot.cereal import messaging
+from iqpilot.common.swaglog import cloudlog
+
+METRICS_REFRESH_EVERY = 100
+
+
+class EgpuDockTelemetry:
+
+  def __init__(self, pm, big: bool):
+    self.pm = pm
+    self.big = big
+    self.valid = True
+    self.sends = 0
+    self.metrics: dict[str, float] = {}
+    self._power_limit: int | None = None
+    self._asm_usb = None
+
+  def _device(self):
+    from tinygrad.device import Device
+    return Device
+
+  def _open_asm_usb(self):
+    import usb1
+    from iqpilot.system.hardware.usb import EGPU_DOCK_USB_IDS
+    context = usb1.USBContext()
+    for vendor_id, product_id in EGPU_DOCK_USB_IDS:
+      handle = context.openByVendorIDAndProductID(vendor_id, product_id, skip_on_error=True)
+      if handle is not None:
+        return handle
+    context.close()
+    return None
+
+  def _read_ina(self):
+    Device = self._device()
+    if "AMD" in Device._opened_devices and self._asm_usb is None:
+      try:
+        raw = Device["AMD"].iface.pci_dev.usb.usb.control_read(0xC0, 5)
+        return struct.unpack("<Hh?", bytes(raw))
+      except Exception:
+        pass
+    if self._asm_usb is None:
+      self._asm_usb = self._open_asm_usb()
+    if self._asm_usb is None:
+      raise RuntimeError("no egpu ASM usb handle")
+    try:
+      raw = self._asm_usb.controlRead(0xC0, 0xC0, 0, 0, 5, timeout=100)
+    except Exception:
+      self._asm_usb = None
+      raise
+    return struct.unpack("<Hh?", bytes(raw))
+
+  def power_limit(self, smu) -> int:
+    if self._power_limit is None:
+      self._power_limit = smu._send_msg(smu.smu_mod.PPSMC_MSG_GetPptLimit, 0, read_back_arg=True, timeout=100)
+    return self._power_limit
+
+  def send(self) -> None:
+    Device = self._device()
+    msg = messaging.new_message("egpuDockState")
+    state = msg.egpuDockState
+    self.sends += 1
+
+    if self.big and "AMD" in Device._opened_devices and self.sends % METRICS_REFRESH_EVERY == 1:
+      try:
+        smu = Device["AMD"].iface.dev_impl.smu
+        smu._send_msg(smu.smu_mod.PPSMC_MSG_TransferTableSmu2Dram, smu.smu_mod.TABLE_SMU_METRICS, timeout=100)
+        metrics = smu.read_table(smu.smu_mod.SmuMetricsExternal_t, smu.smu_mod.TABLE_SMU_METRICS).SmuMetrics
+        self.metrics = {"tempC": metrics.AvgTemperature[smu.smu_mod.TEMP_HOTSPOT],
+                        "memoryTempC": metrics.AvgTemperature[smu.smu_mod.TEMP_MEM],
+                        "powerDrawW": metrics.AverageSocketPower,
+                        "powerLimitW": self.power_limit(smu),
+                        "gpuUsagePercent": metrics.AverageGfxActivity,
+                        "gpuClockMhz": metrics.AverageGfxclkFrequencyPostDs,
+                        "fanSpeedRpm": metrics.AvgFanRpm}
+        self.valid = True
+      except Exception:
+        if self.valid:
+          cloudlog.exception("egpu dock state read failed")
+        self.valid = False
+        self.metrics.clear()
+
+    if self.big:
+      for k, v in self.metrics.items():
+        setattr(state, k, v)
+
+    asm_valid = False
+    try:
+      state.supplyVoltage, state.supplyCurrent, state.supplyFault = self._read_ina()
+      asm_valid = True
+    except Exception:
+      pass
+    if "AMD" in Device._opened_devices:
+      try:
+        state.pcieLtssm = Device["AMD"].iface.pci_dev.usb.read(0xB450, 1)[0]
+      except Exception:
+        pass
+
+    msg.valid = asm_valid and (not self.big or self.valid)
+    self.pm.send("egpuDockState", msg)

iqpilot/selfdrive/iqmodeld/emac_input_state.py

@@ -0,0 +1,10 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+
+from iqpilot.selfdrive.iqmodeld.temporal_state import (
+  SplitTemporalState as SplitInputState,
+  TemporalInputState as EmacInputState,
+)
+
+__all__ = ["EmacInputState", "SplitInputState"]

iqpilot/selfdrive/iqmodeld/emac_model_meta.py

@@ -0,0 +1,9 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from iqpilot._proprietary_loader import ProprietaryModuleMissing, load_private_module
+
+try:
+  load_private_module(__name__, "iqpilot_private.models.emac_model_meta")
+except ProprietaryModuleMissing:
+  from iqpilot.models_private_src.emac_model_meta import *

iqpilot/selfdrive/iqmodeld/iqegpumodeld.py

@@ -0,0 +1,377 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import os
+import pickle
+import subprocess
+import sys
+import time
+
+from iqpilot.system.hardware import TICI
+
+os.environ.setdefault("GMMU", "0")
+if TICI:
+  os.environ.setdefault("DEV", "QCOM")
+else:
+  os.environ.setdefault("DEV", "CPU")
+
+import numpy as np
+from setproctitle import setproctitle
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import car, log
+from iqpilot.cereal.messaging import SubMaster
+from iqpilot.cereal.services import SERVICE_LIST
+from iqdbc.car.car_helpers import get_demo_car_params
+
+from iqpilot.common.params import Params
+from iqpilot.common.realtime import DT_MDL
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.selfdrive.controls.lib.desire_helper import DesireHelper
+from iqpilot.system import sentry
+
+from iqpilot.common.steer_delay import lateral_action_delay
+from iqpilot.selfdrive.iqmodeld.daemon import CalibrationAtlas, CameraIngress, FrameDropMeter
+from iqpilot.selfdrive.iqmodeld.driving_action import (
+  DESIRE_LEN, LAT_SMOOTH_SECONDS, LONG_SMOOTH_SECONDS, get_action_from_model,
+)
+from iqpilot.selfdrive.iqmodeld.egpu_helpers import (
+  download_onnx, egpu_pkl_path, egpu_present_consented, egpu_selected, local_onnx, patch_tinygrad_fetch_fw,
+  quarantine_artifact, resolve_backend, usbgpu_present,
+)
+from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+from iqpilot.selfdrive.iqmodeld.egpu_pipeline import EgpuPipeline, EgpuPipelineError, make_big_channel_payload
+from iqpilot.selfdrive.iqmodeld.egpu_telemetry import EgpuDockTelemetry
+from iqpilot.selfdrive.iqmodeld.messaging import DrivePacketMemory, populate_drive_messages, populate_odometry_message
+from iqpilot.selfdrive.iqmodeld.metadata import Meta20hz
+from iqpilot.selfdrive.iqmodeld.model_channel import BIG_CHANNEL, ModelChannel
+from iqpilot.selfdrive.iqmodeld.model_warp import FrameWarp
+from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
+
+PROCESS_NAME = "iqpilot.selfdrive.iqmodeld.iqegpumodeld"
+
+PRESENCE_POLL_S = 5.0
+COMPILE_TIMEOUT_S = 3600
+LINK_UP_TIMEOUT_S = 10.0
+SETUP_RETRY_BASE_S = 3.0
+SETUP_RETRY_MAX_S = 30.0
+
+
+def park(reason: str) -> None:
+  cloudlog.warning(f"iqegpumodeld parked: {reason}")
+  params = Params()
+  params.put_bool("UsbGpuFailed", True)
+  params.put("UsbGpuLastError", reason[:512])
+  while True:
+    time.sleep(1)
+
+
+def _wait_for_egpu(params: Params) -> None:
+  while not usbgpu_present():
+    params.put_bool("UsbGpuPresent", False)
+    time.sleep(PRESENCE_POLL_S)
+  params.put_bool("UsbGpuPresent", True)
+  try:
+    from iqpilot.system.hardware.egpu_dock.flash import link_up
+  except Exception:
+    return
+  deadline = time.monotonic() + LINK_UP_TIMEOUT_S
+  while time.monotonic() < deadline:
+    try:
+      if link_up():
+        return
+    except Exception:
+      return
+    time.sleep(0.5)
+
+
+def _compile_in_subprocess(meta: dict, onnx_path: str, pkl_path: str) -> None:
+  cmd = [sys.executable, "-m", "iqpilot.selfdrive.iqmodeld.tools.compile_egpu_model",
+         "--model", meta["key"], "--onnx", onnx_path, "--output", pkl_path]
+  compile_env = {**os.environ, "DEV": "USB+AMD:LLVM", "FLOAT16": "1",
+                 "JIT_BATCH_SIZE": "0", "GMMU": "0"}
+  proc = subprocess.run(cmd, timeout=COMPILE_TIMEOUT_S, capture_output=True, text=True,
+                        env=compile_env, preexec_fn=lambda: os.nice(20))
+  if proc.returncode != 0:
+    tail = (proc.stderr or proc.stdout or "").strip()[-800:]
+    raise RuntimeError(f"eGPU model compile failed (rc={proc.returncode}): {tail}")
+
+
+def _ensure_artifact(params: Params, meta: dict) -> str:
+  pkl_path = egpu_pkl_path(meta)
+  if os.path.isfile(pkl_path):
+    return pkl_path
+
+  params.put_bool("UsbGpuCompiled", False)
+  onnx_path = local_onnx(meta)
+  if onnx_path is None:
+    params.put("UsbGpuSetupProgress", "0.0")
+    cloudlog.warning(f"iqegpumodeld downloading {meta['key']} onnx ({meta.get('download', {}).get('size', 0) / 1e6:.0f}MB)")
+    last = [-1.0]
+
+    def _prog(p: float) -> None:
+      if p - last[0] >= 0.02 or p >= 1.0:
+        last[0] = p
+        params.put("UsbGpuSetupProgress", f"{p:.3f}")
+
+    onnx_path = download_onnx(meta, progress_cb=_prog)
+
+  cloudlog.warning(f"iqegpumodeld compiling {meta['key']} for USB-AMD (one-time, can take minutes)")
+  _compile_in_subprocess(meta, onnx_path, pkl_path)
+  cloudlog.warning(f"iqegpumodeld compiled -> {pkl_path}")
+  return pkl_path
+
+
+def _load_infer_fn(pkl_path: str, meta: dict):
+  patch_tinygrad_fetch_fw()
+  from tinygrad.tensor import Tensor
+
+  with open(pkl_path, "rb") as f:
+    bundle = pickle.load(f)
+  if bundle.get("model_sha256") != meta["sha256"]:
+    quarantine_artifact(pkl_path, "pkl model sha mismatch")
+    raise RuntimeError(f"artifact model sha {bundle.get('model_sha256')} != {meta['sha256']}")
+  if int(bundle.get("output_len", -1)) != int(meta["output_len"]):
+    quarantine_artifact(pkl_path, "pkl output_len mismatch")
+    raise RuntimeError(f"artifact output_len {bundle.get('output_len')} != {meta['output_len']}")
+  jit = bundle["run_model"]
+  input_dev = bundle.get("input_device", "AMD")
+  input_spec = bundle["input_spec"]
+
+  def infer(inputs: dict[str, np.ndarray]) -> np.ndarray:
+    tensors = {name: Tensor(np.ascontiguousarray(inputs[name]), device=input_dev).realize()
+               for name in input_spec}
+    out, = jit(**tensors)
+    return out.numpy().reshape(-1)
+
+  return infer, input_spec
+
+
+def _warmup(infer_fn, input_spec: dict, output_len: int) -> float:
+  zeros = {name: np.zeros(shape, dtype=dtype) for name, (shape, dtype) in input_spec.items()}
+  t0 = time.perf_counter()
+  out = infer_fn(zeros)
+  dt = time.perf_counter() - t0
+  if out.shape[0] != output_len or not np.isfinite(out).all():
+    raise RuntimeError(f"warmup produced invalid output (len={out.shape[0]})")
+  return dt
+
+
+def main(demo: bool = False) -> None:
+  cloudlog.warning("iqegpumodeld init")
+  sentry.set_tag("daemon", PROCESS_NAME)
+  cloudlog.bind(daemon=PROCESS_NAME)
+  setproctitle(PROCESS_NAME)
+  try:
+    os.sched_setaffinity(0, {4, 5, 6})
+    os.nice(-10)
+  except OSError as e:
+    cloudlog.warning(f"iqegpumodeld affinity/nice failed ({e}); continuing at defaults")
+
+  params = Params()
+  backend = resolve_backend(params.get_bool("IQEmacEnabled"), egpu_selected(params), egpu_present_consented(params))
+  if backend != "egpu":
+    park(f"backend resolution is {backend!r}, not egpu; refusing to own the big channel")
+
+  channel = ModelChannel(BIG_CHANNEL, create=True)
+
+  cloudlog.warning("iqegpumodeld waiting for camerad")
+  cameras = CameraIngress(None)
+  layout = cameras.layout
+
+  _wait_for_egpu(params)
+  params.put_bool("UsbGpuLoading", True)
+  attempt = 0
+  while True:
+    try:
+      meta = resolve_egpu_model(params)
+      if meta is None:
+        raise RuntimeError("selected big model is not in the catalog; check connectivity or pick another model")
+      if meta.get("split"):
+        params.put_bool("UsbGpuLoading", False)
+        park(f"model {meta['key']} needs the Mac backend; the eGPU runs fused models only")
+      warp = FrameWarp(cameras._primary.width, cameras._primary.height, meta["frame_skip"])
+      pkl_path = _ensure_artifact(params, meta)
+      infer_fn, input_spec = _load_infer_fn(pkl_path, meta)
+      warm_s = _warmup(infer_fn, input_spec, meta["output_len"])
+      break
+    except Exception as e:
+      attempt += 1
+      params.put("UsbGpuLastError", str(e)[:512])
+      cloudlog.warning(f"iqegpumodeld setup attempt {attempt} failed: {e}; retrying")
+      if not usbgpu_present():
+        _wait_for_egpu(params)
+      time.sleep(min(SETUP_RETRY_MAX_S, SETUP_RETRY_BASE_S * attempt))
+
+  params.put_bool("UsbGpuLoading", False)
+  params.put_bool("UsbGpuCompiled", True)
+  params.put("UsbGpuSetupProgress", "1.0")
+  cloudlog.warning(f"iqegpumodeld model: {meta['key']} ({meta['model_name']})")
+  cloudlog.warning(f"iqegpumodeld model up (warmup {warm_s * 1e3:.0f}ms)")
+
+  pipeline = EgpuPipeline(meta, infer_fn)
+  telemetry_pm = messaging.PubMaster(["egpuDockState"])
+  telemetry = EgpuDockTelemetry(telemetry_pm, big=True)
+  telemetry_every = max(1, round((1.0 / DT_MDL) / SERVICE_LIST["egpuDockState"].frequency))
+
+  sub = SubMaster(["deviceState", "carState", "roadCameraState", "extrinsicsCalibration",
+                   "driverMonitoringState", "carControl", "lateralDelay", "iqNavState", "radarState"])
+  if demo:
+    CP = get_demo_car_params()
+  else:
+    CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
+  long_delay = CP.longitudinalActuatorDelay + LONG_SMOOTH_SECONDS
+
+  parser = PhaseParser()
+  memory = DrivePacketMemory()
+  desire_logic = DesireHelper()
+  frame_meter = FrameDropMeter(20.0)
+  warps = CalibrationAtlas()
+  prev_action = log.ModelDataV2.Action()
+  slices = {k: v for k, v in meta["output_slices"].items() if k != "pad"}
+
+  produced = 0
+  stats: dict[str, list[float]] = {k: [] for k in ("pull", "warp", "infer", "publish", "loop")}
+  iter_count = 0
+  skip_count = 0
+  last_pulled_fid = -1
+  last_frame_mono = time.monotonic()
+  t_loop = time.perf_counter()
+  cloudlog.warning("iqegpumodeld starting")
+
+  while True:
+    frame_pair = cameras.pull()
+    t_pull = time.perf_counter()
+    if frame_pair is None:
+      if time.monotonic() - last_frame_mono > 2.0:
+        cloudlog.warning("iqegpumodeld camera stream silent >2s; reconnecting VisionIPC")
+        cameras = CameraIngress(None)
+        last_frame_mono = time.monotonic()
+      continue
+    last_frame_mono = time.monotonic()
+    main_buf, extra_buf, main_stamp, extra_stamp = frame_pair
+
+    stats["pull"].append(t_pull - t_loop)
+    stats["loop"].append(time.perf_counter() - t_loop)
+    t_loop = time.perf_counter()
+    if last_pulled_fid >= 0 and main_stamp.frame_id > last_pulled_fid + 1:
+      skip_count += main_stamp.frame_id - last_pulled_fid - 1
+    last_pulled_fid = main_stamp.frame_id
+    iter_count += 1
+    if iter_count % 200 == 0:
+      pcts = {k: {"p50": round(sorted(v)[len(v) // 2] * 1e3, 1),
+                  "p90": round(sorted(v)[int(len(v) * 0.9)] * 1e3, 1)}
+              for k, v in stats.items() if v}
+      cloudlog.event("iqegpu_stats", **pcts, cam_skips=skip_count, window=iter_count)
+      msg = " ".join(f"{k}=p50:{v['p50']:.0f}/p90:{v['p90']:.0f}ms" for k, v in pcts.items())
+      cloudlog.warning(f"iqegpumodeld stages: {msg} cam_skips={skip_count} over {iter_count}")
+      for v in stats.values():
+        v.clear()
+      skip_count = 0
+
+    sub.update(0)
+
+    v_ego = max(sub["carState"].vEgo, 0.0)
+    lat_delay = lateral_action_delay(params, CP, sub["lateralDelay"].lateralDelay) + LAT_SMOOTH_SECONDS
+    main_tfm, extra_tfm, live_calib_seen = warps.refresh(sub, layout.main_is_wide, layout.dual_camera)
+    dropped_frames, frame_drop_ratio, _ = frame_meter.sample(main_stamp.frame_id)
+
+    traffic = np.zeros(2, dtype=np.float32)
+    traffic[int(sub["driverMonitoringState"].isRHD)] = 1
+    desire_vec = np.zeros(DESIRE_LEN, dtype=np.float32)
+    if 0 <= desire_logic.desire < DESIRE_LEN:
+      desire_vec[desire_logic.desire] = 1
+
+    frame_delay = DT_MDL
+    action_delay = DT_MDL / 2
+    lat_action_t = lat_delay + frame_delay + action_delay
+    long_action_t = long_delay + frame_delay + action_delay
+    action_t = np.array([lat_action_t, long_action_t], dtype=np.float32)
+
+    started_at = time.perf_counter()
+    try:
+      warped = warp.run(main_buf, extra_buf, main_tfm, extra_tfm)
+    except Exception as e:
+      park(f"warp run failed: {e}")
+    t_warp = time.perf_counter()
+    stats["warp"].append(t_warp - started_at)
+
+    try:
+      output = pipeline.run(warped, desire_vec, traffic, action_t)
+    except EgpuPipelineError as e:
+      park(str(e))
+    except Exception as e:
+      park(f"eGPU inference failed: {e}")
+    t_infer = time.perf_counter()
+    stats["infer"].append(t_infer - t_warp)
+
+    execution_time = time.perf_counter() - started_at
+    sliced = {k: output[np.newaxis, sl] for k, sl in slices.items()}
+    outputs = parser.parse_vision_outputs(sliced)
+
+    action = get_action_from_model(outputs, prev_action, v_ego, float(lat_action_t), float(long_action_t),
+                                   lat_smooth_seconds=meta.get("lat_smooth_seconds"))
+    prev_action = action
+
+    model_msg = messaging.new_message("modelV2")
+    driving_msg = messaging.new_message("drivingModelData")
+    pose_msg = messaging.new_message("cameraOdometry")
+    iq_msg = messaging.new_message("iqDriveModelData")
+
+    populate_drive_messages(
+      driving_msg, model_msg, outputs, action, memory,
+      main_stamp.frame_id, extra_stamp.frame_id, sub["roadCameraState"].frameId,
+      frame_drop_ratio, main_stamp.timestamp_eof, execution_time,
+      live_calib_seen, Meta20hz,
+    )
+
+    model_msg.modelV2.big = True
+
+    desire_state = model_msg.modelV2.meta.desireState
+    lane_change_prob = desire_state[log.Desire.laneChangeLeft] + desire_state[log.Desire.laneChangeRight]
+    desire_logic.update(sub["carState"], sub["carControl"].latActive, lane_change_prob,
+                        sub["iqNavState"], model_msg.modelV2, sub["radarState"])
+    model_msg.modelV2.meta.laneChangeState = desire_logic.lane_change_state
+    model_msg.modelV2.meta.laneChangeDirection = desire_logic.lane_change_direction
+    driving_msg.drivingModelData.meta.laneChangeState = desire_logic.lane_change_state
+    driving_msg.drivingModelData.meta.laneChangeDirection = desire_logic.lane_change_direction
+    iq_msg.iqDriveModelData.turnSignalDirection = desire_logic.lane_turn_direction
+
+    populate_odometry_message(pose_msg, outputs, main_stamp.frame_id, dropped_frames,
+                              main_stamp.timestamp_eof, live_calib_seen)
+
+    channel.write(main_stamp.frame_id, make_big_channel_payload(
+      main_stamp.frame_id, live_calib_seen, execution_time, (t_infer - t_warp) * 1e3, {
+        "modelV2": model_msg.to_bytes(),
+        "drivingModelData": driving_msg.to_bytes(),
+        "cameraOdometry": pose_msg.to_bytes(),
+        "iqDriveModelData": iq_msg.to_bytes(),
+      }))
+    stats["publish"].append(time.perf_counter() - t_infer)
+    produced += 1
+    if produced == 1 or produced % 100 == 0:
+      infer_ms = (t_infer - t_warp) * 1e3
+      cloudlog.warning(f"iqegpumodeld producing: frame={main_stamp.frame_id} total={execution_time * 1e3:.0f}ms infer={infer_ms:.0f}ms count={produced}")
+
+    if produced % telemetry_every == 0:
+      telemetry.send()
+
+    frame_meter.commit(main_stamp.frame_id)
+
+
+if __name__ == "__main__":
+  try:
+    import argparse
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--demo", action="store_true")
+    args = ap.parse_args()
+    main(demo=args.demo)
+  except KeyboardInterrupt:
+    cloudlog.warning("iqegpumodeld got SIGINT")
+  except Exception:
+    import traceback
+    sentry.capture_exception()
+    cloudlog.exception("iqegpumodeld crashed, parking")
+    park(f"crashed: {traceback.format_exc(limit=8)}")

iqpilot/selfdrive/iqmodeld/model_channel.py

@@ -0,0 +1,58 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import mmap
+import os
+import pickle
+import struct
+
+from iqpilot.common.swaglog import cloudlog
+
+SMALL_CHANNEL = "/dev/shm/iqpilot_smallmodel"
+BIG_CHANNEL = "/dev/shm/iqpilot_bigmodel"
+SHM_SIZE = 8 * 1024 * 1024
+HEADER = struct.Struct("<QqQ")
+
+
+class ModelChannel:
+  def __init__(self, path: str, create: bool):
+    if create:
+      fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
+      os.ftruncate(fd, SHM_SIZE)
+    else:
+      fd = os.open(path, os.O_RDWR)
+    self.mm = mmap.mmap(fd, SHM_SIZE)
+    os.close(fd)
+    if create:
+      self.mm[:HEADER.size] = HEADER.pack(0, -1, 0)
+
+  def write(self, frame_id: int, payload: dict) -> None:
+    data = pickle.dumps(payload, protocol=pickle.HIGHEST_PROTOCOL)
+    if HEADER.size + len(data) > SHM_SIZE:
+      cloudlog.error(f"model payload {len(data)} bytes exceeds shm {SHM_SIZE}, dropping frame {frame_id}")
+      return
+    seq = HEADER.unpack(self.mm[:HEADER.size])[0]
+    HEADER.pack_into(self.mm, 0, seq + 1, frame_id, len(data))
+    self.mm[HEADER.size:HEADER.size + len(data)] = data
+    HEADER.pack_into(self.mm, 0, seq + 2, frame_id, len(data))
+
+  def peek_frame_id(self) -> int | None:
+    seq, frame_id, length = HEADER.unpack(self.mm[:HEADER.size])
+    if seq == 0 or seq % 2 != 0 or length == 0:
+      return None
+    return frame_id
+
+  def read(self) -> tuple[int, dict] | None:
+    seq1, frame_id, length = HEADER.unpack(self.mm[:HEADER.size])
+    if seq1 == 0 or seq1 % 2 != 0 or length == 0:
+      return None
+    data = bytes(self.mm[HEADER.size:HEADER.size + length])
+    seq2 = HEADER.unpack(self.mm[:HEADER.size])[0]
+    if seq1 != seq2:
+      return None
+    try:
+      return frame_id, pickle.loads(data)
+    except Exception:
+      return None

iqpilot/selfdrive/iqmodeld/model_warp.py

@@ -0,0 +1,80 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import os
+import pickle
+
+import numpy as np
+
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.system.hardware.hw import Paths
+
+
+def _load_bundle(pkl_path: str, cam_w: int, cam_h: int, frame_skip: int) -> dict:
+  with open(pkl_path, "rb") as f:
+    bundle = pickle.load(f)
+  if bundle.get("frame_skip") != frame_skip:
+    raise RuntimeError(f"frame_skip {bundle.get('frame_skip')} != {frame_skip}")
+  if (cam_w, cam_h) not in bundle:
+    raise RuntimeError(f"missing {cam_w}x{cam_h}; has {[k for k in bundle if isinstance(k, tuple)]}")
+  _verify_selftest(bundle, cam_w, cam_h)
+  return bundle
+
+
+def _verify_selftest(bundle: dict, cam_w: int, cam_h: int) -> None:
+  want = bundle.get("selftest")
+  if not want:
+    raise RuntimeError("warp artifact predates the self-test; recompiling")
+  from iqpilot.system.camerad.cameras.nv12_info import get_nv12_info
+  from iqpilot.selfdrive.iqmodeld.tools.compile_warp import selftest_digest
+  nv12_size = get_nv12_info(cam_w, cam_h)[3]
+  got = selftest_digest(bundle[(cam_w, cam_h)], cam_w, cam_h, nv12_size)
+  if got != want:
+    raise RuntimeError(f"warp self-test {got[:12]} != {want[:12]}; artifact computes differently here")
+
+
+class FrameWarp:
+
+  def __init__(self, cam_w: int, cam_h: int, frame_skip: int):
+    from tinygrad.tensor import Tensor
+
+    pkl_path = os.path.join(Paths.model_root(), f"emac_warp_{cam_w}x{cam_h}_tinygrad.pkl")
+    bundle = None
+    if os.path.isfile(pkl_path):
+      try:
+        bundle = _load_bundle(pkl_path, cam_w, cam_h, frame_skip)
+      except Exception as e:
+        cloudlog.warning(f"warp artifact unusable ({e}); discarding and recompiling")
+        os.remove(pkl_path)
+    if bundle is None:
+      cloudlog.warning(f"warp artifact missing; compiling for {cam_w}x{cam_h} (one-time)")
+      from iqpilot.selfdrive.iqmodeld.tools.compile_warp import compile_warp
+      compile_warp(cam_w, cam_h, pkl_path, frame_skip=frame_skip)
+      cloudlog.warning(f"warp compiled -> {pkl_path}")
+      bundle = _load_bundle(pkl_path, cam_w, cam_h, frame_skip)
+    self._jit = bundle[(cam_w, cam_h)]
+
+    self._npy = {"tfm": np.zeros((3, 3), dtype=np.float32), "big_tfm": np.zeros((3, 3), dtype=np.float32)}
+    self._tensors = {k: Tensor(v, device="NPY").realize() for k, v in self._npy.items()}
+    self._blob_cache: dict[tuple[str, int], object] = {}
+    self._Tensor = Tensor
+
+  def _frame_tensor(self, key: str, buf):
+    from tinygrad.device import Device
+    arr = np.frombuffer(buf.data, dtype=np.uint8)
+    ck = (key, arr.ctypes.data)
+    t = self._blob_cache.get(ck)
+    if t is None:
+      t = self._Tensor.from_blob(arr.ctypes.data, (arr.size,), dtype="uint8", device=Device.DEFAULT)
+      self._blob_cache[ck] = t
+    return t
+
+  def run(self, main_buf, extra_buf, main_tfm: np.ndarray, extra_tfm: np.ndarray) -> np.ndarray:
+    self._npy["tfm"][:] = main_tfm
+    self._npy["big_tfm"][:] = extra_tfm
+    warped = self._jit(tfm=self._tensors["tfm"], big_tfm=self._tensors["big_tfm"],
+                       frame=self._frame_tensor("img", main_buf),
+                       big_frame=self._frame_tensor("big_img", extra_buf))
+    return warped.numpy().astype(np.uint8, copy=False)

iqpilot/selfdrive/iqmodeld/modeld_selector.py

@@ -0,0 +1,373 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from collections import deque
+
+from iqpilot.cereal.messaging import PubMaster, log_from_bytes
+from setproctitle import setproctitle
+
+from iqpilot.common.filter_simple import FirstOrderFilter
+from iqpilot.common.params import Params
+from iqpilot.common.realtime import config_realtime_process
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.selfdrive.iqmodeld.model_channel import BIG_CHANNEL, SMALL_CHANNEL, ModelChannel
+
+PROCESS_NAME = "iqpilot.selfdrive.iqmodeld.modeld_selector"
+
+BIG_MODEL_DEADLINE = float(os.getenv("IQEMAC_BIG_DEADLINE_MS", "45")) / 1000.0
+BIG_MAX_LAG_FRAMES = int(os.getenv("IQEMAC_MAX_BIG_LAG_FRAMES", "6"))
+BIG_FUTURE_ACCEPT = int(os.getenv("IQEMAC_BIG_FUTURE_ACCEPT", "2"))
+BIG_ANCHOR_MS = float(os.getenv("IQEMAC_BIG_ANCHOR_MS", "90"))
+BIG_WAIT_FLOOR_S = 0.002
+BIG_WAIT_CEIL_S = float(os.getenv("IQEMAC_BIG_WAIT_CEIL_MS", "58")) / 1000.0
+BIG_MISS_LIMIT = int(os.getenv("IQEMAC_BIG_MISS_LIMIT", "80"))
+ACTIVATE_WINDOW = int(os.getenv("IQEMAC_ACTIVATE_WINDOW", "50"))
+ACTIVATE_FRAC = float(os.getenv("IQEMAC_ACTIVATE_FRAC", "0.7"))
+REARM_LIMIT = int(os.getenv("IQEMAC_REARM_LIMIT", "2"))
+MODEL_FREQ = 20.0
+WARMUP_FRAMES = 40
+STATUS_WINDOW = int(os.getenv("IQEMAC_STATUS_EVERY", "20"))
+
+SELECTOR_SERVICES = ["modelV2", "drivingModelData", "cameraOdometry", "iqDriveModelData"]
+
+EMAC_STATUS_KEYS = {
+  "active": "MacModelActive", "failed": "MacModelFailed", "last_error": "MacModelLastError",
+  "latency_ms": "MacModelLatencyMs", "status": "MacModelStatus",
+  "reachable": "MacModelReachable", "progress": "MacModelDownloadProgress",
+}
+EGPU_STATUS_KEYS = {
+  "active": "UsbGpuActive", "failed": "UsbGpuFailed", "last_error": "UsbGpuLastError",
+  "latency_ms": "UsbGpuLatencyMs", "status": "UsbGpuStatus",
+  "reachable": "UsbGpuPresent", "progress": "UsbGpuSetupProgress",
+}
+
+
+def backend_status_keys(emac_enabled: bool, egpu_enabled: bool, egpu_present: bool = False) -> dict[str, str]:
+  from iqpilot.selfdrive.iqmodeld.egpu_helpers import resolve_backend
+  return EGPU_STATUS_KEYS if resolve_backend(emac_enabled, egpu_enabled, egpu_present) == "egpu" else EMAC_STATUS_KEYS
+
+
+def resolve_status_keys(params):
+  from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_present_consented, egpu_selected
+  return backend_status_keys(params.get_bool("IQEmacEnabled"), egpu_selected(params), egpu_present_consented(params))
+
+
+def resolve_model_name(params, keys) -> str:
+  if keys is EGPU_STATUS_KEYS:
+    from iqpilot.selfdrive.iqmodeld.egpu_model import DEFAULT_EGPU_MODEL, resolve_egpu_model
+    resolved = resolve_egpu_model(params, allow_refresh=False)
+    return resolved["key"] if resolved else DEFAULT_EGPU_MODEL
+  name = params.get("IQEmacModel") or b"lebrowski"
+  return name.decode() if isinstance(name, bytes) else name
+
+
+class AsyncParamWriter:
+
+  def __init__(self, params: Params):
+    self._params = params
+    self._pending: dict[str, object] = {}
+    self._lock = threading.Lock()
+    self._event = threading.Event()
+    threading.Thread(target=self._drain, daemon=True).start()
+
+  def put(self, key: str, value) -> None:
+    with self._lock:
+      self._pending[key] = value
+    self._event.set()
+
+  def put_bool(self, key: str, value: bool) -> None:
+    self.put(key, bool(value))
+
+  def _drain(self) -> None:
+    while True:
+      self._event.wait()
+      self._event.clear()
+      with self._lock:
+        batch, self._pending = self._pending, {}
+      for key, value in batch.items():
+        try:
+          if isinstance(value, bool):
+            self._params.put_bool(key, value)
+          else:
+            self._params.put(key, value)
+        except Exception:
+          cloudlog.exception(f"async param write failed: {key}")
+
+
+def wait_for_big(big_channel, target: int, deadline: float, min_frame: int = -1,
+                 max_lag_frames: int = BIG_MAX_LAG_FRAMES) -> tuple[dict | None, int | None]:
+  big_peek = None
+  grab_at = deadline - 0.004
+  while time.perf_counter() < deadline:
+    bfid = big_channel.peek_frame_id()
+    big_peek = bfid
+    if bfid == target - 1 and time.perf_counter() < grab_at:
+      time.sleep(0.0005)
+      continue
+    if bfid is not None and min_frame < bfid <= target + BIG_FUTURE_ACCEPT and target - bfid <= max_lag_frames:
+      got = big_channel.read()
+      if got is not None and got[0] == bfid:
+        return got[1], big_peek
+      break
+    if bfid is None or bfid <= min_frame or bfid > target + BIG_FUTURE_ACCEPT or target - bfid > max_lag_frames:
+      break
+    time.sleep(0.0005)
+  return None, big_peek
+
+
+class BigLatch:
+
+  def __init__(self, miss_limit: int = BIG_MISS_LIMIT, activate_window: int = ACTIVATE_WINDOW,
+               activate_frac: float = ACTIVATE_FRAC, rearm_limit: int = REARM_LIMIT):
+    self.miss_limit = miss_limit
+    self.activate_window = activate_window
+    self.activate_need = int(round(activate_window * activate_frac))
+    self.rearm_limit = rearm_limit
+    self.active = False
+    self.done = False
+    self._miss = 0
+    self._window: deque[bool] = deque(maxlen=activate_window)
+    self._retires = 0
+
+  def update(self, used_big: bool) -> tuple[bool, bool]:
+    if self.done:
+      return False, False
+    if not self.active:
+      self._window.append(used_big)
+      if len(self._window) >= self.activate_window and sum(self._window) >= self.activate_need:
+        self.active = True
+        self._miss = 0
+        self._window.clear()
+        return True, False
+    if used_big:
+      self._miss = 0
+    elif self.active:
+      self._miss += 1
+      if self._miss >= self.miss_limit:
+        self.active = False
+        self._miss = 0
+        self._window.clear()
+        self._retires += 1
+        self.done = self._retires > self.rearm_limit
+        return False, True
+    return False, False
+
+
+def _patch_and_send(pm: PubMaster, payload: dict, frame_drop_perc: float, selector_dropped: int,
+                    target: int, source_lag: int, mismatch: bool | None = None) -> None:
+  msgs = payload["msgs"]
+  if mismatch is None:
+    mismatch = source_lag > 0
+
+  model_msg = log_from_bytes(msgs["modelV2"]).as_builder()
+  if mismatch:
+    model_msg.modelV2.frameId = target
+    model_msg.modelV2.frameAge = max(model_msg.modelV2.frameAge, source_lag)
+  model_msg.modelV2.frameDropPerc = frame_drop_perc
+  pm.send("modelV2", model_msg)
+
+  driving_msg = log_from_bytes(msgs["drivingModelData"]).as_builder()
+  if mismatch:
+    driving_msg.drivingModelData.frameId = target
+  driving_msg.drivingModelData.frameDropPerc = frame_drop_perc
+  pm.send("drivingModelData", driving_msg)
+
+  pose_msg = log_from_bytes(msgs["cameraOdometry"]).as_builder()
+  if mismatch:
+    pose_msg.cameraOdometry.frameId = target
+  pose_msg.valid = bool(payload["live_calib_seen"]) and selector_dropped < 1 and not mismatch
+  pm.send("cameraOdometry", pose_msg)
+
+  pm.send("iqDriveModelData", msgs["iqDriveModelData"])
+
+
+def _read_float(params, key: str, default: float) -> float:
+  v = params.get(key)
+  try:
+    return float(v) if v is not None else default
+  except (TypeError, ValueError):
+    return default
+
+
+def main() -> None:
+  cloudlog.warning("modeld_selector init")
+  cloudlog.bind(daemon=PROCESS_NAME)
+  setproctitle(PROCESS_NAME)
+  config_realtime_process([0, 1, 2, 3], 54)
+
+  params = Params()
+  keys = resolve_status_keys(params)
+  pwriter = AsyncParamWriter(params)
+  pwriter.put_bool(keys["active"], False)
+  pwriter.put_bool(keys["failed"], False)
+  pm = PubMaster(SELECTOR_SERVICES)
+
+  small_channel: ModelChannel | None = None
+  big_channel: ModelChannel | None = None
+  latch = BigLatch()
+  big_used_count = 0
+  run_count = 0
+  last_published = -1
+  last_big_published = -1
+  frame_dropped_filter = FirstOrderFilter(0.0, 10.0, 1.0 / MODEL_FREQ)
+  recent_big = deque(maxlen=STATUS_WINDOW)
+  model_name = resolve_model_name(params, keys)
+  last_backend_check = 0.0
+  last_latency_ms = 0.0
+  last_source_lag = 0
+  miss_reasons = {"no_head": 0, "already_used": 0, "far_future": 0,
+                  "too_stale": 0, "head_prev_timeout": 0, "read_race": 0}
+
+  cloudlog.warning(f"modeld_selector starting (max_big_lag_frames={BIG_MAX_LAG_FRAMES})")
+  while True:
+    if small_channel is None:
+      try:
+        small_channel = ModelChannel(SMALL_CHANNEL, create=False)
+      except OSError:
+        time.sleep(0.05)
+        continue
+    if big_channel is None:
+      try:
+        big_channel = ModelChannel(BIG_CHANNEL, create=False)
+      except OSError:
+        big_channel = None
+
+    fid = small_channel.peek_frame_id()
+    if fid is None or fid == last_published:
+      time.sleep(0.0005)
+      continue
+    if last_published >= 0 and fid < last_published - 1:
+      cloudlog.warning(f"modeld_selector frame reset {last_published} -> {fid}; re-arming")
+      last_published = -1
+      last_big_published = -1
+      big_used_count = 0
+      run_count = 0
+      latch = BigLatch()
+      pwriter.put_bool(keys["active"], False)
+      pwriter.put_bool(keys["failed"], False)
+
+    now_mono = time.monotonic()
+    if now_mono - last_backend_check > 1.0:
+      last_backend_check = now_mono
+      new_keys = resolve_status_keys(params)
+      if new_keys is not keys:
+        cloudlog.warning(f"modeld_selector backend changed {keys['active']} -> {new_keys['active']}; re-arming")
+        pwriter.put_bool(keys["active"], False)
+        pwriter.put_bool(keys["failed"], False)
+        keys = new_keys
+        model_name = resolve_model_name(params, keys)
+        recent_big.clear()
+        last_big_published = -1
+        big_used_count = 0
+        run_count = 0
+        latch = BigLatch()
+        pwriter.put_bool(keys["active"], False)
+        pwriter.put_bool(keys["failed"], False)
+    target = fid
+    t_start = time.perf_counter()
+
+    small_payload = None
+    got = small_channel.read()
+    if got is not None and got[0] == target:
+      small_payload = got[1]
+
+    payload = None
+    used_big = False
+    big_peek = None
+    if big_channel is not None and not latch.done:
+      deadline = t_start + BIG_MODEL_DEADLINE
+      sof_ns = (small_payload or {}).get("timestamp_sof")
+      if sof_ns:
+        remaining = (BIG_ANCHOR_MS / 1000.0) - (time.clock_gettime(time.CLOCK_BOOTTIME) - sof_ns / 1e9)
+        deadline = t_start + min(max(remaining, BIG_WAIT_FLOOR_S), BIG_WAIT_CEIL_S)
+      payload, big_peek = wait_for_big(big_channel, target, deadline,
+                                       last_big_published, BIG_MAX_LAG_FRAMES)
+      used_big = payload is not None
+      if not used_big:
+        if big_peek is None:
+          miss_reasons["no_head"] += 1
+        elif big_peek <= last_big_published:
+          miss_reasons["already_used"] += 1
+        elif big_peek > target + BIG_FUTURE_ACCEPT:
+          miss_reasons["far_future"] += 1
+        elif target - big_peek > BIG_MAX_LAG_FRAMES:
+          miss_reasons["too_stale"] += 1
+        elif big_peek == target - 1:
+          miss_reasons["head_prev_timeout"] += 1
+        else:
+          miss_reasons["read_race"] += 1
+
+    if payload is None:
+      payload = small_payload
+      if payload is None:
+        got = small_channel.read()
+        if got is not None and got[0] == target:
+          payload = got[1]
+
+    activated_now, failed_now = latch.update(used_big)
+    if activated_now:
+      pwriter.put_bool(keys["active"], True)
+      pwriter.put_bool(keys["failed"], False)
+      cloudlog.warning(f"modeld_selector switched to BIG model at frame {target}")
+    elif failed_now:
+      pwriter.put_bool(keys["active"], False)
+      pwriter.put_bool(keys["failed"], latch.done)
+      pwriter.put(keys["last_error"], "big model stalled onroad; local fallback latched"
+                 if latch.done else "big model stalled onroad; small active, big may re-arm")
+      if latch.done:
+        cloudlog.warning(f"modeld_selector big stalled, staying on small until next ignition (frame {target})")
+      else:
+        cloudlog.warning(f"modeld_selector big stalled, small active; big may re-arm after a clean streak (frame {target})")
+
+    if payload is not None:
+      selector_dropped = max(0, target - last_published - 1) if last_published >= 0 else 0
+      frames_dropped = frame_dropped_filter.update(min(selector_dropped, 10))
+      if run_count < WARMUP_FRAMES:
+        frame_dropped_filter.x = 0.0
+        frames_dropped = 0.0
+      run_count += 1
+      recent_big.append(used_big)
+      if used_big:
+        big_used_count += 1
+        big_fid = int(payload.get("frame_id", big_peek if big_peek is not None else target))
+        last_big_published = min(big_fid, target)
+        last_latency_ms = float(payload.get("model_execution_time", 0.0)) * 1e3
+      source_lag = max(0, target - int(payload.get("frame_id", target)))
+      frame_mismatch = int(payload.get("frame_id", target)) != target
+      last_source_lag = source_lag
+      if run_count % STATUS_WINDOW == 0:
+        hit_rate = (sum(recent_big) / len(recent_big)) if recent_big else 0.0
+        pwriter.put(keys["latency_ms"], last_latency_ms)
+        pwriter.put(keys["status"], json.dumps({
+          "active": latch.active,
+          "failed": latch.done,
+          "hit_rate": round(hit_rate, 3),
+          "latency_ms": round(last_latency_ms, 1),
+          "source_lag_frames": last_source_lag,
+          "model": model_name,
+          "reachable": params.get_bool(keys["reachable"]),
+          "download_progress": _read_float(params, keys["progress"], 1.0),
+          "ts_mono": round(time.monotonic(), 1),
+        }))
+      if run_count % 100 == 0:
+        cloudlog.warning(f"modeld_selector misses: {miss_reasons}")
+        cloudlog.warning(f"modeld_selector: big_used={big_used_count}/{run_count} "
+                         f"last_big_peek={big_peek} target={target} active={latch.active} "
+                         f"max_big_lag={BIG_MAX_LAG_FRAMES}")
+
+      frame_drop_perc = 100.0 * frames_dropped / (1.0 + frames_dropped)
+      _patch_and_send(pm, payload, frame_drop_perc, selector_dropped, target, source_lag, frame_mismatch)
+      last_published = target
+
+
+if __name__ == "__main__":
+  try:
+    main()
+  except KeyboardInterrupt:
+    cloudlog.warning("modeld_selector got SIGINT")

iqpilot/selfdrive/iqmodeld/models/runners/model_runner.py

@@ -129,7 +129,7 @@ class ModelRunner(RunnerRoot):
     if not active:
       raise ValueError("runner started without an active model bundle")
 
-    self.models = {spec.type.raw: ArtifactSpec(spec) for spec in active.models}
+    self.models = {spec.type.raw: ArtifactSpec(spec) for spec in _qcom_models(active)}
     self.is_20hz_3d = False
     self.is_20hz = active.is20hz
     self.inputs = {}
@@ -180,8 +180,15 @@ class ModelRunner(RunnerRoot):
 
 # ---- runner selection (which backend to build for the active bundle) ----------
 
+def _qcom_models(bundle) -> list:
+  # usbeMac artifacts ride along in a bundle for the eGPU host; they are never
+  # loaded on QCOM and must not affect runner classification
+  return [m for m in bundle.models if m.type.raw != ModelType.usbeMac]
+
+
 def _single_artifact_prefix(bundle, prefix: str) -> bool:
-  return len(bundle.models) == 1 and bundle.models[0].artifact.fileName.startswith(prefix)
+  models = _qcom_models(bundle)
+  return len(models) == 1 and models[0].artifact.fileName.startswith(prefix)
 
 
 def _is_fused_bundle(bundle) -> bool:
@@ -193,7 +200,7 @@ def _is_supercombo_bundle(bundle) -> bool:
 
 
 def _is_split_bundle(bundle) -> bool:
-  present = {m.type.raw for m in bundle.models}
+  present = {m.type.raw for m in _qcom_models(bundle)}
   split_kinds = {ModelType.vision, ModelType.policy, ModelType.offPolicy, ModelType.onPolicy}
   return not present.isdisjoint(split_kinds)
 
@@ -205,7 +212,9 @@ def get_model_runner() -> "ModelRunner":
   from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.tinygrad_runner import (TinygradRunner,
                                                                                              TinygradSplitRunner)
   bundle = _fetch_bundle()
-  if not (bundle and bundle.models):
+  # an eMac-only bundle (no QCOM-loadable models) runs the stock default on
+  # device; the big host serves the bundle's precompiled artifact
+  if not (bundle and bundle.models and _qcom_models(bundle)):
     return TinygradRunner(ModelType.supercombo)
 
   if _is_supercombo_bundle(bundle):
@@ -219,4 +228,4 @@ def get_model_runner() -> "ModelRunner":
     return TinygradCombinedSplitRunner()
   if _is_split_bundle(bundle):
     return TinygradSplitRunner()
-  return TinygradRunner(bundle.models[0].type.raw)
+  return TinygradRunner(_qcom_models(bundle)[0].type.raw)

iqpilot/selfdrive/iqmodeld/temporal_state.py

@@ -0,0 +1,131 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+DEFAULT_FRAME_SKIP = 4
+
+MODEL_INPUT_SPEC: dict[str, tuple[tuple[int, ...], str]] = {
+  "img": ((1, 12, 128, 256), "uint8"),
+  "big_img": ((1, 12, 128, 256), "uint8"),
+  "desire_pulse": ((1, 25, 8), "float32"),
+  "traffic_convention": ((1, 2), "float32"),
+  "features_buffer": ((1, 24, 512), "float32"),
+  "action_t": ((1, 2), "float32"),
+}
+
+
+def spec_from_meta(meta: dict) -> dict[str, tuple[tuple[int, ...], str]] | None:
+  shapes = meta.get("input_shapes")
+  if not shapes:
+    return None
+  return {name: (tuple(shape), "uint8" if name in ("img", "big_img") else "float32")
+          for name, shape in shapes.items()}
+
+
+class TemporalInputState:
+  def __init__(self, frame_skip: int, spec: dict[str, tuple[tuple[int, ...], str]] = MODEL_INPUT_SPEC):
+    self.frame_skip = frame_skip
+    img = spec["img"][0]
+    fb = spec["features_buffer"][0]
+    dp = spec["desire_pulse"][0]
+
+    self.n_frames = img[1] // 6
+    img_q_shape = (frame_skip * (self.n_frames - 1) + 1, 6, img[2], img[3])
+    self._img_shape = img
+    self._fb_shape = fb
+    self._dp_shape = dp
+    feat_dim = math.prod(fb[2:])
+
+    self.img_q = np.zeros(img_q_shape, dtype=np.uint8)
+    self.big_img_q = np.zeros(img_q_shape, dtype=np.uint8)
+    self.feat_q = np.zeros((frame_skip * fb[1], fb[0], feat_dim), dtype=np.float32)
+    self.desire_q = np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32)
+    self.prev_desire = np.zeros(dp[2], dtype=np.float32)
+    self.prev_feat = np.zeros((fb[0], feat_dim), dtype=np.float32)
+
+  @staticmethod
+  def _shift_append(q: np.ndarray, new_val: np.ndarray) -> None:
+    q[:-1] = q[1:]
+    q[-1] = new_val
+
+  def push_and_materialize(self, warped: np.ndarray, desire_pulse: np.ndarray,
+                           traffic_convention: np.ndarray, action_t: np.ndarray,
+                           ) -> dict[str, np.ndarray]:
+    fs = self.frame_skip
+
+    cur = desire_pulse.astype(np.float32).copy()
+    cur[0] = 0
+    pulse = np.where(cur - self.prev_desire > 0.99, cur, 0).astype(np.float32)
+    self.prev_desire[:] = cur
+
+    self._shift_append(self.img_q, warped[0])
+    self._shift_append(self.big_img_q, warped[1])
+    self._shift_append(self.desire_q, pulse.reshape(self._dp_shape[0], self._dp_shape[2]))
+    self._shift_append(self.feat_q, self.prev_feat)
+
+    dp = self._dp_shape
+    return {
+      "img": np.ascontiguousarray(self.img_q[::fs]).reshape(self._img_shape),
+      "big_img": np.ascontiguousarray(self.big_img_q[::fs]).reshape(self._img_shape),
+      "features_buffer": np.ascontiguousarray(self.feat_q[::fs]).reshape(self._fb_shape),
+      "desire_pulse": self.desire_q.reshape(dp[1], fs, dp[0], dp[2]).max(axis=1).reshape(dp),
+      "traffic_convention": traffic_convention.astype(np.float32).reshape(1, -1),
+      "action_t": action_t.astype(np.float32).reshape(1, -1),
+    }
+
+  def note_hidden_state(self, model_output: np.ndarray, hidden_slice: slice) -> None:
+    self.prev_feat[:] = model_output[hidden_slice].reshape(self.prev_feat.shape)
+
+
+class SplitTemporalState:
+
+  def __init__(self, frame_skip: int, img_shape: tuple[int, ...],
+               feature_shape: tuple[int, ...], desire_shape: tuple[int, ...]):
+    self.frame_skip = frame_skip
+    self._img_shape = tuple(img_shape)
+    self._fb_shape = tuple(feature_shape)
+    self._dp_shape = tuple(desire_shape)
+
+    n_frames = img_shape[1] // 6
+    img_q_shape = (frame_skip * (n_frames - 1) + 1, 6, img_shape[2], img_shape[3])
+    self.img_q = np.zeros(img_q_shape, dtype=np.uint8)
+    self.big_img_q = np.zeros(img_q_shape, dtype=np.uint8)
+    self.feat_q = np.zeros((frame_skip * (feature_shape[1] - 1) + 1, feature_shape[0], feature_shape[2]),
+                           dtype=np.float32)
+    self.desire_q = np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), dtype=np.float32)
+    self.prev_desire = np.zeros(desire_shape[2], dtype=np.float32)
+
+  def materialize_vision(self, warped: np.ndarray, desire: np.ndarray) -> dict[str, np.ndarray]:
+    fs = self.frame_skip
+    cur = desire.astype(np.float32).copy()
+    cur[0] = 0
+    pulse = np.where(cur - self.prev_desire > 0.99, cur, 0).astype(np.float32)
+    self.prev_desire[:] = cur
+
+    TemporalInputState._shift_append(self.img_q, warped[0])
+    TemporalInputState._shift_append(self.big_img_q, warped[1])
+    TemporalInputState._shift_append(self.desire_q, pulse.reshape(self._dp_shape[0], self._dp_shape[2]))
+    return {
+      "img": np.ascontiguousarray(self.img_q[::fs]).reshape(self._img_shape),
+      "big_img": np.ascontiguousarray(self.big_img_q[::fs]).reshape(self._img_shape),
+    }
+
+  def materialize_policy(self, vision_feature: np.ndarray, traffic_convention: np.ndarray,
+                         action_t: np.ndarray | None = None) -> dict[str, np.ndarray]:
+    fs = self.frame_skip
+    TemporalInputState._shift_append(self.feat_q, vision_feature.reshape(self._fb_shape[0], self._fb_shape[2]))
+    dp = self._dp_shape
+    out = {
+      "features_buffer": np.ascontiguousarray(self.feat_q[::fs]).reshape(self._fb_shape),
+      "desire_pulse": self.desire_q.reshape(dp[1], fs, dp[0], dp[2]).max(axis=1).reshape(dp),
+      "traffic_convention": traffic_convention.astype(np.float32).reshape(1, -1),
+    }
+    if action_t is not None:
+      out["action_t"] = action_t.astype(np.float32).reshape(1, -1)
+    return out

iqpilot/selfdrive/iqmodeld/tests/test_egpu_stock_parity.py

@@ -0,0 +1,163 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import types
+
+import pytest
+
+from iqpilot.cereal import log, messaging
+from iqpilot.cereal.services import SERVICE_LIST
+
+
+class TestTelemetryContract:
+  def test_service_is_published_at_stock_cadence(self):
+    assert "egpuDockState" in SERVICE_LIST
+    assert SERVICE_LIST["egpuDockState"].frequency == 10.
+
+  def test_message_carries_every_stock_field(self):
+    msg = messaging.new_message("egpuDockState")
+    state = msg.egpuDockState
+    for field in ("tempC", "memoryTempC", "powerDrawW", "powerLimitW", "gpuUsagePercent",
+                  "gpuClockMhz", "fanSpeedRpm", "pcieLtssm", "supplyVoltage", "supplyCurrent"):
+      setattr(state, field, 1)
+      assert getattr(state, field) == 1
+
+  def test_metrics_refresh_matches_stock(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_telemetry import METRICS_REFRESH_EVERY
+    assert METRICS_REFRESH_EVERY == 100
+
+  def test_send_without_a_gpu_publishes_an_invalid_message(self):
+    from iqpilot.selfdrive.iqmodeld import egpu_telemetry
+    sent = []
+    telemetry = egpu_telemetry.EgpuDockTelemetry(types.SimpleNamespace(send=lambda n, m: sent.append((n, m))), big=True)
+    telemetry._device = lambda: types.SimpleNamespace(_opened_devices=set())
+    telemetry.send()
+    assert sent and sent[0][0] == "egpuDockState"
+    assert sent[0][1].valid is False
+
+
+class TestBigFrameFlag:
+  def test_model_message_carries_the_big_flag(self):
+    msg = messaging.new_message("modelV2")
+    msg.modelV2.big = True
+    assert msg.modelV2.big
+
+
+class TestStatusParams:
+  def test_loading_param_exists_and_is_cleared_like_stock(self):
+    from pathlib import Path
+    root = Path(__file__).resolve().parents[3]
+    keys = (root / "common" / "params_keys.h").read_text()
+    assert '{"UsbGpuLoading"' in keys
+    line = next(ln for ln in keys.splitlines() if '"UsbGpuLoading"' in ln)
+    for flag in ("CLEAR_ON_MANAGER_START", "CLEAR_ON_OFFROAD_TRANSITION", "CLEAR_ON_IGNITION_ON"):
+      assert flag in line
+
+
+class TestAlerts:
+  def test_both_stock_big_model_events_exist(self):
+    assert hasattr(log.OnroadEvent.EventName, "bigModelLoading")
+    assert hasattr(log.OnroadEvent.EventName, "bigModelFailed")
+
+  def test_alerts_are_wired_with_stock_severities(self):
+    from iqpilot.selfdrive.selfdrived.events import EVENTS, ET
+    EventName = log.OnroadEvent.EventName
+    loading = EVENTS[EventName.bigModelLoading]
+    failed = EVENTS[EventName.bigModelFailed]
+    assert ET.NO_ENTRY in loading
+    assert ET.SOFT_DISABLE in failed and ET.PERMANENT in failed
+
+
+class TestFirmwareGate:
+  def test_runtime_refuses_a_dock_on_other_firmware(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import usbgpu_present
+    from iqpilot.system.hardware.usb import EGPU_DOCK_FW_PRODUCT, EGPU_DOCK_USB_IDS
+    vid, pid = EGPU_DOCK_USB_IDS[0]
+    d = tmp_path / "1-1"
+    d.mkdir()
+    (d / "idVendor").write_text(f"{vid:04x}\n")
+    (d / "idProduct").write_text(f"{pid:04x}\n")
+    (d / "product").write_text("custom deadbeef-CLEAN\n")
+    assert not usbgpu_present(str(tmp_path))
+    (d / "product").write_text(EGPU_DOCK_FW_PRODUCT + "\n")
+    assert usbgpu_present(str(tmp_path))
+
+
+class TestAutoFlash:
+  def test_hardwared_drives_the_flasher_offroad_only(self):
+    from iqpilot.system.hardware.hardwared import EgpuDockFlasher
+    f = EgpuDockFlasher()
+    calls = []
+    f.flash = lambda: calls.append(1)
+    stale = [{"vendorId": 0xADD1, "productId": 0x0001, "product": "custom deadbeef-CLEAN"}]
+    f.update(False, stale)
+    assert f.attempts == 0, "must not flash onroad"
+    f.update(True, stale)
+    assert f.attempts == 1
+    if f.thread is not None:
+      f.thread.join(timeout=5)
+
+  def test_matching_firmware_is_never_flashed(self):
+    from iqpilot.system.hardware.egpu_dock.flash import bundled_version
+    from iqpilot.system.hardware.hardwared import EgpuDockFlasher
+    f = EgpuDockFlasher()
+    f.flash = lambda: pytest.fail("flashed a dock that already matches")
+    f.update(True, [{"vendorId": 0xADD1, "productId": 0x0001, "product": bundled_version()}])
+    assert f.attempts == 0
+
+  def test_attempts_are_bounded_like_stock(self):
+    from iqpilot.system.hardware.hardwared import EgpuDockFlasher
+    assert EgpuDockFlasher.MAX_ATTEMPTS == 3
+    assert EgpuDockFlasher.RETRY_INTERVAL == 20.
+
+
+class TestDockIsItsOwnConsent:
+
+  def _params(self, **flags):
+    class P:
+      def get_bool(self, k):
+        return bool(flags.get(k, False))
+      def get(self, k, *a, **kw):
+        return None
+    return P()
+
+  def _sysfs_with_dock(self, tmp_path, product=None):
+    from iqpilot.system.hardware.usb import EGPU_DOCK_FW_PRODUCT, EGPU_DOCK_USB_IDS
+    vid, pid = EGPU_DOCK_USB_IDS[0]
+    d = tmp_path / "1-1"
+    d.mkdir()
+    (d / "idVendor").write_text(f"{vid:04x}\n")
+    (d / "idProduct").write_text(f"{pid:04x}\n")
+    (d / "product").write_text((product or EGPU_DOCK_FW_PRODUCT) + "\n")
+    return str(tmp_path)
+
+  def test_a_plugged_in_dock_selects_itself(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    assert egpu_selected(self._params(), self._sysfs_with_dock(tmp_path))
+
+  def test_nothing_plugged_in_selects_nothing(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    assert not egpu_selected(self._params(), str(tmp_path))
+
+  def test_a_dock_on_foreign_firmware_does_not_select_itself(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    assert not egpu_selected(self._params(), self._sysfs_with_dock(tmp_path, "custom deadbeef-CLEAN"))
+
+  def test_the_user_can_force_it_off(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    root = self._sysfs_with_dock(tmp_path)
+    assert not egpu_selected(self._params(IQEgpuDisabled=True), root)
+
+  def test_the_param_can_force_it_on_without_hardware(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected
+    assert egpu_selected(self._params(IQEgpuEnabled=True), str(tmp_path))
+
+  def test_present_dock_wins_even_with_emac_enabled(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected, resolve_backend, usbgpu_present
+    root = self._sysfs_with_dock(tmp_path)
+    assert resolve_backend(True, egpu_selected(self._params(), root), usbgpu_present(root)) == "egpu"
+
+  def test_force_param_without_hardware_yields_to_emac(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_selected, resolve_backend, usbgpu_present
+    assert resolve_backend(True, egpu_selected(self._params(IQEgpuEnabled=True), str(tmp_path)),
+                           usbgpu_present(str(tmp_path))) == "emac"

iqpilot/selfdrive/iqmodeld/tests/test_egpu_worker.py

@@ -0,0 +1,509 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import io
+import json
+import time
+import urllib.request
+
+import numpy as np
+import pytest
+
+from iqpilot.selfdrive.iqmodeld.egpu_helpers import (
+  resolve_backend, resolve_download_url, usbgpu_present,
+)
+from iqpilot.selfdrive.iqmodeld.egpu_pipeline import (
+  EgpuPipeline, EgpuPipelineError, make_big_channel_payload,
+)
+from iqpilot.selfdrive.iqmodeld.egpu_model import EGPU_MODELS, get_egpu_model
+from iqpilot.selfdrive.iqmodeld.temporal_state import MODEL_INPUT_SPEC as INPUT_SPEC
+
+
+class FakeParams:
+  def __init__(self, **flags):
+    self._flags = {k: bool(v) for k, v in flags.items()}
+
+  def get_bool(self, key: str) -> bool:
+    return self._flags.get(key, False)
+
+
+def _fake_usb_device(root, vid: str, pid: str, name: str = "1-1", product: str | None = None):
+  from iqpilot.system.hardware.usb import EGPU_DOCK_FW_PRODUCT
+  d = root / name
+  d.mkdir()
+  (d / "idVendor").write_text(vid + "\n")
+  (d / "idProduct").write_text(pid + "\n")
+  (d / "product").write_text((product if product is not None else EGPU_DOCK_FW_PRODUCT) + "\n")
+
+
+class TestPresence:
+  def test_present(self, tmp_path):
+    _fake_usb_device(tmp_path, "add1", "0001")
+    assert usbgpu_present(str(tmp_path))
+
+  def test_foreign_firmware_absent(self, tmp_path):
+    _fake_usb_device(tmp_path, "add1", "0001", product="custom deadbeef-CLEAN")
+    assert not usbgpu_present(str(tmp_path))
+
+  def test_wrong_ids_absent(self, tmp_path):
+    _fake_usb_device(tmp_path, "05ac", "12a8")
+    assert not usbgpu_present(str(tmp_path))
+
+  def test_empty_bus_absent(self, tmp_path):
+    assert not usbgpu_present(str(tmp_path))
+
+  def test_unreadable_entries_skipped(self, tmp_path):
+    (tmp_path / "usb1").mkdir()
+    _fake_usb_device(tmp_path, "add1", "0001", name="1-2")
+    assert usbgpu_present(str(tmp_path))
+
+
+class TestBackendResolution:
+  def test_none(self):
+    assert resolve_backend(False, False) is None
+
+  def test_emac_only(self):
+    assert resolve_backend(True, False) == "emac"
+
+  def test_egpu_only(self):
+    assert resolve_backend(False, True) == "egpu"
+
+  def test_force_param_yields_to_emac_without_hardware(self):
+    assert resolve_backend(True, True) == "emac"
+
+  def test_present_dock_wins_over_emac(self):
+    assert resolve_backend(True, True, True) == "egpu"
+
+
+class TestManagerGating:
+  @pytest.fixture
+  def pc(self):
+    return pytest.importorskip("iqpilot.system.manager.process_config")
+
+  def test_egpu_needs_presence(self, pc, monkeypatch):
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: True)
+    assert pc.egpu_enabled(True, FakeParams(IQEgpuEnabled=True), None)
+    assert pc.egpu_enabled(True, FakeParams(), None)
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: False)
+    assert not pc.egpu_enabled(True, FakeParams(IQEgpuEnabled=True), None)
+    assert not pc.egpu_enabled(True, FakeParams(), None)
+
+  def test_present_dock_wins_over_left_on_emac(self, pc, monkeypatch):
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: True)
+    both = FakeParams(IQEmacEnabled=True, IQEgpuEnabled=True)
+    assert not pc.emac_enabled(True, both, None)
+    assert pc.egpu_enabled(True, both, None)
+
+  def test_emac_runs_when_no_dock(self, pc, monkeypatch):
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: False)
+    assert pc.emac_enabled(True, FakeParams(IQEmacEnabled=True), None)
+
+  def test_disabled_dock_yields_to_emac(self, pc, monkeypatch):
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: True)
+    both = FakeParams(IQEmacEnabled=True, IQEgpuDisabled=True)
+    assert pc.emac_enabled(True, both, None)
+    assert not pc.egpu_enabled(True, both, None)
+
+  def test_disabled_dock_runs_no_backend_when_no_emac(self, pc, monkeypatch):
+    monkeypatch.setattr(pc, "usbgpu_present", lambda: True)
+    off = FakeParams(IQEgpuDisabled=True)
+    assert not pc.egpu_enabled(True, off, None)
+    assert not pc.emac_enabled(True, off, None)
+
+  def test_selector_runs_for_either_backend(self, pc):
+    assert pc.big_model_enabled(True, FakeParams(IQEmacEnabled=True), None)
+    assert pc.big_model_enabled(True, FakeParams(IQEgpuEnabled=True), None)
+    assert not pc.big_model_enabled(True, FakeParams(), None)
+
+  def test_iqegpumodeld_registered(self, pc):
+    assert "iqegpumodeld" in pc.managed_processes
+    assert "maciqmodeld" in pc.managed_processes
+
+
+class TestDownloadResolve:
+  def test_direct_url_passthrough(self):
+    assert resolve_download_url("https://x/y.onnx", "0" * 64, 5) == "https://x/y.onnx"
+
+  def test_commalfs_batch(self, monkeypatch):
+    seen = {}
+
+    def fake_urlopen(req, timeout=0):
+      seen["url"] = req.full_url
+      seen["body"] = json.loads(req.data)
+      return io.BytesIO(json.dumps(
+        {"objects": [{"actions": {"download": {"href": "https://signed/url"}}}]}).encode())
+
+    monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
+    sha = "a5" * 32
+    url = resolve_download_url(f"commalfs:{sha}", sha, 1234)
+    assert url == "https://signed/url"
+    assert seen["body"]["objects"] == [{"oid": sha, "size": 1234}]
+    assert seen["url"].endswith("/info/lfs/objects/batch")
+
+
+def _zero_infer(output_len: int, fill=None):
+  calls = []
+
+  def infer(inputs):
+    for name, (shape, dtype) in INPUT_SPEC.items():
+      assert tuple(inputs[name].shape) == shape, name
+      assert inputs[name].dtype == np.dtype(dtype), name
+    calls.append({k: v.copy() for k, v in inputs.items()})
+    out = np.zeros(output_len, dtype=np.float32)
+    if fill is not None:
+      out[:] = fill
+    return out
+
+  infer.calls = calls
+  return infer
+
+
+def _frame_inputs(seed=0):
+  rng = np.random.default_rng(seed)
+  warped = rng.integers(0, 256, (2, 6, 128, 256)).astype(np.uint8)
+  desire = np.zeros(8, dtype=np.float32)
+  traffic = np.array([1.0, 0.0], dtype=np.float32)
+  action_t = np.array([0.25, 0.55], dtype=np.float32)
+  return warped, desire, traffic, action_t
+
+
+class TestEgpuPipeline:
+  def setup_method(self):
+    self.meta = get_egpu_model()
+
+  def test_split_model_rejected(self):
+    split_meta = {**get_egpu_model(), "key": "some_split", "split": True}
+    with pytest.raises(EgpuPipelineError, match="split"):
+      EgpuPipeline(split_meta, _zero_infer(split_meta["output_len"]))
+
+  def test_registry_is_fused_only(self):
+    assert not any(m.get("split") for m in EGPU_MODELS.values())
+
+  def test_run_shapes_and_output(self):
+    infer = _zero_infer(self.meta["output_len"])
+    pipe = EgpuPipeline(self.meta, infer)
+    out = pipe.run(*_frame_inputs())
+    assert out.shape == (self.meta["output_len"],)
+    assert len(infer.calls) == 1
+
+  def test_hidden_state_feeds_next_features_buffer(self):
+    output_len = self.meta["output_len"]
+    hidden = self.meta["output_slices"]["hidden_state"]
+
+    def infer(inputs):
+      out = np.zeros(output_len, dtype=np.float32)
+      out[hidden] = np.arange(hidden.stop - hidden.start, dtype=np.float32)
+      return out
+
+    pipe = EgpuPipeline(self.meta, infer)
+    pipe.run(*_frame_inputs(1))
+    np.testing.assert_array_equal(
+      pipe.state.prev_feat.reshape(-1), np.arange(hidden.stop - hidden.start, dtype=np.float32))
+    pipe.run(*_frame_inputs(2))
+    np.testing.assert_array_equal(
+      pipe.state.feat_q[-1].reshape(-1), np.arange(hidden.stop - hidden.start, dtype=np.float32))
+
+  def test_desire_rising_edge_pulse(self):
+    infer = _zero_infer(self.meta["output_len"])
+    pipe = EgpuPipeline(self.meta, infer)
+    warped, _, traffic, action_t = _frame_inputs()
+    desire_on = np.zeros(8, dtype=np.float32)
+    desire_on[3] = 1.0
+    pipe.run(warped, desire_on, traffic, action_t)
+    assert infer.calls[-1]["desire_pulse"][0, -1, 3] == 1.0
+    for _ in range(5):
+      pipe.run(warped, desire_on, traffic, action_t)
+    assert infer.calls[-1]["desire_pulse"][0, :, 3].sum() == 1.0
+
+  def test_wrong_output_len_raises(self):
+    pipe = EgpuPipeline(self.meta, _zero_infer(self.meta["output_len"] - 1))
+    with pytest.raises(EgpuPipelineError, match="length"):
+      pipe.run(*_frame_inputs())
+
+  def test_non_finite_output_raises(self):
+    pipe = EgpuPipeline(self.meta, _zero_infer(self.meta["output_len"], fill=np.nan))
+    with pytest.raises(EgpuPipelineError, match="finite"):
+      pipe.run(*_frame_inputs())
+
+
+class TestChannelContract:
+  def _real_msgs(self):
+    import iqpilot.cereal.messaging as messaging
+    msgs = {}
+    for svc in ("modelV2", "drivingModelData", "cameraOdometry", "iqDriveModelData"):
+      m = messaging.new_message(svc)
+      msgs[svc] = m.to_bytes()
+    return msgs
+
+  def test_payload_keys_match_selector_contract(self):
+    payload = make_big_channel_payload(7, True, 0.031, 24.0, {"modelV2": b"x"})
+    assert payload["source"] == "egpu_big"
+    for key in ("frame_id", "live_calib_seen", "model_execution_time", "msgs"):
+      assert key in payload
+
+  def test_selector_consumes_egpu_payload(self, tmp_path):
+    from iqpilot.selfdrive.iqmodeld.model_channel import ModelChannel
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import wait_for_big
+
+    chan = ModelChannel(str(tmp_path / "big"), create=True)
+    payload = make_big_channel_payload(100, True, 0.03, 25.0, self._real_msgs())
+    chan.write(100, payload)
+
+    got, peek = wait_for_big(chan, 100, time.perf_counter() + 0.01)
+    assert peek == 100
+    assert got is not None
+    assert got["source"] == "egpu_big"
+    assert got["frame_id"] == 100
+
+  def test_selector_patch_and_send_parses_egpu_msgs(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import _patch_and_send
+
+    sent = {}
+
+    class PM:
+      def send(self, service, msg):
+        sent[service] = msg
+
+    payload = make_big_channel_payload(42, True, 0.03, 25.0, self._real_msgs())
+    _patch_and_send(PM(), payload, frame_drop_perc=0.0, selector_dropped=0, target=42, source_lag=0)
+    assert set(sent) == {"modelV2", "drivingModelData", "cameraOdometry", "iqDriveModelData"}
+    assert sent["modelV2"].modelV2.frameDropPerc == 0.0
+    assert sent["cameraOdometry"].valid
+
+  def test_selector_lag_patches_frame_id(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import _patch_and_send
+
+    sent = {}
+
+    class PM:
+      def send(self, service, msg):
+        sent[service] = msg
+
+    payload = make_big_channel_payload(40, True, 0.03, 25.0, self._real_msgs())
+    _patch_and_send(PM(), payload, frame_drop_perc=0.0, selector_dropped=0, target=42, source_lag=2)
+    assert sent["modelV2"].modelV2.frameId == 42
+    assert not sent["cameraOdometry"].valid
+
+
+def _import_worker():
+  try:
+    import iqpilot.selfdrive.iqmodeld.iqegpumodeld as w
+    return w
+  except ImportError as e:
+    if any(tag in str(e) for tag in ("pyx", "visionipc", "proprietary_runtime")):
+      pytest.skip(f"device-only import chain unavailable on this host: {e}")
+    raise
+
+
+class TestWorkerModule:
+  def test_module_imports_off_device(self):
+    w = _import_worker()
+    assert w.PROCESS_NAME.endswith("iqegpumodeld")
+    assert callable(w.main)
+
+  def test_warmup_validates_output(self):
+    w = _import_worker()
+    spec = {name: (shape, dtype) for name, (shape, dtype) in INPUT_SPEC.items()}
+    def good(inputs):
+      return np.zeros(10, dtype=np.float32)
+    assert w._warmup(good, spec, 10) >= 0.0
+    with pytest.raises(RuntimeError, match="invalid"):
+      w._warmup(good, spec, 11)
+
+
+class TestSelectorBackendKeys:
+  def test_emac_default(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import EMAC_STATUS_KEYS, backend_status_keys
+    assert backend_status_keys(False, False) is EMAC_STATUS_KEYS
+    assert backend_status_keys(True, False) is EMAC_STATUS_KEYS
+
+  def test_egpu_selected(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import EGPU_STATUS_KEYS, backend_status_keys
+    assert backend_status_keys(False, True) is EGPU_STATUS_KEYS
+    assert backend_status_keys(False, True)["active"] == "UsbGpuActive"
+    assert backend_status_keys(False, True)["failed"] == "UsbGpuFailed"
+
+  def test_emac_wins_when_both(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import EMAC_STATUS_KEYS, backend_status_keys
+    assert backend_status_keys(True, True) is EMAC_STATUS_KEYS
+
+  def test_key_maps_cover_same_roles(self):
+    from iqpilot.selfdrive.iqmodeld.modeld_selector import EGPU_STATUS_KEYS, EMAC_STATUS_KEYS
+    assert set(EGPU_STATUS_KEYS) == set(EMAC_STATUS_KEYS)
+
+
+class TestBackendSeparation:
+  EGPU_SOURCES = (
+    "egpu_helpers.py", "egpu_pipeline.py", "egpu_model.py", "iqegpumodeld.py",
+    "big_catalog.py", "tools/compile_egpu_model.py",
+  )
+  BANNED_IMPORTS = ("emac_input_state", "emac_model_meta", "maciqmodeld", "mac_protocol", "mac_client")
+
+  def _sources(self):
+    import pathlib
+    root = pathlib.Path(__file__).resolve().parents[1]
+    return {name: (root / name).read_text() for name in self.EGPU_SOURCES}
+
+  def test_no_emac_module_imports(self):
+    for name, src in self._sources().items():
+      for banned in self.BANNED_IMPORTS:
+        assert f"import {banned}" not in src and f"iqmodeld.{banned}" not in src, f"{name} imports {banned}"
+
+  def test_no_macmodel_params(self):
+    for name, src in self._sources().items():
+      assert "MacModel" not in src, f"{name} references MacModel* params"
+
+  def test_emac_shim_reexports_temporal_state(self):
+    from iqpilot.selfdrive.iqmodeld import emac_input_state, temporal_state
+    assert emac_input_state.EmacInputState is temporal_state.TemporalInputState
+    assert emac_input_state.SplitInputState is temporal_state.SplitTemporalState
+
+  def test_emac_modules_are_not_in_the_public_tree(self):
+    import pathlib
+    root = pathlib.Path(__file__).resolve().parents[1]
+    for gone in ("mac_protocol.py", "mac_client.py", "maciqmodeld.py", "bulk_transport.py"):
+      assert not (root / gone).exists(), f"{gone} must live only in konn3kt_private"
+
+
+class TestMetaDrivenInputSpec:
+
+  def _run_one(self, meta):
+    seen = {}
+    def infer(inputs):
+      seen.update({k: v.shape for k, v in inputs.items()})
+      return np.zeros(meta["output_len"], dtype=np.float32)
+    pipe = EgpuPipeline(meta, infer)
+    pipe.run(np.zeros((2, 6, 128, 256), np.uint8), np.zeros(8, np.float32),
+             np.array([1, 0], np.float32), np.zeros(2, np.float32))
+    return seen
+
+  def test_default_contract_unchanged(self):
+    meta = get_egpu_model()
+    seen = self._run_one(meta)
+    assert seen["features_buffer"] == (1, 24, 512)
+    assert seen["desire_pulse"] == (1, 25, 8)
+
+  def test_registry_shapes_drive_the_state(self):
+    meta = dict(get_egpu_model())
+    meta["output_len"] = 18452
+    meta["output_slices"] = dict(meta["output_slices"], hidden_state=slice(2066, 18450))
+    meta["input_shapes"] = {
+      "img": (1, 12, 128, 256), "big_img": (1, 12, 128, 256),
+      "desire_pulse": (1, 33, 8), "traffic_convention": (1, 2),
+      "action_t": (1, 2), "features_buffer": (1, 32, 32, 512),
+    }
+    seen = self._run_one(meta)
+    assert seen["features_buffer"] == (1, 32, 32, 512)
+    assert seen["desire_pulse"] == (1, 33, 8)
+
+
+class TestCatalogResolution:
+
+  def _params(self, model, doc=None):
+    class P:
+      def get(self, k):
+        if k == "IQEmacModel":
+          return model
+        if k == "IQEmacCatalogCache":
+          return json.dumps(doc) if doc else None
+        return None
+    return P()
+
+  def _doc(self):
+    return {"schema": 1, "bundles": [{
+      "short_name": "ttx", "display_name": "TTx", "index": 1,
+      "model_name": "big_driving_supercombo",
+      "wire": {"output_len": 2580, "frame_skip": 4, "pipeline": True,
+               "output_slices": {"plan": [917, 1907], "hidden_state": [2066, 2578], "pad": [-2, None]},
+               "input_shapes": {"img": [1, 12, 128, 256], "big_img": [1, 12, 128, 256],
+                                "desire_pulse": [1, 33, 8], "traffic_convention": [1, 2],
+                                "action_t": [1, 2], "features_buffer": [1, 32, 512]},
+               "lat_smooth_seconds": 0.1},
+      "source": {"kind": "comma_lfs", "sha256": "c" * 64, "size": 1},
+    }]}
+
+  def test_unset_selection_is_the_builtin_default(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+    m = resolve_egpu_model(self._params(None))
+    assert m["key"] == "lebrowski" and m["sha256"].startswith("a501760a")
+
+  def test_catalog_selection_resolves_with_shapes_and_smoothing(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+    m = resolve_egpu_model(self._params("ttx", self._doc()))
+    assert m["key"] == "ttx"
+    assert m["input_shapes"]["features_buffer"] == (1, 32, 512)
+    assert m["input_shapes"]["desire_pulse"] == (1, 33, 8)
+    assert m["lat_smooth_seconds"] == 0.1
+    assert m["output_slices"]["pad"] == slice(-2, None)
+
+  def test_unknown_selection_is_a_park_not_a_silent_default(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+    assert resolve_egpu_model(self._params("ghost", self._doc()), allow_refresh=False) is None
+
+  def test_bench_model_is_not_selectable(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+    assert resolve_egpu_model(self._params("comma_small", self._doc()), allow_refresh=False) is None
+
+  def test_registry_carries_no_model_list(self):
+    from iqpilot.selfdrive.iqmodeld.egpu_model import EGPU_MODELS
+    assert set(EGPU_MODELS) == {"lebrowski", "comma_small"}
+
+
+class TestConsentAndIntegrity:
+  def test_disabled_param_denies_present_dock(self, monkeypatch):
+    from iqpilot.selfdrive.iqmodeld import egpu_helpers
+    monkeypatch.setattr(egpu_helpers, "usbgpu_present", lambda sysfs_root=egpu_helpers.USB_SYSFS_ROOT: True)
+    assert egpu_helpers.egpu_present_consented(FakeParams()) is True
+    assert egpu_helpers.egpu_present_consented(FakeParams(IQEgpuDisabled=True)) is False
+
+  def test_local_onnx_quarantines_bad_content(self, tmp_path, monkeypatch):
+    import hashlib
+    from iqpilot.selfdrive.iqmodeld import egpu_helpers
+    onnx = tmp_path / "m.onnx"
+    onnx.write_bytes(b"good")
+    meta = {"sha256": hashlib.sha256(b"good").hexdigest(), "download": {"size": 4}}
+    monkeypatch.setattr(egpu_helpers, "onnx_cache_path", lambda m: str(onnx))
+    assert egpu_helpers.local_onnx(meta) == str(onnx)
+    onnx.write_bytes(b"bad!")
+    assert egpu_helpers.local_onnx(meta) is None
+    assert not onnx.exists()
+    assert (tmp_path / "m.onnx.unusable").exists()
+
+
+class TestEgpuDockStatus:
+  def _run(self, seq):
+    from iqpilot.system.hardware.egpu_dock.status import EgpuDockStatus
+    st = EgpuDockStatus()
+    fired = {}
+    def set_alert(name, cond, extra=None):
+      fired[name] = (bool(cond), extra)
+    for args in seq:
+      st.update(*args, set_alert)
+    return {k: v for k, v in fired.items() if v[0]}
+
+  def _dock(self, speed=10000, product="custom ed4e39b7-CLEAN"):
+    return [{"vendorId": 0xADD1, "productId": 0x0001, "product": product, "speedMbps": speed}]
+
+  def test_no_dock_no_alerts(self):
+    assert self._run([(True, [], False, False, None, True, None)]) == {}
+
+  def test_usb2_dock_warns_slow(self):
+    fired = self._run([(True, self._dock(speed=480), False, False, None, True, None)])
+    assert fired.get("Offroad_EgpuUsbSlow") == (True, "480 Mbps")
+
+  def test_power_fault_reports_pcie_unavailable(self):
+    class St:
+      supplyFault = True
+      supplyVoltage = 0
+      pcieLtssm = 0x78
+      tempC = memoryTempC = 40.0
+      fanSpeedRpm = 1500
+    d = self._dock()
+    fired = self._run([
+      (True, d, False, False, None, True, None),
+      (False, d, False, True, None, True, None),
+      (False, d, False, False, b"1", True, St()),
+    ])
+    assert "Offroad_EgpuPcieUnavailable" in fired

iqpilot/selfdrive/iqmodeld/tests/test_emac_input_state.py

@@ -0,0 +1,111 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import os
+
+import numpy as np
+import pytest
+
+os.environ.setdefault("DEV", "CPU")
+
+from iqpilot.selfdrive.iqmodeld.emac_input_state import EmacInputState
+from iqpilot.selfdrive.iqmodeld.emac_model_meta import FRAME_SKIP, OUTPUT_LEN, OUTPUT_SLICES
+from iqpilot.selfdrive.iqmodeld.temporal_state import MODEL_INPUT_SPEC as INPUT_SPEC
+
+N_FRAMES_TEST = 30
+IMG_SHAPE = INPUT_SPEC["img"][0]
+DESIRE_LEN = INPUT_SPEC["desire_pulse"][0][2]
+
+
+class _CaptureRunner:
+
+  def __init__(self):
+    self.captured: dict[str, np.ndarray] | None = None
+
+  def __call__(self, inputs):
+    from tinygrad import Tensor
+    self.captured = {k: v.numpy().copy() for k, v in inputs.items()}
+    return {"outputs": Tensor(np.zeros((1, OUTPUT_LEN), dtype=np.float32))}
+
+
+@pytest.fixture(scope="module")
+def reference():
+  from iqpilot.selfdrive.iqmodeld.tools.compile_supercombo import (
+    POLICY_INPUTS, make_input_queues, make_run_policy,
+  )
+
+  input_shapes = {name: shape for name, (shape, _) in INPUT_SPEC.items()}
+  metadata = {"input_shapes": input_shapes}
+  capture = _CaptureRunner()
+  run_policy = make_run_policy(capture, metadata, FRAME_SKIP)
+  queues, npy = make_input_queues(input_shapes, FRAME_SKIP, device="CPU")
+  return run_policy, queues, npy, capture, POLICY_INPUTS
+
+
+def _rising_edge(raw_desire: np.ndarray, prev: np.ndarray) -> np.ndarray:
+  cur = raw_desire.astype(np.float32).copy()
+  cur[0] = 0
+  pulse = np.where(cur - prev > 0.99, cur, 0).astype(np.float32)
+  prev[:] = cur
+  return pulse
+
+
+def test_materialized_inputs_match_tinygrad_reference(reference):
+  from tinygrad import Tensor
+
+  run_policy, queues, npy, capture, policy_inputs = reference
+  rng = np.random.default_rng(1234)
+  state = EmacInputState(FRAME_SKIP)
+  ref_prev_desire = np.zeros(DESIRE_LEN, dtype=np.float32)
+
+  hidden = np.zeros((1, 512), dtype=np.float32)
+  for frame in range(N_FRAMES_TEST):
+    warped = rng.integers(0, 256, (2, 6, IMG_SHAPE[2], IMG_SHAPE[3]), dtype=np.int64).astype(np.uint8)
+    raw_desire = np.zeros(DESIRE_LEN, dtype=np.float32)
+    if frame % 3:
+      raw_desire[int(rng.integers(0, DESIRE_LEN))] = 1.0
+    traffic = rng.standard_normal(2).astype(np.float32)
+    action_t = rng.standard_normal(2).astype(np.float32)
+
+    npy["desire"][:] = _rising_edge(raw_desire, ref_prev_desire)
+    npy["traffic_convention"][:] = traffic
+    npy["action_t"][:] = action_t
+    npy["prev_feat"][:] = hidden
+    run_policy(warped=Tensor(warped), **{k: queues[k] for k in policy_inputs})
+    ref_inputs = capture.captured
+
+    state.prev_feat[:] = hidden
+    mat = state.push_and_materialize(warped, raw_desire, traffic, action_t)
+
+    for name in INPUT_SPEC:
+      assert ref_inputs[name].shape == tuple(INPUT_SPEC[name][0]), name
+      np.testing.assert_array_equal(
+        mat[name].astype(ref_inputs[name].dtype), ref_inputs[name],
+        err_msg=f"frame {frame}: materialized {name} diverges from tinygrad reference")
+
+    fake_output = rng.standard_normal(OUTPUT_LEN).astype(np.float32)
+    state.note_hidden_state(fake_output, OUTPUT_SLICES["hidden_state"])
+    hidden = fake_output[OUTPUT_SLICES["hidden_state"]].reshape(1, 512).copy()
+
+
+def test_note_hidden_state_slice():
+  state = EmacInputState(FRAME_SKIP)
+  out = np.arange(OUTPUT_LEN, dtype=np.float32)
+  state.note_hidden_state(out, OUTPUT_SLICES["hidden_state"])
+  np.testing.assert_array_equal(state.prev_feat.reshape(-1), out[OUTPUT_SLICES["hidden_state"]])
+
+
+def test_desire_pulse_rising_edge_only_once():
+  state = EmacInputState(FRAME_SKIP)
+  held = np.zeros(DESIRE_LEN, dtype=np.float32)
+  held[3] = 1.0
+  warped = np.zeros((2, 6, IMG_SHAPE[2], IMG_SHAPE[3]), dtype=np.uint8)
+  zeros2 = np.zeros(2, dtype=np.float32)
+
+  first = state.push_and_materialize(warped, held, zeros2, zeros2)
+  assert first["desire_pulse"][0, -1, 3] == 1.0
+  second = state.push_and_materialize(warped, held, zeros2, zeros2)
+  assert state.desire_q[-1].max() == 0.0
+  assert second["desire_pulse"][0, -1, 3] == 1.0

iqpilot/selfdrive/iqmodeld/tests/test_lat_delay_source.py

@@ -34,6 +34,7 @@ def _daemon(params, steer_control_type):
   return SimpleNamespace(
     _params=params,
     _car_params=car_params,
+    _channel=None,
     _sub={"lateralDelay": SimpleNamespace(lateralDelay=LIVE_DELAY)},
     _runtime=SimpleNamespace(lat_delay=None, PLANPLUS_CONTROL=None, model_smoothing_max_extra_sec=None),
     _warps=SimpleNamespace(set_offset=lambda _: None),

iqpilot/selfdrive/iqmodeld/tests/test_selector_version_gate.py

@@ -0,0 +1,31 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+
+The eMac bundles ship `minimum_selector_version = 17`, and the version
+gate lives in the COMPILED private selector bundle, not in this repo. If that
+bundle is rebuilt from stale source the gate still reads 16, every eMac bundle
+is silently dropped as "too new", and the selector simply shows no eMac models
+— with no error anywhere. Assert the effective gate instead, so a stale
+private bundle fails here rather than on a device.
+"""
+from iqpilot.selfdrive.iqmodeld.emac_model_meta import EMAC_BUNDLE_MIN_SELECTOR_VERSION
+from iqpilot.selfdrive.iqmodeld.models.helpers import is_bundle_version_compatible
+
+
+def test_gate_accepts_the_version_our_emac_bundles_ship():
+  assert is_bundle_version_compatible({"minimumSelectorVersion": EMAC_BUNDLE_MIN_SELECTOR_VERSION}), (
+    f"the effective selector gate rejects minimumSelectorVersion="
+    f"{EMAC_BUNDLE_MIN_SELECTOR_VERSION}; the private selector bundle is stale. "
+    f"Rebuild it from BOTH iqpilot/models_private_src/helpers.py "
+    f"(CURRENT_SELECTOR_VERSION) and fetcher.py (MANIFEST_VERSION)."
+  )
+
+
+def test_gate_still_accepts_older_bundles():
+  # the window is a range, not a floor: bumping it must not orphan the existing catalogue
+  assert is_bundle_version_compatible({"minimumSelectorVersion": 12})
+  assert is_bundle_version_compatible({"minimumSelectorVersion": 16})
+
+
+def test_gate_rejects_a_bundle_from_the_future():
+  assert not is_bundle_version_compatible({"minimumSelectorVersion": EMAC_BUNDLE_MIN_SELECTOR_VERSION + 5})

iqpilot/selfdrive/iqmodeld/tests/test_split_input_state.py

@@ -0,0 +1,131 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+
+eMac split-model "prepared input equivalence": SplitInputState must reproduce,
+byte-exact, the queue semantics of compile_split_runtime's execute_bundle —
+the real tinygrad reference graph run on CPU with stub vision/policy runners,
+over a multi-frame random sequence with desire rising edges.
+"""
+from __future__ import annotations
+
+import os
+
+import numpy as np
+import pytest
+
+os.environ.setdefault("DEV", "CPU")
+
+from iqpilot.selfdrive.iqmodeld.emac_input_state import EmacInputState, SplitInputState
+
+N_FRAMES_TEST = 30
+FRAME_SKIP = 4
+IMG_SHAPE = (1, 12, 16, 32)  # small spatial dims: queue math is shape-generic
+FB_SHAPE = (1, 25, 512)
+DP_SHAPE = (1, 25, 8)
+VISION_OUT_LEN = 1576
+HIDDEN_SLICE = slice(1064, 1576)
+
+VISION_SHAPES = {"img": IMG_SHAPE, "big_img": IMG_SHAPE}
+POLICY_SHAPES = {"desire_pulse": DP_SHAPE, "traffic_convention": (1, 2), "features_buffer": FB_SHAPE}
+
+
+class _StubRunner:
+  """Stands in for OnnxRunner inside execute_bundle: returns a preset output
+  and records the materialized inputs it was fed."""
+
+  def __init__(self, out_len: int):
+    self.out_len = out_len
+    self.next_output: np.ndarray | None = None
+    self.captured: dict[str, np.ndarray] | None = None
+
+  def __call__(self, inputs):
+    from tinygrad import Tensor
+    self.captured = {k: v.numpy().copy() for k, v in inputs.items()}
+    out = self.next_output if self.next_output is not None else np.zeros((1, self.out_len), dtype=np.float32)
+    return {"outputs": Tensor(out.astype(np.float32))}
+
+
+@pytest.fixture(scope="module")
+def reference():
+  from tinygrad import Tensor
+  from iqpilot.selfdrive.iqmodeld.tools.compile_split_runtime import _role_executor
+
+  meta_by_role = {
+    "vision": {"input_shapes": dict(VISION_SHAPES), "output_slices": {"hidden_state": HIDDEN_SLICE}},
+    "policy": {"input_shapes": dict(POLICY_SHAPES), "output_slices": {}},
+  }
+  vision, policy = _StubRunner(VISION_OUT_LEN), _StubRunner(1000)
+  execute_bundle = _role_executor({"vision": vision, "policy": policy}, meta_by_role, FRAME_SKIP)
+
+  feat_q = Tensor(np.zeros((FRAME_SKIP * (FB_SHAPE[1] - 1) + 1, FB_SHAPE[0], FB_SHAPE[2]), dtype=np.float32),
+                  device="CPU").contiguous().realize()
+  desire_q = Tensor(np.zeros((FRAME_SKIP * DP_SHAPE[1], DP_SHAPE[0], DP_SHAPE[2]), dtype=np.float32),
+                    device="CPU").contiguous().realize()
+  return execute_bundle, feat_q, desire_q, vision, policy
+
+
+def test_split_inputs_match_tinygrad_reference(reference):
+  from tinygrad import Tensor
+
+  execute_bundle, feat_q, desire_q, vision_stub, policy_stub = reference
+  rng = np.random.default_rng(4321)
+  state = SplitInputState(FRAME_SKIP, IMG_SHAPE, FB_SHAPE, DP_SHAPE)
+  ref_prev_desire = np.zeros(DP_SHAPE[2], dtype=np.float32)
+
+  for frame in range(N_FRAMES_TEST):
+    warped = rng.integers(0, 256, (2, 6, IMG_SHAPE[2], IMG_SHAPE[3]), dtype=np.int64).astype(np.uint8)
+    raw_desire = np.zeros(DP_SHAPE[2], dtype=np.float32)
+    if frame % 3:
+      raw_desire[int(rng.integers(0, DP_SHAPE[2]))] = 1.0
+    traffic = rng.standard_normal((1, 2)).astype(np.float32)
+    vision_out = rng.standard_normal((1, VISION_OUT_LEN)).astype(np.float32)
+    vision_stub.next_output = vision_out
+
+    # --- ours ---
+    vis_inputs = state.materialize_vision(warped, raw_desire)
+    pol_inputs = state.materialize_policy(vision_out[0, HIDDEN_SLICE], traffic[0])
+
+    # --- reference graph: rising edge happens outside execute_bundle (run_fused) ---
+    cur = raw_desire.copy()
+    cur[0] = 0
+    ref_pulse = np.where(cur - ref_prev_desire > 0.99, cur, 0).astype(np.float32)
+    ref_prev_desire[:] = cur
+
+    execute_bundle(
+      img=Tensor(vis_inputs["img"], device="CPU").realize(),
+      big_img=Tensor(vis_inputs["big_img"], device="CPU").realize(),
+      feat_q=feat_q, desire_q=desire_q,
+      desire=Tensor(ref_pulse, device="CPU").realize(),
+      traffic_convention=Tensor(traffic, device="CPU").realize(),
+      action_t=Tensor(np.zeros((1, 2), dtype=np.float32), device="CPU").realize(),
+    )
+    ref = policy_stub.captured
+    assert ref is not None
+
+    assert ref["features_buffer"].tobytes() == pol_inputs["features_buffer"].tobytes(), f"features frame {frame}"
+    assert ref["desire_pulse"].tobytes() == pol_inputs["desire_pulse"].tobytes(), f"desire frame {frame}"
+    assert ref["traffic_convention"].tobytes() == pol_inputs["traffic_convention"].tobytes()
+    # vision saw exactly what our img queues materialized
+    vref = vision_stub.captured
+    assert vref["img"].tobytes() == vis_inputs["img"].tobytes(), f"img frame {frame}"
+    assert vref["big_img"].tobytes() == vis_inputs["big_img"].tobytes(), f"big_img frame {frame}"
+
+
+def test_split_img_queue_matches_fused_state():
+  # img/desire mechanics are shared with the fused mirror: same warps must
+  # materialize identical img/big_img in both states
+  rng = np.random.default_rng(7)
+  fused_spec = {
+    "img": (IMG_SHAPE, "uint8"), "big_img": (IMG_SHAPE, "uint8"),
+    "desire_pulse": (DP_SHAPE, "float32"), "traffic_convention": ((1, 2), "float32"),
+    "features_buffer": ((1, 24, 512), "float32"), "action_t": ((1, 2), "float32"),
+  }
+  fused = EmacInputState(FRAME_SKIP, fused_spec)
+  split = SplitInputState(FRAME_SKIP, IMG_SHAPE, FB_SHAPE, DP_SHAPE)
+  for _ in range(12):
+    warped = rng.integers(0, 256, (2, 6, IMG_SHAPE[2], IMG_SHAPE[3]), dtype=np.int64).astype(np.uint8)
+    desire = np.zeros(DP_SHAPE[2], dtype=np.float32)
+    f = fused.push_and_materialize(warped, desire, np.zeros(2, dtype=np.float32), np.zeros(2, dtype=np.float32))
+    s = split.materialize_vision(warped, desire)
+    assert f["img"].tobytes() == s["img"].tobytes()
+    assert f["big_img"].tobytes() == s["big_img"].tobytes()

iqpilot/selfdrive/iqmodeld/tools/compile_egpu_model.py

@@ -0,0 +1,144 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import pickle
+import time
+
+os.environ.setdefault("DEV", "USB+AMD:LLVM")
+os.environ.setdefault("FLOAT16", "1")
+os.environ.setdefault("JIT_BATCH_SIZE", "0")
+os.environ.setdefault("GMMU", "0")
+
+import numpy as np
+
+from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_pkl_path, local_onnx, patch_tinygrad_fetch_fw
+from iqpilot.selfdrive.iqmodeld.egpu_model import EGPU_MODELS, get_egpu_model, resolve_egpu_model
+from iqpilot.selfdrive.iqmodeld.temporal_state import MODEL_INPUT_SPEC, spec_from_meta
+
+INPUT_SPEC = dict(MODEL_INPUT_SPEC)
+
+patch_tinygrad_fetch_fw()
+
+SEED = 42
+
+
+def set_input_spec(meta: dict) -> None:
+  spec = spec_from_meta(meta)
+  if spec is not None:
+    INPUT_SPEC.clear()
+    INPUT_SPEC.update(spec)
+
+
+def make_run_model(model_runner):
+  def run_model(**inputs):
+    out = next(iter(model_runner({k: inputs[k] for k in INPUT_SPEC}).values())).cast("float32")
+    return out.reshape(-1),
+  return run_model
+
+
+def _random_inputs(seed: int):
+  from tinygrad.device import Device
+  from tinygrad.tensor import Tensor
+  rng = np.random.default_rng(seed)
+  out = {}
+  for name, (shape, dtype) in INPUT_SPEC.items():
+    if dtype == "uint8":
+      arr = rng.integers(0, 256, shape).astype(np.uint8)
+    else:
+      arr = rng.standard_normal(shape).astype(np.float32)
+    out[name] = Tensor(arr, device=Device.DEFAULT).realize()
+  return out
+
+
+def _run(fn, seed: int) -> np.ndarray:
+  from tinygrad.device import Device
+  st = time.perf_counter()
+  outs = fn(**_random_inputs(seed))
+  Device.default.synchronize()
+  print(f"  run(seed={seed}) {(time.perf_counter() - st) * 1e3:6.1f} ms")
+  return outs[0].numpy().reshape(-1)
+
+
+def compile_model(meta: dict, onnx_path: str, out_path: str) -> str:
+  from tinygrad.device import Device
+  from tinygrad.engine.jit import TinyJit
+  from tinygrad.nn.onnx import OnnxRunner
+
+  if meta.get("split"):
+    raise RuntimeError(f"model {meta['key']} is a split model; eGPU v1 compiles fused models only")
+
+  jit = TinyJit(make_run_model(OnnxRunner(onnx_path)), prune=True)
+
+  print("capture + replay")
+  for _ in range(2):
+    baseline = _run(jit, SEED)
+  if baseline.shape[0] != meta["output_len"]:
+    raise RuntimeError(f"model output length {baseline.shape[0]} != registry {meta['output_len']}")
+  if not np.isfinite(baseline).all():
+    raise RuntimeError("compiled model produced non-finite outputs")
+
+  print("pickle round trip")
+  jit = pickle.loads(pickle.dumps(jit))
+  if not np.array_equal(_run(jit, SEED), baseline):
+    raise RuntimeError("outputs differ from baseline after pickle round trip")
+  if np.array_equal(_run(jit, SEED + 1), baseline):
+    raise RuntimeError("outputs insensitive to inputs after pickle round trip")
+
+  from tinygrad.tensor import Tensor
+  zeros = {name: Tensor(np.zeros(shape, dtype=dtype), device=Device.DEFAULT).realize()
+           for name, (shape, dtype) in INPUT_SPEC.items()}
+  flat = jit(**zeros)[0].numpy().reshape(-1)
+  from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
+  from iqpilot.selfdrive.iqmodeld.tools.compile_supercombo import _slice_outputs, _validate_pose_outputs
+  _validate_pose_outputs(PhaseParser().parse_vision_outputs(_slice_outputs(flat, meta["output_slices"])))
+
+  bundle = {
+    "run_model": jit,
+    "model_key": meta["key"],
+    "model_sha256": meta["sha256"],
+    "output_len": int(meta["output_len"]),
+    "frame_skip": int(meta["frame_skip"]),
+    "input_spec": {name: (tuple(shape), dtype) for name, (shape, dtype) in INPUT_SPEC.items()},
+    "input_device": Device.DEFAULT,
+  }
+  os.makedirs(os.path.dirname(out_path), exist_ok=True)
+  tmp = out_path + ".part"
+  with open(tmp, "wb") as f:
+    pickle.dump(bundle, f, protocol=pickle.HIGHEST_PROTOCOL)
+  os.replace(tmp, out_path)
+  return out_path
+
+
+def main() -> None:
+  p = argparse.ArgumentParser()
+  p.add_argument("--model", default=None, help=f"registry key, one of {sorted(EGPU_MODELS)}")
+  p.add_argument("--onnx", default=None)
+  p.add_argument("--output", default=None)
+  args = p.parse_args()
+
+  if args.model is not None:
+    if args.model in EGPU_MODELS:
+      meta = get_egpu_model(args.model)
+    else:
+      from iqpilot.common.params import Params
+      meta = resolve_egpu_model(Params(), args.model)
+      if meta is None:
+        raise SystemExit(f"unknown model {args.model!r}: not a built-in ({sorted(EGPU_MODELS)}) and not in the synced catalog")
+  else:
+    meta = get_egpu_model()
+  set_input_spec(meta)
+
+  onnx_path = args.onnx or local_onnx(meta)
+  if onnx_path is None or not os.path.isfile(onnx_path):
+    raise SystemExit(f"onnx not found for {meta['key']}; pass --onnx or let iqegpumodeld download it first")
+
+  out = compile_model(meta, onnx_path, args.output or egpu_pkl_path(meta))
+  print(f"saved eGPU jit to {out} ({os.path.getsize(out) / 1e6:.2f} MB)")
+
+
+if __name__ == "__main__":
+  main()

iqpilot/selfdrive/iqmodeld/tools/compile_emac_warp.py

@@ -0,0 +1,9 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from iqpilot.selfdrive.iqmodeld.tools.compile_warp import MODEL_SIZE, compile_warp, main
+
+__all__ = ["MODEL_SIZE", "compile_warp", "main"]
+
+if __name__ == "__main__":
+  main()

iqpilot/selfdrive/iqmodeld/tools/compile_warp.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+
+Compile the backend-neutral warp-only artifact: NV12 camera frames + 3x3
+transforms -> (2, 6, model_h/2, model_w/2) uint8 warped tensor, on the device
+GPU (QCOM). maciqmodeld runs this locally
+and feed the output to their backend, so the big model's image pipeline is
+bit-identical to comma's fused pkl warp stage.
+
+Run ON the device (needs the QCOM backend):
+  cd /data/openpilot && DEV=QCOM WARP_DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 \
+    python3 iqpilot/selfdrive/iqmodeld/tools/compile_warp.py \
+    --camera-resolutions 1928x1208 --output /data/models/emac_warp.pkl
+The artifact is then split per-resolution into Paths.model_root().
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import pickle
+from functools import partial
+
+import numpy as np
+
+SELFTEST_SEED = 20260817
+
+from iqpilot.selfdrive.iqmodeld.temporal_state import DEFAULT_FRAME_SKIP, MODEL_INPUT_SPEC
+from iqpilot.selfdrive.iqmodeld.tools.compile_supercombo import (
+  NV12Frame, WARP_INPUTS, compile_jit, make_random_images, make_warp, make_warp_input_queues,
+)
+
+MODEL_SIZE = (MODEL_INPUT_SPEC["img"][0][3] * 2, MODEL_INPUT_SPEC["img"][0][2] * 2)  # (512, 256)
+
+
+def _parse_size(s: str) -> tuple[int, int]:
+  w, h = s.lower().split("x")
+  return int(w), int(h)
+
+
+def compile_warp(cam_w: int, cam_h: int, out_path: str | None = None,
+                 frame_skip: int = DEFAULT_FRAME_SKIP) -> str:
+  """Compile the warp-only QCOM JIT for one camera resolution and write the pkl.
+  Returns the artifact path. Callable from the workers so a fresh device
+  self-provisions the warp instead of erroring — needs the QCOM backend."""
+  # the QCOM warp env must be set before tinygrad is imported here
+  os.environ.setdefault("DEV", "QCOM")
+  os.environ.setdefault("WARP_DEV", "QCOM")
+  os.environ.setdefault("IMAGE", "1")
+  os.environ.setdefault("FLOAT16", "1")
+  os.environ.setdefault("NOLOCALS", "1")
+  os.environ.setdefault("JIT_BATCH_SIZE", "0")
+  from tinygrad.engine.jit import TinyJit
+  from iqpilot.system.camerad.cameras.nv12_info import get_nv12_info
+  from iqpilot.system.hardware.hw import Paths
+
+  model_w, model_h = MODEL_SIZE
+  input_shapes = {name: shape for name, (shape, _) in MODEL_INPUT_SPEC.items()}
+  nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
+  make_random_warp_inputs = partial(make_random_images, keys=["frame", "big_frame"],
+                                    shape=nv12.size, device=os.getenv("WARP_DEV"))
+  warp_jit = TinyJit(make_warp(nv12, model_w, model_h, frame_skip), prune=True)
+  make_warp_queues = partial(make_warp_input_queues, input_shapes, frame_skip)
+  compiled = compile_jit(warp_jit, make_random_warp_inputs, WARP_INPUTS, make_warp_queues)
+
+  # historical artifact name: already-provisioned devices keep their warp
+  out_path = out_path or os.path.join(Paths.model_root(), f"emac_warp_{cam_w}x{cam_h}_tinygrad.pkl")
+  os.makedirs(os.path.dirname(out_path), exist_ok=True)
+  tmp = out_path + ".part"
+  bundle = {(cam_w, cam_h): compiled, "frame_skip": frame_skip, "model_size": MODEL_SIZE}
+  bundle["selftest"] = selftest_digest(compiled, cam_w, cam_h, nv12.size)
+  with open(tmp, "wb") as f:
+    pickle.dump(bundle, f)
+  os.replace(tmp, out_path)  # atomic: a reader never sees a half-written pkl
+  return out_path
+
+
+def main() -> None:
+  p = argparse.ArgumentParser()
+  p.add_argument("--camera-resolutions", type=_parse_size, nargs="+", default=[(1928, 1208)])
+  p.add_argument("--output", default=None)
+  p.add_argument("--frame-skip", type=int, default=DEFAULT_FRAME_SKIP)
+  args = p.parse_args()
+  for cam_w, cam_h in args.camera_resolutions:
+    out = compile_warp(cam_w, cam_h, args.output, frame_skip=args.frame_skip)
+    print(f"saved warp JIT to {out} ({os.path.getsize(out) / 1e6:.2f} MB)")
+
+
+if __name__ == "__main__":
+  main()
+
+
+def selftest_inputs(cam_w: int, cam_h: int, nv12_size: int):
+  """A fixed synthetic frame pair and pair of matrices. Deterministic so the
+  digest is reproducible on the device that compiled the artifact."""
+  rng = np.random.default_rng(SELFTEST_SEED)
+  frame = rng.integers(0, 256, nv12_size, dtype=np.uint8)
+  big_frame = rng.integers(0, 256, nv12_size, dtype=np.uint8)
+  tfm = np.array([[0.7, 0.02, 300.0], [0.01, 0.7, 240.0], [0.0, 0.0, 1.0]], dtype=np.float32)
+  big_tfm = np.array([[0.5, 0.01, 380.0], [0.02, 0.5, 300.0], [0.0, 0.0, 1.0]], dtype=np.float32)
+  return frame, big_frame, tfm, big_tfm
+
+
+def selftest_digest(compiled, cam_w: int, cam_h: int, nv12_size: int) -> str:
+  """Hash the warp's output for a fixed input.
+
+  A warp artifact pinned to one tinygrad can still unpickle under another and
+  then compute silently wrong, which reaches the model as a garbage image and
+  looks like a bad model rather than a stale artifact. A version string cannot
+  see that; running it can."""
+  from tinygrad.tensor import Tensor
+  frame, big_frame, tfm, big_tfm = selftest_inputs(cam_w, cam_h, nv12_size)
+  dev = os.getenv("WARP_DEV") or "QCOM"
+  out = compiled(tfm=Tensor(tfm, device="NPY").realize(),
+                 big_tfm=Tensor(big_tfm, device="NPY").realize(),
+                 frame=Tensor(frame, device=dev).realize(),
+                 big_frame=Tensor(big_frame, device=dev).realize())
+  return hashlib.sha256(out.numpy().astype(np.uint8).tobytes()).hexdigest()