IQ.Pilot Release Commit @ 461be14

iqpilot/selfdrive/iqmodeld/egpu_helpers.py

@@ -6,12 +6,16 @@ from __future__ import annotations
 import hashlib
 import json
 import os
+
+from iqpilot.common.swaglog import cloudlog
 import urllib.request
 from pathlib import Path
 
 from iqpilot.system.hardware.usb import egpu_dock_ready
 
 USB_SYSFS_ROOT = "/sys/bus/usb/devices"
+FIRMWARE_MIRROR = os.getenv("IQ_EGPU_FIRMWARE_MIRROR", "/data/firmware/tinygrad")
+TINYGRAD_CACHE = "/data/.cache"
 
 COMMA_LFS_BATCH_URL = "https://gitlab.com/commaai/openpilot-lfs.git/info/lfs/objects/batch"
 
@@ -62,6 +66,11 @@ def egpu_policy_pkl_path(meta: dict) -> str:
   return os.path.join(Paths.model_root(), f"egpu_{meta['key']}_{meta['sha256'][:8]}_amd_policy.pkl")
 
 
+def egpu_oob_pkl_path(meta: dict) -> str:
+  from iqpilot.system.hardware.hw import Paths
+  return os.path.join(Paths.model_root(), f"egpu_{meta['key']}_{meta['sha256'][:8]}_amd_policy_oob.pkl")
+
+
 def onnx_cache_path(meta: dict) -> str:
   from iqpilot.system.hardware.hw import Paths
   return os.path.join(Paths.model_root(), f"{meta['model_name']}_{meta['sha256'][:8]}.onnx")
@@ -119,9 +128,14 @@ def download_onnx(meta: dict, progress_cb=None) -> str:
   if not download_url:
     raise RuntimeError(f"model {meta['key']} has no download source; stage the onnx at {onnx_cache_path(meta)}")
 
-  url = resolve_download_url(download_url, meta["sha256"], size)
   path = onnx_cache_path(meta)
   os.makedirs(os.path.dirname(path), exist_ok=True)
+  try:
+    from iqpilot.selfdrive.iqmodeld.model_bundle_downloader import download_hf_file
+    return download_hf_file(f"onnx/{meta['sha256']}.onnx", path, meta["sha256"], int(size or 0), progress_cb=progress_cb)
+  except Exception as e:
+    cloudlog.warning(f"onnx {meta['key']} unavailable from HF ({e}); falling back to {download_url.split(':', 1)[0]}")
+  url = resolve_download_url(download_url, meta["sha256"], size)
   tmp = path + ".part"
   digest = hashlib.sha256()
   got = 0
@@ -142,12 +156,20 @@ def download_onnx(meta: dict, progress_cb=None) -> str:
   return path
 
 
-def download_precompiled(meta: dict, progress_cb=None, policy: bool = False) -> str | None:
-  art = meta.get("egpu_policy_artifact" if policy else "egpu_artifact")
-  if not art or not art.get("objects"):
+def download_precompiled(meta: dict, progress_cb=None, policy: bool = False, oob: bool = False) -> str | None:
+  field = "egpu_oob_artifact" if oob else "egpu_policy_artifact" if policy else "egpu_artifact"
+  art = meta.get(field)
+  if not art or not (art.get("objects") or art.get("hf_path")):
     return None
-  from iqpilot.selfdrive.iqmodeld.model_bundle_downloader import download_lfs_bundle
-  dest = egpu_policy_pkl_path(meta) if policy else egpu_pkl_path(meta)
+  from iqpilot.selfdrive.iqmodeld.model_bundle_downloader import download_hf_file, download_lfs_bundle
+  dest = egpu_oob_pkl_path(meta) if oob else egpu_policy_pkl_path(meta) if policy else egpu_pkl_path(meta)
+  if art.get("hf_path"):
+    try:
+      return download_hf_file(art["hf_path"], dest, art["sha256"], int(art.get("size", 0)), progress_cb=progress_cb)
+    except Exception as e:
+      cloudlog.warning(f"precompiled {meta['key']} unavailable from HF ({e}); trying LFS")
+      if not art.get("objects"):
+        raise
   return download_lfs_bundle(art["objects"], dest, art["sha256"], int(art.get("size", 0)), progress_cb=progress_cb)
 
 
@@ -161,12 +183,27 @@ def patch_tinygrad_fetch_fw() -> None:
   _orig = helpers.fetch_fw
 
   def fetch_fw(path, name, sha256):
+    mirror = pathlib.Path(FIRMWARE_MIRROR) / path / name
+    if mirror.is_file():
+      blob = mirror.read_bytes()
+      if hashlib.sha256(blob).hexdigest() == sha256:
+        return blob
     p = pathlib.Path(f"/lib/firmware/{path}/{name}.zst")
     if p.is_file():
       blob = zstandard.ZstdDecompressor().stream_reader(p.read_bytes()).read()
       if hashlib.sha256(blob).hexdigest() == sha256:
         return blob
-    return _orig(path, name, sha256)
+    blob = _orig(path, name, sha256)
+    # The dock's GPU firmware otherwise lives only in tinygrad's per-user download cache, which is
+    # a network fetch the first time a new HOME sees it; onroad the car is usually offline.
+    try:
+      mirror.parent.mkdir(parents=True, exist_ok=True)
+      tmp = mirror.with_suffix(mirror.suffix + ".part")
+      tmp.write_bytes(blob)
+      os.replace(tmp, mirror)
+    except OSError:
+      pass
+    return blob
 
   fetch_fw._iq_patched = True
   helpers.fetch_fw = fetch_fw

iqpilot/selfdrive/iqmodeld/egpu_policy.py

@@ -3,11 +3,18 @@ Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed
 """
 from __future__ import annotations
 
+import io
 import math
+import os
+import pickle
+import shutil
+import struct
+import tempfile
 
 import numpy as np
 
 POLICY_FORMAT = 2
+OOB_MAGIC = b"IQEGPUOOB1"
 QUEUE_NAMES = ("img_q", "big_img_q", "feat_q", "desire_q")
 PACKED_ORDER = ("desire", "traffic_convention", "action_t", "prev_feat")
 
@@ -115,3 +122,50 @@ class PolicyRunner:
     flat = out.numpy().reshape(-1)
     v["prev_feat"][:] = flat[self._hidden].reshape(v["prev_feat"].shape)
     return flat
+
+
+def dump_oob(obj, f) -> None:
+  # Out-of-band pickle buffers keep the host peak at one tensor while the weights stream to the
+  # dock; a plain pickle keeps every weight referenced in the memo until load() returns (~1.7GB).
+  f.write(OOB_MAGIC)
+  with tempfile.TemporaryFile(dir=os.path.dirname(os.path.abspath(f.name)) or ".") as tmp:
+    def buffer_callback(pb: pickle.PickleBuffer):
+      m = pb.raw()
+      tmp.write(struct.pack("<q", m.nbytes))
+      tmp.write(m)
+      pb.release()
+    stream = io.BytesIO()
+    pickle.Pickler(stream, protocol=5, buffer_callback=buffer_callback).dump(obj)
+    opcodes = stream.getvalue()
+    f.write(struct.pack("<q", len(opcodes)))
+    f.write(opcodes)
+    tmp.seek(0)
+    shutil.copyfileobj(tmp, f)
+
+
+def is_oob(path: str) -> bool:
+  with open(path, "rb") as f:
+    return f.read(len(OOB_MAGIC)) == OOB_MAGIC
+
+
+def load_oob(f):
+  if f.read(len(OOB_MAGIC)) != OOB_MAGIC:
+    raise ValueError("not an out-of-band bundle")
+  opcodes = f.read(struct.unpack("<q", f.read(8))[0])
+
+  def buffers():
+    while (h := f.read(8)):
+      pb = pickle.PickleBuffer(bytearray(struct.unpack("<q", h)[0]))
+      f.readinto(pb)
+      yield pb
+
+  return pickle.load(io.BytesIO(opcodes), buffers=buffers())
+
+
+def load_bundle(path: str):
+  with open(path, "rb") as f:
+    if f.read(len(OOB_MAGIC)) == OOB_MAGIC:
+      f.seek(0)
+      return load_oob(f)
+    f.seek(0)
+    return pickle.load(f)

iqpilot/selfdrive/iqmodeld/egpu_prefetch.py

@@ -0,0 +1,99 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import glob
+import os
+os.environ.setdefault("XDG_CACHE_HOME", "/data/.cache")
+import time
+
+from iqpilot.common.params import Params
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.selfdrive.iqmodeld.egpu_helpers import download_precompiled, egpu_policy_pkl_path, egpu_selected, patch_tinygrad_fetch_fw, usbgpu_present
+from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
+
+POLL_S = 30.0
+RETRY_S = 120.0
+
+
+def _selected_meta(params: Params) -> dict | None:
+  key = params.get("IQEmacModel", encoding="utf8")
+  try:
+    return resolve_egpu_model(params, key)
+  except Exception as e:
+    cloudlog.warning(f"egpu_prefetch cannot resolve {key!r}: {e}")
+    return None
+
+
+def _drop_stale_partials(keep: str) -> None:
+  root = os.path.dirname(keep)
+  for path in glob.glob(os.path.join(root, "egpu_*_amd_policy.pkl.part")) + glob.glob(os.path.join(root, "big_driving_supercombo_*.onnx.part")):
+    if not path.startswith(keep):
+      try:
+        os.remove(path)
+      except OSError:
+        pass
+
+
+def prefetch_once(params: Params) -> bool:
+  if not usbgpu_present() or not egpu_selected(params):
+    return False
+  meta = _selected_meta(params)
+  if meta is None:
+    return False
+  dst = egpu_policy_pkl_path(meta)
+  if os.path.isfile(dst):
+    return True
+  if not meta.get("egpu_policy_artifact"):
+    return False
+  _drop_stale_partials(dst)
+  params.put("UsbGpuSetupProgress", "0.0")
+  last = [-1.0]
+
+  def _prog(p: float) -> None:
+    if p - last[0] >= 0.02 or p >= 1.0:
+      last[0] = p
+      params.put("UsbGpuSetupProgress", f"{p:.3f}")
+
+  cloudlog.warning(f"egpu_prefetch downloading {meta['key']} policy artifact offroad")
+  out = download_precompiled(meta, progress_cb=_prog, policy=True)
+  cloudlog.warning(f"egpu_prefetch ready -> {out}")
+  return out is not None
+
+
+_firmware_warm = False
+
+
+def warm_firmware() -> None:
+  global _firmware_warm
+  if _firmware_warm or not usbgpu_present():
+    return
+  os.environ.setdefault("DEV", "USB+AMD:LLVM")
+  os.environ.setdefault("GMMU", "0")
+  patch_tinygrad_fetch_fw()
+  from tinygrad.device import Device
+  Device["AMD"]
+  _firmware_warm = True
+  cloudlog.warning("egpu_prefetch: dock opened offroad; firmware cached and mirrored")
+
+
+def main() -> None:
+  params = Params()
+  while True:
+    try:
+      warm_firmware()
+    except Exception as e:
+      cloudlog.warning(f"egpu_prefetch firmware warm failed: {e}")
+    try:
+      prefetch_once(params)
+    except Exception as e:
+      cloudlog.warning(f"egpu_prefetch failed: {e}")
+      params.put("UsbGpuLastError", str(e)[:512])
+      time.sleep(RETRY_S)
+      continue
+    time.sleep(POLL_S)
+
+
+if __name__ == "__main__":
+  main()

iqpilot/selfdrive/iqmodeld/iqegpumodeld.py

@@ -4,6 +4,7 @@ Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed
 from __future__ import annotations
 
 import os
+os.environ.setdefault("XDG_CACHE_HOME", "/data/.cache")
 import pickle
 import subprocess
 import sys
@@ -38,7 +39,7 @@ from iqpilot.selfdrive.iqmodeld.driving_action import (
   DESIRE_LEN, LAT_SMOOTH_SECONDS, LONG_SMOOTH_SECONDS, get_action_from_model,
 )
 from iqpilot.selfdrive.iqmodeld.egpu_helpers import (
-  download_onnx, download_precompiled, egpu_pkl_path, egpu_policy_pkl_path, egpu_present_consented, egpu_selected, local_onnx,
+  download_onnx, download_precompiled, egpu_oob_pkl_path, egpu_pkl_path, egpu_policy_pkl_path, egpu_present_consented, egpu_selected, local_onnx,
   patch_tinygrad_fetch_fw, quarantine_artifact, resolve_backend, usbgpu_present,
 )
 from iqpilot.selfdrive.iqmodeld.egpu_model import resolve_egpu_model
@@ -47,7 +48,7 @@ from iqpilot.selfdrive.iqmodeld.egpu_telemetry import EgpuDockTelemetry
 from iqpilot.selfdrive.iqmodeld.messaging import DrivePacketMemory, populate_drive_messages, populate_odometry_message
 from iqpilot.selfdrive.iqmodeld.metadata import Meta20hz
 from iqpilot.selfdrive.iqmodeld.model_channel import BIG_CHANNEL, ModelChannel
-from iqpilot.selfdrive.iqmodeld.egpu_policy import POLICY_FORMAT, PolicyRunner
+from iqpilot.selfdrive.iqmodeld.egpu_policy import POLICY_FORMAT, PolicyRunner, load_bundle
 from iqpilot.selfdrive.iqmodeld.model_warp import FrameWarp
 from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
 
@@ -56,6 +57,9 @@ PROCESS_NAME = "iqpilot.selfdrive.iqmodeld.iqegpumodeld"
 PRESENCE_POLL_S = 5.0
 COMPILE_TIMEOUT_S = 3600
 LINK_UP_TIMEOUT_S = 10.0
+SETUP_EXIT_AFTER = 3
+MIN_LOAD_AVAIL_MB = 350
+MEMORY_WAIT_S = 90.0
 SETUP_RETRY_BASE_S = 3.0
 SETUP_RETRY_MAX_S = 30.0
 
@@ -106,14 +110,38 @@ _precompiled_tried = False
 
 def _ensure_artifact(params: Params, meta: dict) -> str:
   global _precompiled_tried
+  oob_path = egpu_oob_pkl_path(meta)
+  if os.path.isfile(oob_path):
+    return oob_path
   policy_path = egpu_policy_pkl_path(meta)
-  if os.path.isfile(policy_path):
-    return policy_path
   legacy_path = egpu_pkl_path(meta)
 
   params.put_bool("UsbGpuCompiled", False)
   params.put_bool("UsbGpuReady", False)
 
+  if meta.get("egpu_oob_artifact") and not _precompiled_tried:
+    _precompiled_tried = True
+    params.put("UsbGpuSetupProgress", "0.0")
+    oob_last = [-1.0]
+
+    def _oob_prog(p: float) -> None:
+      if p - oob_last[0] >= 0.02 or p >= 1.0:
+        oob_last[0] = p
+        params.put("UsbGpuSetupProgress", f"{p:.3f}")
+
+    try:
+      cloudlog.warning(f"iqegpumodeld downloading precompiled {meta['key']} (streamable) "
+                       f"({int(meta['egpu_oob_artifact'].get('size', 0)) / 1e6:.0f}MB)")
+      precompiled = download_precompiled(meta, progress_cb=_oob_prog, oob=True)
+      if precompiled is not None:
+        cloudlog.warning(f"iqegpumodeld precompiled ready -> {precompiled}")
+        return precompiled
+    except Exception as e:
+      cloudlog.warning(f"iqegpumodeld streamable artifact unavailable ({e}); falling back")
+
+  if os.path.isfile(policy_path):
+    return policy_path
+
   if meta.get("egpu_policy_artifact") and not _precompiled_tried:
     _precompiled_tried = True
     params.put("UsbGpuSetupProgress", "0.0")
@@ -157,12 +185,34 @@ def _ensure_artifact(params: Params, meta: dict) -> str:
   return policy_path
 
 
+def _mem_available_mb() -> int:
+  try:
+    with open("/proc/meminfo") as f:
+      for line in f:
+        if line.startswith("MemAvailable:"):
+          return int(line.split()[1]) // 1024
+  except OSError:
+    pass
+  return 1 << 20
+
+
+def _wait_for_memory(need_mb: int) -> None:
+  deadline = time.monotonic() + MEMORY_WAIT_S
+  avail = _mem_available_mb()
+  while avail < need_mb and time.monotonic() < deadline:
+    cloudlog.warning(f"iqegpumodeld waiting for memory: {avail}MB available, need {need_mb}MB")
+    time.sleep(5.0)
+    avail = _mem_available_mb()
+  if avail < need_mb:
+    raise RuntimeError(f"insufficient memory to load the dock model: {avail}MB available, need {need_mb}MB")
+
+
 def _load_infer_fn(pkl_path: str, meta: dict):
   patch_tinygrad_fetch_fw()
   from tinygrad.tensor import Tensor
 
-  with open(pkl_path, "rb") as f:
-    bundle = pickle.load(f)
+  _wait_for_memory(MIN_LOAD_AVAIL_MB)
+  bundle = load_bundle(pkl_path)
   if bundle.get("model_sha256") != meta["sha256"]:
     quarantine_artifact(pkl_path, "pkl model sha mismatch")
     raise RuntimeError(f"artifact model sha {bundle.get('model_sha256')} != {meta['sha256']}")
@@ -241,8 +291,14 @@ def main(demo: bool = False) -> None:
       break
     except Exception as e:
       attempt += 1
-      params.put("UsbGpuLastError", str(e)[:512])
-      cloudlog.warning(f"iqegpumodeld setup attempt {attempt} failed: {e}; retrying")
+      subs = "; ".join(f"{type(x).__name__}: {x}" for x in (getattr(e, "exceptions", None) or []))
+      params.put("UsbGpuLastError", (f"{e} [{subs}]" if subs else str(e))[:512])
+      cloudlog.warning(f"iqegpumodeld setup attempt {attempt} failed: {e}; {subs}; retrying")
+      if attempt >= SETUP_EXIT_AFTER:
+        # tinygrad keeps the dock's flock in a failed device init, so a stale process can never
+        # reopen it; exit and let the manager respawn a clean one.
+        cloudlog.error(f"iqegpumodeld giving up after {attempt} setup failures; exiting for a clean restart")
+        sys.exit(1)
       if not usbgpu_present():
         _wait_for_egpu(params)
       time.sleep(min(SETUP_RETRY_MAX_S, SETUP_RETRY_BASE_S * attempt))

iqpilot/selfdrive/iqmodeld/model_bundle_downloader.py

@@ -27,6 +27,63 @@ def _requests_auth():
   return None
 
 
+def _hf():
+  import importlib
+  for mod in ("iqpilot_private.models.git_auth", "iqpilot.selfdrive.iqmodeld.models.git_auth"):
+    try:
+      m = importlib.import_module(mod)
+      return m.get_hf_headers(), m.hf_resolve_url
+    except Exception:
+      continue
+  return None, None
+
+
+def download_hf_file(hf_path: str, dst: str, sha256: str, size: int, progress_cb=None) -> str:
+  import requests
+  headers, resolve = _hf()
+  if resolve is None:
+    raise RuntimeError("no HF credentials available")
+  url = resolve(hf_path)
+  os.makedirs(os.path.dirname(dst), exist_ok=True)
+  tmp = dst + ".hfpart"
+  last_error: Exception | None = None
+  for _attempt in range(STREAM_RETRIES):
+    try:
+      have = os.path.getsize(tmp) if os.path.isfile(tmp) else 0
+      if size and have > size:
+        os.remove(tmp)
+        have = 0
+      if not size or have < size:
+        req_headers = dict(headers)
+        if have:
+          req_headers["Range"] = f"bytes={have}-"
+        with requests.get(url, headers=req_headers, stream=True, timeout=HTTP_TIMEOUT_S, allow_redirects=True) as r:
+          r.raise_for_status()
+          if have and r.status_code != 206:
+            have = 0
+          with open(tmp, "ab" if have else "wb") as f:
+            got = have
+            for chunk in r.iter_content(CHUNK):
+              f.write(chunk)
+              got += len(chunk)
+              if progress_cb is not None and size:
+                progress_cb(min(1.0, got / size))
+      digest = hashlib.sha256()
+      with open(tmp, "rb") as f:
+        for chunk in iter(lambda: f.read(CHUNK), b""):
+          digest.update(chunk)
+      if size and os.path.getsize(tmp) != size:
+        raise RuntimeError(f"size mismatch: {os.path.getsize(tmp)}/{size} bytes")
+      if sha256 and digest.hexdigest() != sha256:
+        os.remove(tmp)
+        raise RuntimeError("sha256 mismatch")
+      os.replace(tmp, dst)
+      return dst
+    except Exception as e:
+      last_error = e
+  raise RuntimeError(f"HF download failed: {last_error}")
+
+
 def _lfs_endpoint(base_url: str) -> str:
   return base_url.split("/raw/", 1)[0] + ".git/info/lfs"
 
@@ -47,41 +104,108 @@ def _resolve_oid(session, base_url: str, oid: str, size: int, auth):
   return action["href"], action.get("header", {})
 
 
+def _part_path(dst: str, oid: str) -> str:
+  return os.path.join(dst + ".parts", oid)
+
+
+def _part_complete(path: str, oid: str, size: int) -> bool:
+  if not os.path.isfile(path) or os.path.getsize(path) != size:
+    return False
+  digest = hashlib.sha256()
+  with open(path, "rb") as f:
+    for chunk in iter(lambda: f.read(CHUNK), b""):
+      digest.update(chunk)
+  return digest.hexdigest() == oid
+
+
+def _fetch_part(session, base_url: str, obj: dict, path: str, auth, progress) -> None:
+  size = int(obj["size"])
+  have = os.path.getsize(path) if os.path.isfile(path) else 0
+  if have > size:
+    os.remove(path)
+    have = 0
+  href, headers = _resolve_oid(session, base_url, obj["oid"], size, auth)
+  obj_auth = None if headers.get("Authorization") else auth
+  # LFS parts are content-addressed (oid == sha256), so a half-written part can be resumed with a
+  # Range request and verified afterwards instead of being thrown away on every restart.
+  if have:
+    headers = {**headers, "Range": f"bytes={have}-"}
+  with session.get(href, headers=headers, stream=True, timeout=HTTP_TIMEOUT_S, auth=obj_auth) as r:
+    r.raise_for_status()
+    if have and r.status_code != 206:
+      have = 0
+    with open(path, "ab" if have else "wb") as f:
+      for chunk in r.iter_content(CHUNK):
+        f.write(chunk)
+        progress(len(chunk))
+
+
 def download_lfs_bundle(objects: list, dst: str, sha256: str, size: int, progress_cb=None) -> str:
   import requests
   auth = _requests_auth()
   session = requests.Session()
-  os.makedirs(os.path.dirname(dst), exist_ok=True)
-  tmp = dst + ".part"
+  os.makedirs(dst + ".parts", exist_ok=True)
   total = int(size) or sum(int(o["size"]) for o in objects)
+  done_bytes = sum(int(o["size"]) for o in objects if _part_complete(_part_path(dst, o["oid"]), o["oid"], int(o["size"])))
+  got = [done_bytes]
+
+  def progress(n: int) -> None:
+    got[0] += n
+    if progress_cb is not None and total:
+      progress_cb(min(1.0, got[0] / total))
+
   last_error: Exception | None = None
   for base_url in MODELS_BASE_URLS:
-    for attempt in range(STREAM_RETRIES):
+    for _attempt in range(STREAM_RETRIES):
       try:
-        digest = hashlib.sha256()
-        got = 0
-        with open(tmp, "wb") as f:
-          for obj in objects:
-            href, headers = _resolve_oid(session, base_url, obj["oid"], int(obj["size"]), auth)
-            obj_auth = None if headers.get("Authorization") else auth
-            with session.get(href, headers=headers, stream=True, timeout=120, auth=obj_auth) as r:
-              r.raise_for_status()
-              for chunk in r.iter_content(CHUNK):
-                f.write(chunk)
-                digest.update(chunk)
-                got += len(chunk)
-                if progress_cb is not None and total:
-                  progress_cb(min(1.0, got / total))
-        if total and got != total:
-          raise RuntimeError(f"size mismatch: {got}/{total} bytes")
-        if sha256 and digest.hexdigest() != sha256:
-          raise RuntimeError("sha256 mismatch")
-        os.replace(tmp, dst)
-        return dst
+        for obj in objects:
+          path = _part_path(dst, obj["oid"])
+          if _part_complete(path, obj["oid"], int(obj["size"])):
+            continue
+          got[0] = done_bytes
+          _fetch_part(session, base_url, obj, path, auth, progress)
+          if not _part_complete(path, obj["oid"], int(obj["size"])):
+            if os.path.getsize(path) >= int(obj["size"]):
+              os.remove(path)
+            raise RuntimeError(f"part {obj['oid'][:12]} incomplete or failed verification")
+          done_bytes += int(obj["size"])
+          got[0] = done_bytes
+        break
       except Exception as e:
         last_error = e
-        try:
-          os.remove(tmp)
-        except OSError:
-          pass
-  raise RuntimeError(f"model bundle download failed: {last_error}")
+    else:
+      continue
+    break
+  else:
+    raise RuntimeError(f"model bundle download failed: {last_error}")
+
+  tmp = dst + ".part"
+  digest = hashlib.sha256()
+  with open(tmp, "wb") as out:
+    for obj in objects:
+      with open(_part_path(dst, obj["oid"]), "rb") as f:
+        for chunk in iter(lambda: f.read(CHUNK), b""):
+          out.write(chunk)
+          digest.update(chunk)
+  if total and os.path.getsize(tmp) != total:
+    os.remove(tmp)
+    raise RuntimeError(f"size mismatch: {os.path.getsize(tmp) if os.path.exists(tmp) else 0}/{total} bytes")
+  if sha256 and digest.hexdigest() != sha256:
+    os.remove(tmp)
+    for obj in objects:
+      try:
+        os.remove(_part_path(dst, obj["oid"]))
+      except OSError:
+        pass
+    raise RuntimeError("sha256 mismatch")
+  os.replace(tmp, dst)
+  for obj in objects:
+    try:
+      os.remove(_part_path(dst, obj["oid"]))
+    except OSError:
+      pass
+  try:
+    os.rmdir(dst + ".parts")
+  except OSError:
+    pass
+  return dst

iqpilot/selfdrive/iqmodeld/tests/test_egpu_firmware_mirror.py

@@ -0,0 +1,30 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import hashlib
+import os
+
+from iqpilot.selfdrive.iqmodeld import egpu_helpers as eh
+
+
+def test_fetch_fw_mirrors_and_serves_offline(tmp_path, monkeypatch):
+  from tinygrad import helpers
+  blob = os.urandom(4096)
+  sha = hashlib.sha256(blob).hexdigest()
+  calls = []
+
+  def orig(path, name, sha256):
+    calls.append((path, name))
+    return blob
+
+  monkeypatch.setattr(helpers, "fetch_fw", orig, raising=False)
+  helpers.fetch_fw._iq_patched = False
+  monkeypatch.setattr(eh, "FIRMWARE_MIRROR", str(tmp_path / "mirror"))
+  eh.patch_tinygrad_fetch_fw()
+  assert helpers.fetch_fw("amdgpu", "gc.bin", sha) == blob and calls == [("amdgpu", "gc.bin")]
+  mirrored = tmp_path / "mirror" / "amdgpu" / "gc.bin"
+  assert mirrored.read_bytes() == blob
+  assert helpers.fetch_fw("amdgpu", "gc.bin", sha) == blob and len(calls) == 1
+  mirrored.write_bytes(b"corrupt")
+  assert helpers.fetch_fw("amdgpu", "gc.bin", sha) == blob and len(calls) == 2
+  assert mirrored.read_bytes() == blob

iqpilot/selfdrive/iqmodeld/tests/test_egpu_host_mock.py

@@ -0,0 +1,34 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import os
+import subprocess
+import sys
+
+import pytest
+
+from iqpilot.selfdrive.iqmodeld.tools.egpu_host_mock import tinygrad_tree
+
+PROBE = """
+import os
+os.environ["JIT_BATCH_SIZE"] = "0"
+from iqpilot.selfdrive.iqmodeld.tools.egpu_host_mock import activate
+activate("gfx1200")
+from tinygrad import Tensor
+from tinygrad.device import Device
+from tinygrad.engine.jit import TinyJit
+dev = Device["AMD"]
+assert dev.arch == "gfx1200", dev.arch
+assert type(dev.iface).__name__ == "MOCKUSBIface", type(dev.iface).__name__
+run = TinyJit(lambda x: (x * 2 + 1).sum(axis=1).realize())
+for i in range(3):
+  run(Tensor.ones(64, 64, device="AMD") * i)
+print("MOCK_OK")
+"""
+
+
+@pytest.mark.skipif(not os.path.isdir(os.path.join(tinygrad_tree(), "test", "mockgpu")), reason="tinygrad mockgpu tree not checked out")
+def test_mock_dock_captures_a_jit_without_hardware():
+  out = subprocess.run([sys.executable, "-c", PROBE], capture_output=True, text=True, timeout=600)
+  assert out.returncode == 0, out.stderr[-2000:]
+  assert "MOCK_OK" in out.stdout

iqpilot/selfdrive/iqmodeld/tests/test_egpu_oob.py

@@ -0,0 +1,62 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import os
+import pickle
+
+import numpy as np
+import pytest
+
+os.environ["DEV"] = "CPU"
+
+from iqpilot.selfdrive.iqmodeld.egpu_policy import dump_oob, is_oob, load_bundle
+
+
+def _bundle():
+  from tinygrad import Tensor
+  w = Tensor(np.arange(4096, dtype=np.float32).reshape(64, 64), device="CPU").realize()
+  return {"format": 2, "weights": w, "spec": {"a": ((1, 2), "float32")}, "blob": os.urandom(100_000)}
+
+
+def test_oob_round_trip_matches_plain_pickle(tmp_path):
+  b = _bundle()
+  oob = tmp_path / "b.oob"
+  with open(oob, "wb") as f:
+    dump_oob(b, f)
+  assert is_oob(str(oob))
+  got = load_bundle(str(oob))
+  np.testing.assert_array_equal(got["weights"].numpy(), b["weights"].numpy())
+  assert got["blob"] == b["blob"] and got["spec"] == b["spec"] and got["format"] == 2
+  plain = tmp_path / "b.pkl"
+  with open(plain, "wb") as f:
+    pickle.dump({"x": 1, "blob": b["blob"]}, f, protocol=pickle.HIGHEST_PROTOCOL)
+  assert not is_oob(str(plain))
+  assert load_bundle(str(plain))["blob"] == b["blob"]
+
+
+def test_memory_guard_raises_when_starved(monkeypatch):
+  from iqpilot.selfdrive.iqmodeld import iqegpumodeld as d
+  monkeypatch.setattr(d, "_mem_available_mb", lambda: 90)
+  monkeypatch.setattr(d, "MEMORY_WAIT_S", 0.0)
+  with pytest.raises(RuntimeError, match="insufficient memory"):
+    d._wait_for_memory(350)
+  monkeypatch.setattr(d, "_mem_available_mb", lambda: 900)
+  d._wait_for_memory(350)
+
+
+def test_opcode_rewrite_equals_oob_load(tmp_path):
+  from tinygrad import Tensor
+  from iqpilot.selfdrive.iqmodeld.tools.oob_rewrite import rewrite_oob
+  big = Tensor(np.random.default_rng(0).standard_normal((512, 512)).astype(np.float32), device="CPU").realize()
+  small = Tensor(np.arange(16, dtype=np.float32), device="CPU").realize()
+  b = {"format": 2, "w": big, "s": small, "meta": {"k": "v"}, "raw": os.urandom(200_000)}
+  plain = tmp_path / "plain.pkl"
+  with open(plain, "wb") as f:
+    pickle.dump(b, f, protocol=5)
+  oob = tmp_path / "oob.pkl"
+  moved, _ = rewrite_oob(str(plain), str(oob))
+  assert moved >= 2 and is_oob(str(oob))
+  got = load_bundle(str(oob))
+  np.testing.assert_array_equal(got["w"].numpy(), b["w"].numpy())
+  np.testing.assert_array_equal(got["s"].numpy(), b["s"].numpy())
+  assert got["raw"] == b["raw"] and got["meta"] == {"k": "v"}

iqpilot/selfdrive/iqmodeld/tests/test_model_bundle_downloader.py

@@ -0,0 +1,133 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+import hashlib
+import http.server
+import os
+import threading
+
+import pytest
+
+from iqpilot.selfdrive.iqmodeld import model_bundle_downloader as dl
+
+
+class _RangeHandler(http.server.BaseHTTPRequestHandler):
+  store: dict[str, bytes] = {}
+  cut_first: dict[str, int] = {}
+  hits: list[tuple[str, str | None]] = []
+
+  def log_message(self, *a):
+    pass
+
+  def do_GET(self):
+    oid = self.path.rsplit("/", 1)[-1]
+    data = self.store[oid]
+    rng = self.headers.get("Range")
+    self.hits.append((oid, rng))
+    start = int(rng.split("=")[1].rstrip("-")) if rng else 0
+    body = data[start:]
+    cut = self.cut_first.pop(oid, None)
+    if cut is not None:
+      body = body[:cut]
+    self.send_response(206 if rng else 200)
+    self.send_header("Content-Length", str(len(body)))
+    if rng:
+      self.send_header("Content-Range", f"bytes {start}-{start + len(body) - 1}/{len(data)}")
+    self.end_headers()
+    self.wfile.write(body)
+
+
+@pytest.fixture
+def server():
+  srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _RangeHandler)
+  t = threading.Thread(target=srv.serve_forever, daemon=True)
+  t.start()
+  yield srv
+  srv.shutdown()
+  srv.server_close()
+
+
+def _objects(parts):
+  return [{"oid": hashlib.sha256(p).hexdigest(), "size": len(p)} for p in parts]
+
+
+def test_resume_continues_a_cut_part_and_reuses_finished_parts(server, tmp_path, monkeypatch):
+  parts = [os.urandom(300_000), os.urandom(300_000), os.urandom(120_000)]
+  objs = _objects(parts)
+  _RangeHandler.store = {o["oid"]: p for o, p in zip(objs, parts, strict=True)}
+  _RangeHandler.hits = []
+  _RangeHandler.cut_first = {objs[1]["oid"]: 100_000}
+  port = server.server_address[1]
+  monkeypatch.setattr(dl, "_requests_auth", lambda: None)
+  monkeypatch.setattr(dl, "_resolve_oid", lambda session, base, oid, size, auth: (f"http://127.0.0.1:{port}/o/{oid}", {}))
+  monkeypatch.setattr(dl, "MODELS_BASE_URLS", ("http://unused",))
+  monkeypatch.setattr(dl, "STREAM_RETRIES", 3)
+  monkeypatch.setattr(dl, "CHUNK", 64 * 1024)
+  whole = b"".join(parts)
+  dst = str(tmp_path / "model.pkl")
+  out = dl.download_lfs_bundle(objs, dst, hashlib.sha256(whole).hexdigest(), len(whole))
+  with open(dst, "rb") as f:
+    assert out == dst and f.read() == whole
+  assert not os.path.exists(dst + ".parts")
+  ranges = [r for o, r in _RangeHandler.hits if o == objs[1]["oid"]]
+  assert ranges[0] is None and ranges[1] == "bytes=100000-"
+  assert sum(1 for o, _ in _RangeHandler.hits if o == objs[0]["oid"]) == 1
+
+
+def test_corrupt_finished_part_is_refetched(server, tmp_path, monkeypatch):
+  parts = [os.urandom(200_000), os.urandom(50_000)]
+  objs = _objects(parts)
+  _RangeHandler.store = {o["oid"]: p for o, p in zip(objs, parts, strict=True)}
+  _RangeHandler.hits = []
+  _RangeHandler.cut_first = {}
+  port = server.server_address[1]
+  monkeypatch.setattr(dl, "_requests_auth", lambda: None)
+  monkeypatch.setattr(dl, "_resolve_oid", lambda session, base, oid, size, auth: (f"http://127.0.0.1:{port}/o/{oid}", {}))
+  monkeypatch.setattr(dl, "MODELS_BASE_URLS", ("http://unused",))
+  dst = str(tmp_path / "model.pkl")
+  os.makedirs(dst + ".parts")
+  with open(dl._part_path(dst, objs[0]["oid"]), "wb") as f:
+    f.write(os.urandom(200_000))
+  whole = b"".join(parts)
+  dl.download_lfs_bundle(objs, dst, hashlib.sha256(whole).hexdigest(), len(whole))
+  with open(dst, "rb") as f:
+    assert f.read() == whole
+
+
+def test_hf_single_file_resumes_after_cut(server, tmp_path, monkeypatch):
+  data = os.urandom(700_000)
+  oid = hashlib.sha256(data).hexdigest()
+  _RangeHandler.store = {oid: data}
+  _RangeHandler.hits = []
+  _RangeHandler.cut_first = {oid: 250_000}
+  port = server.server_address[1]
+  monkeypatch.setattr(dl, "_hf", lambda: ({"Authorization": "Bearer test"}, lambda p: f"http://127.0.0.1:{port}/o/{oid}"))
+  monkeypatch.setattr(dl, "STREAM_RETRIES", 3)
+  monkeypatch.setattr(dl, "CHUNK", 64 * 1024)
+  dst = str(tmp_path / "policy.pkl")
+  out = dl.download_hf_file("egpu/policy/x.pkl", dst, oid, len(data))
+  with open(dst, "rb") as f:
+    assert out == dst and f.read() == data
+  ranges = [r for o, r in _RangeHandler.hits if o == oid]
+  assert ranges[0] is None and ranges[1] == "bytes=250000-"
+  assert not os.path.exists(dst + ".hfpart")
+
+
+def test_download_onnx_prefers_hf_then_falls_back(tmp_path, monkeypatch):
+  from iqpilot.selfdrive.iqmodeld import egpu_helpers as eh
+  meta = {"key": "m", "sha256": "ab" * 32, "download": {"kind": "comma_lfs", "size": 5}}
+  monkeypatch.setattr(eh, "onnx_cache_path", lambda m: str(tmp_path / "m.onnx"))
+  monkeypatch.setattr("iqpilot.selfdrive.iqmodeld.egpu_model.download_descriptor", lambda m: ("commalfs:" + m["sha256"], 5), raising=False)
+  calls = []
+  import iqpilot.selfdrive.iqmodeld.model_bundle_downloader as dlm
+  monkeypatch.setattr(dlm, "download_hf_file", lambda path, dst, sha, size, progress_cb=None: (calls.append(("hf", path)), open(dst, "wb").close(), dst)[2])
+  monkeypatch.setattr(eh, "resolve_download_url", lambda *a, **k: (calls.append(("lfs",)), "http://unused")[1])
+  out = eh.download_onnx(meta)
+  assert calls == [("hf", "onnx/" + "ab" * 32 + ".onnx")] and out == str(tmp_path / "m.onnx")
+  calls.clear()
+  def boom(*a, **k):
+    calls.append(("hf-fail",)); raise RuntimeError("hf down")
+  monkeypatch.setattr(dlm, "download_hf_file", boom)
+  with pytest.raises(Exception):
+    eh.download_onnx(meta)
+  assert calls[:2] == [("hf-fail",), ("lfs",)]

iqpilot/selfdrive/iqmodeld/tools/compile_egpu_model.py

@@ -7,14 +7,20 @@ import argparse
 import gc
 import os
 import pickle
+import sys
 import time
 
-os.environ.setdefault("DEV", "USB+AMD:LLVM")
 os.environ.setdefault("FLOAT16", "1")
 os.environ.setdefault("JIT_BATCH_SIZE", "0")
 os.environ.setdefault("GMMU", "0")
 os.environ.setdefault("TC_OPT", "2")
 
+HOST = "--host" in sys.argv
+if HOST:
+  from iqpilot.selfdrive.iqmodeld.tools.egpu_host_mock import DEFAULT_ARCH, activate
+  activate(sys.argv[sys.argv.index("--arch") + 1] if "--arch" in sys.argv else DEFAULT_ARCH)
+os.environ.setdefault("DEV", "USB+AMD:LLVM")
+
 import numpy as np
 
 from iqpilot.selfdrive.iqmodeld.egpu_helpers import egpu_pkl_path, local_onnx, patch_tinygrad_fetch_fw
@@ -150,7 +156,7 @@ def compile_policy_model(meta: dict, onnx_path: str, out_path: str) -> str:
   from tinygrad.engine.jit import TinyJit
   from tinygrad.nn.onnx import OnnxRunner
 
-  from iqpilot.selfdrive.iqmodeld.egpu_policy import POLICY_FORMAT, PackedInputs, make_queues, make_run_policy
+  from iqpilot.selfdrive.iqmodeld.egpu_policy import POLICY_FORMAT, PackedInputs, dump_oob, load_bundle, make_queues, make_run_policy
 
   if meta.get("split"):
     raise RuntimeError(f"model {meta['key']} is a split model; eGPU compiles fused models only")
@@ -176,7 +182,7 @@ def compile_policy_model(meta: dict, onnx_path: str, out_path: str) -> str:
     baseline = step(SEED + i)
   if baseline.shape[0] != meta["output_len"]:
     raise RuntimeError(f"model output length {baseline.shape[0]} != registry {meta['output_len']}")
-  if not np.isfinite(baseline).all():
+  if not HOST and not np.isfinite(baseline).all():
     raise RuntimeError("compiled policy produced non-finite outputs")
 
   bundle = {
@@ -191,16 +197,15 @@ def compile_policy_model(meta: dict, onnx_path: str, out_path: str) -> str:
   }
   os.makedirs(os.path.dirname(out_path), exist_ok=True)
   tmp = out_path + ".part"
-  print("serialize")
+  print("serialize (out-of-band buffers)")
   with open(tmp, "wb") as f:
-    pickle.dump(bundle, f, protocol=pickle.HIGHEST_PROTOCOL)
+    dump_oob(bundle, f)
 
   del bundle, jit, queues, packed
   gc.collect()
 
   print("reload + validate")
-  with open(tmp, "rb") as f:
-    jit = pickle.load(f)["run_policy"]
+  jit = load_bundle(tmp)["run_policy"]
   queues = make_queues(input_spec, frame_skip, device)
   packed = PackedInputs(input_spec)
   outs = []
@@ -211,6 +216,9 @@ def compile_policy_model(meta: dict, onnx_path: str, out_path: str) -> str:
     flat = out.numpy().reshape(-1)
     packed.views["prev_feat"][:] = flat[meta["output_slices"]["hidden_state"]].reshape(packed.views["prev_feat"].shape)
     outs.append(flat)
+  if HOST:
+    os.replace(tmp, out_path)
+    return out_path
   if not np.array_equal(outs[-1], baseline):
     raise RuntimeError("policy outputs differ from baseline after pickle round trip")
   if np.array_equal(outs[0], outs[-1]):
@@ -234,7 +242,11 @@ def main() -> None:
   p.add_argument("--progress-base", type=float, default=None)
   p.add_argument("--progress-span", type=float, default=0.0)
   p.add_argument("--format", type=int, default=2, choices=(1, 2))
+  p.add_argument("--host", action="store_true", help="compile on a mock dock (no AMD hardware); outputs need a dock parity gate")
+  p.add_argument("--arch", default=None, help="target gfx arch for --host")
   args = p.parse_args()
+  if args.host and args.format != 2:
+    raise SystemExit("--host supports format 2 only")
 
   if args.model is not None:
     if args.model in EGPU_MODELS:

iqpilot/selfdrive/iqmodeld/tools/convert_egpu_oob.py

@@ -0,0 +1,44 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import argparse
+import gc
+import os
+
+os.environ.setdefault("DEV", "USB+AMD:LLVM")
+os.environ.setdefault("GMMU", "0")
+
+from iqpilot.selfdrive.iqmodeld.egpu_helpers import patch_tinygrad_fetch_fw
+from iqpilot.selfdrive.iqmodeld.egpu_policy import dump_oob, is_oob, load_bundle
+
+
+def convert(src: str, dst: str) -> str:
+  patch_tinygrad_fetch_fw()
+  if is_oob(src):
+    if src != dst:
+      os.replace(src, dst)
+    return dst
+  bundle = load_bundle(src)
+  tmp = dst + ".part"
+  with open(tmp, "wb") as f:
+    dump_oob(bundle, f)
+  del bundle
+  gc.collect()
+  load_bundle(tmp)
+  os.replace(tmp, dst)
+  return dst
+
+
+def main() -> None:
+  p = argparse.ArgumentParser()
+  p.add_argument("src")
+  p.add_argument("--out", default=None)
+  args = p.parse_args()
+  out = convert(args.src, args.out or args.src)
+  print(f"converted -> {out} ({os.path.getsize(out) / 1e6:.1f} MB)")
+
+
+if __name__ == "__main__":
+  main()

iqpilot/selfdrive/iqmodeld/tools/egpu_host_mock.py

@@ -0,0 +1,60 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+
+DEFAULT_ARCH = "gfx1200"
+MOCK_DEV = "MOCKUSB+AMD:LLVM"
+
+
+def tinygrad_tree() -> str:
+  override = os.environ.get("IQ_TINYGRAD_TREE")
+  if override:
+    return override
+  here = os.path.dirname(os.path.abspath(__file__))
+  root = os.path.abspath(os.path.join(here, "..", "..", "..", ".."))
+  return os.path.join(root, "components", "tinygrad")
+
+
+def activate(arch: str = DEFAULT_ARCH, execute: bool = False) -> None:
+  assert "tinygrad" not in sys.modules, "egpu_host_mock.activate must run before tinygrad is imported"
+  os.environ["DEV"] = f"{MOCK_DEV}:{arch}"
+  tree = tinygrad_tree()
+  if tree not in sys.path:
+    sys.path.insert(0, tree)
+  from tinygrad.runtime.autogen import libc
+  if sys.platform == "darwin":
+    # A Homebrew-LLVM gfx1200 kernel (no s_code_end padding) hung a real dock; ship only container-built artifacts.
+    print("egpu_host_mock: native macOS LLVM output is for tests only; use scripts/iqpilot/host_egpu_compile_docker.sh for artifacts",
+          file=sys.stderr)
+
+    def memfd_create(name, flags):
+      fd, path = tempfile.mkstemp(prefix=b"iq_mock_" + bytes(name) + b"_")
+      os.unlink(path)
+      return fd
+    libc.memfd_create = memfd_create
+    if not hasattr(libc, "MFD_CLOEXEC"):
+      libc.MFD_CLOEXEC = 1
+  if not execute:
+    import ctypes
+    from test.mockgpu.amd import amdgpu
+    amdgpu.remu.run_asm = lambda *args, **kwargs: 0
+    pm4_wait = amdgpu.PM4Executor._exec_wait_reg_mem
+    sdma_poll = amdgpu.SDMAExecutor._execute_poll_regmem
+
+    # Without kernel execution no memory wait carries information; a blocked wait would need a host write to re-poll it.
+    def pm4_wait_passthrough(self, n):
+      if not pm4_wait(self, n):
+        self.rptr[0] += 7
+      return True
+
+    def sdma_poll_passthrough(self):
+      if not sdma_poll(self):
+        self.rptr[0] += ctypes.sizeof(amdgpu.sdma_pkts.poll_regmem)
+      return True
+    amdgpu.PM4Executor._exec_wait_reg_mem = pm4_wait_passthrough
+    amdgpu.SDMAExecutor._execute_poll_regmem = sdma_poll_passthrough

iqpilot/selfdrive/iqmodeld/tools/oob_rewrite.py

@@ -0,0 +1,64 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import pickletools
+import struct
+
+from iqpilot.selfdrive.iqmodeld.egpu_policy import OOB_MAGIC
+
+MIN_OOB_BYTES = 1 << 16
+NEXT_BUFFER = b"\x97"
+READONLY_BUFFER = b"\x98"
+
+
+def rewrite_oob(src: str, dst: str, min_bytes: int = MIN_OOB_BYTES) -> tuple[int, int]:
+  # tinygrad pickles device buffers as PickleBuffers, which land in-band as BYTEARRAY8/BINBYTES8
+  # without a buffer_callback; moving those opcodes out-of-band is byte-for-byte what a protocol-5
+  # dump with a buffer_callback produces, so nothing has to be unpickled (no dock needed).
+  with open(src, "rb") as f:
+    data = f.read()
+  ops = list(pickletools.genops(data))
+  proto = next((arg for op, arg, _ in ops if op.name == "PROTO"), 0)
+  if proto < 5:
+    raise ValueError(f"{src} is pickle protocol {proto}; out-of-band buffers need protocol 5")
+  moved = 0
+  tmp = dst + ".part"
+  with open(tmp, "wb") as out, open(tmp + ".buf", "wb") as bufs:
+    ops_stream = bytearray()
+    for i, (op, arg, pos) in enumerate(ops):
+      end = ops[i + 1][2] if i + 1 < len(ops) else len(data)
+      if op.name in ("BYTEARRAY8", "BINBYTES8", "BINBYTES") and len(arg) >= min_bytes:
+        ops_stream += NEXT_BUFFER
+        if op.name != "BYTEARRAY8":
+          ops_stream += READONLY_BUFFER
+        bufs.write(struct.pack("<q", len(arg)))
+        bufs.write(arg)
+        moved += 1
+      else:
+        ops_stream += data[pos:end]
+    out.write(OOB_MAGIC)
+    out.write(struct.pack("<q", len(ops_stream)))
+    out.write(ops_stream)
+  with open(tmp, "ab") as out, open(tmp + ".buf", "rb") as bufs:
+    while chunk := bufs.read(1 << 24):
+      out.write(chunk)
+  os.remove(tmp + ".buf")
+  os.replace(tmp, dst)
+  return moved, len(ops)
+
+
+def main() -> None:
+  p = argparse.ArgumentParser()
+  p.add_argument("src")
+  p.add_argument("dst")
+  args = p.parse_args()
+  moved, total = rewrite_oob(args.src, args.dst)
+  print(f"{args.dst}: moved {moved} buffers out-of-band ({total} opcodes, {os.path.getsize(args.dst) / 1e6:.1f} MB)")
+
+
+if __name__ == "__main__":
+  main()

iqpilot/selfdrive/selfdrived/events.py

@@ -375,7 +375,7 @@ EVENTS: dict[int, dict[str, Alert | AlertCallbackType]] = {
       "Pay Attention",
       "",
       AlertStatus.normal, AlertSize.small,
-      Priority.LOW, VisualAlert.none, AudibleAlert.none, .1),
+      Priority.MID, VisualAlert.none, AudibleAlert.none, .1),
   },
 
   EventName.promptDriverDistracted: {
@@ -901,7 +901,7 @@ if HARDWARE.get_device_type() == 'mici':
         "Pay Attention",
         "",
         AlertStatus.normal, AlertSize.small,
-        Priority.LOW, VisualAlert.none, AudibleAlert.none, 2),
+        Priority.MID, VisualAlert.none, AudibleAlert.none, 2),
     },
     EventName.promptDriverDistracted: {
       ET.PERMANENT: Alert(

iqpilot/selfdrive/selfdrived/iq_events.py

@@ -1,5 +1,6 @@
 import iqpilot.cereal.messaging as messaging
 from iqpilot.cereal import log, car, custom
+from iqpilot.common.params import Params
 from iqpilot.common.constants import CV
 from iqpilot.common.atlas_alerts import EventBook as EventsBase, Tier as Priority, Tags as ET, AlertCard as Alert, \
   NoEntryCard as NoEntryAlert, HardDisableCard as ImmediateDisableAlert, ChimeCard as EngagementAlert, \
@@ -94,6 +95,24 @@ _CAMERA_LABELS = {
 }
 
 _POLICE_CHIMED_IDS: set[str] = set()
+_USA_REGION_CODES = frozenset(("US", "USA", "UNITED STATES", "UNITED STATES OF AMERICA"))
+
+
+def _configured_country_code() -> str:
+  try:
+    value = Params().get("OsmLocationName")
+  except Exception:
+    return ""
+  if isinstance(value, bytes):
+    value = value.decode("utf-8", "ignore")
+  return str(value or "").strip().upper()
+
+
+def _alpr_alert_labels(country_code: str) -> tuple[str, str]:
+  is_row = bool(country_code) and country_code not in _USA_REGION_CODES
+  if is_row:
+    return "Traffic / ALPR Camera", "Traffic / ALPR Camera Detected"
+  return "Flock / ALPR Camera", "Flock Camera Detected"
 
 
 def speed_camera_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality) -> Alert:
@@ -101,14 +120,17 @@ def speed_camera_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMas
   ctype = int(getattr(nav.cameraType, "raw", nav.cameraType))
   label = _CAMERA_LABELS.get(ctype, "Speed Camera")
   distance = float(nav.cameraDistance)
+  alpr_detected_label = "Flock Camera Detected"
+  if ctype == int(custom.IQNavState.CameraType.alpr):
+    label, alpr_detected_label = _alpr_alert_labels(_configured_country_code())
   # RF (BLE/WiFi) Flock detection is a live proximity hit with no meaningful
   # distance — flockd/navd flag it with distance 0 on the alpr camera type.
   if ctype == int(custom.IQNavState.CameraType.alpr) and distance <= 0.0:
     return Alert(
-      "Flock Camera Detected",
+      alpr_detected_label,
       "",
       AlertStatus.normal, AlertSize.small,
-      Priority.HIGH, VisualAlert.none, AudibleAlert.prompt, .2)
+      Priority.LOW, VisualAlert.none, AudibleAlert.prompt, .2)
   if metric:
     dist_str = f"{distance:.0f} m" if distance < 1000.0 else f"{distance / 1000.0:.1f} km"
   else:
@@ -134,7 +156,8 @@ def speed_camera_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMas
     f"{label} • {detail}",
     "",
     AlertStatus.normal, AlertSize.small,
-    Priority.HIGH, VisualAlert.none, audible, .2)
+    Priority.LOW if ctype == int(custom.IQNavState.CameraType.alpr) else Priority.HIGH,
+    VisualAlert.none, audible, .2)
 
 
 class IQEvents(EventsBase):

iqpilot/selfdrive/selfdrived/tests/test_iq_camera_alerts.py

@@ -1,13 +1,17 @@
+import copy
 from types import SimpleNamespace
 
-from iqpilot.cereal import car, custom
+from iqpilot.cereal import car, custom, log
+from iqpilot.common.atlas_alerts import HardDisableCard, Tags as ET, Tier as Priority
+from iqpilot.selfdrive.selfdrived.alertmanager import AlertManager
+from iqpilot.selfdrive.selfdrived.events import EVENTS
 from iqpilot.selfdrive.selfdrived import iq_events
 
 
-def alert(camera_type, *, report_id="", chime=False):
+def alert(camera_type, *, report_id="", chime=False, distance=300.0):
   nav = SimpleNamespace(
     cameraType=camera_type,
-    cameraDistance=300.0,
+    cameraDistance=distance,
     cameraSpeedLimit=25.0,
     cameraAlertId=report_id,
     cameraChime=chime,
@@ -31,3 +35,45 @@ def test_police_chime_is_deduplicated_by_report():
   second = alert(custom.IQNavState.CameraType.police, report_id="police-a", chime=True)
   assert first.audible_alert == car.CarControl.HUDControl.AudibleAlert.prompt
   assert second.audible_alert == car.CarControl.HUDControl.AudibleAlert.none
+
+
+def test_alpr_wording_uses_configured_region(monkeypatch):
+  monkeypatch.setattr(iq_events, "_configured_country_code", lambda: "US")
+  assert alert(custom.IQNavState.CameraType.alpr).alert_text_1.startswith("Flock / ALPR Camera")
+  assert alert(custom.IQNavState.CameraType.alpr, distance=0.0).alert_text_1 == "Flock Camera Detected"
+
+  monkeypatch.setattr(iq_events, "_configured_country_code", lambda: "DE")
+  assert alert(custom.IQNavState.CameraType.alpr).alert_text_1.startswith("Traffic / ALPR Camera")
+  assert alert(custom.IQNavState.CameraType.alpr, distance=0.0).alert_text_1 == "Traffic / ALPR Camera Detected"
+
+
+def test_missing_region_is_safe_and_preserves_flock_wording(monkeypatch):
+  class UnavailableParams:
+    def get(self, key):
+      raise OSError(key)
+
+  monkeypatch.setattr(iq_events, "Params", UnavailableParams)
+  assert iq_events._configured_country_code() == ""
+  assert alert(custom.IQNavState.CameraType.alpr, distance=0.0).alert_text_1 == "Flock Camera Detected"
+
+
+def test_driver_attention_and_takeover_alerts_preempt_alpr():
+  flock = alert(custom.IQNavState.CameraType.alpr, distance=0.0)
+  pre_attention = copy.copy(EVENTS[log.OnroadEvent.EventName.preDriverDistracted][ET.PERMANENT])
+  prompt_attention = copy.copy(EVENTS[log.OnroadEvent.EventName.promptDriverDistracted][ET.PERMANENT])
+  takeover = copy.copy(EVENTS[log.OnroadEvent.EventName.driverDistracted][ET.PERMANENT])
+  immediate_disable = HardDisableCard("Regression Test")
+
+  assert flock.priority == Priority.LOW
+  assert pre_attention.priority == flock.priority + 1
+  assert prompt_attention.priority == flock.priority + 1
+  assert takeover.priority > flock.priority
+  assert immediate_disable.priority > flock.priority
+
+  for expected in (pre_attention, prompt_attention, takeover, immediate_disable):
+    manager = AlertManager()
+    flock.alert_type = "flock/warning"
+    expected.alert_type = f"expected/{expected.alert_text_1}"
+    manager.add_many(0, [flock, expected])
+    manager.process_alerts(0, set())
+    assert manager.current_alert is expected