IQ.Pilot Prebuilt Release @ 27f668a

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/api/__init__.py

@@ -0,0 +1,26 @@
+import iqpilot.common.api.comma_connect
+
+
+class Api:
+  def __init__(self, dongle_id):
+    self.service = iqpilot.common.api.comma_connect.CommaConnectApi(dongle_id)
+
+  def request(self, method, endpoint, **params):
+    return self.service.request(method, endpoint, **params)
+
+  def get(self, *args, **kwargs):
+    return self.service.get(*args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.service.post(*args, **kwargs)
+
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self.service.get_token(payload_extra, expiry_hours)
+
+
+def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
+  return iqpilot.common.api.comma_connect.CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+
+
+def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+  return iqpilot.common.api.comma_connect.CommaConnectApi(None).get_key_pair()

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/api/base.py

@@ -0,0 +1,84 @@
+import jwt
+import os
+import requests
+import unicodedata
+from datetime import datetime, timedelta, UTC
+from functools import lru_cache
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.system.version import get_version
+
+# name: jwt signature algorithm
+KEYS = {"id_rsa": "RS256",
+        "id_ecdsa": "ES256"}
+
+
+@lru_cache(maxsize=4)
+def load_signing_key(private_key: str):
+  # PyJWT re-parses a PEM string on every encode; an RSA parse is ~40ms, so cache the key object
+  try:
+    from cryptography.hazmat.primitives.serialization import load_pem_private_key
+    return load_pem_private_key(private_key.encode(), password=None)
+  except Exception:
+    return private_key
+
+
+class BaseApi:
+  def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
+    self.dongle_id = dongle_id
+    self.api_host = api_host
+    self.user_agent = user_agent
+    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
+
+  def get(self, *args, **kwargs):
+    return self.request('GET', *args, **kwargs)
+
+  def post(self, *args, **kwargs):
+    return self.request('POST', *args, **kwargs)
+
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours),
+      **extra_payload
+    }
+    if payload_extra is not None:
+      payload.update(payload_extra)
+    key = load_signing_key(self.private_key) if self.private_key else self.private_key
+    token = jwt.encode(payload, key, algorithm=self.jwt_algorithm)
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
+
+  def get_token(self, payload_extra=None, expiry_hours=1):
+    return self._get_token(payload_extra, expiry_hours)
+
+  def remove_non_ascii_chars(self, text):
+    normalized_text = unicodedata.normalize('NFD', text)
+    ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
+    return ascii_encoded_text.decode()
+
+  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
+    headers = {}
+    if access_token is not None:
+      headers['Authorization'] = "JWT " + access_token
+
+    version = self.remove_non_ascii_chars(get_version())
+    headers['User-Agent'] = self.user_agent + version
+
+    # TODO: add session to Api
+    req = requests if session is None else session
+    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
+
+  @staticmethod
+  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
+    for key in KEYS:
+      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
+        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
+          return KEYS[key], private.read(), public.read()
+    return None, None, None

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/api/comma_connect.py

@@ -0,0 +1,11 @@
+import os
+
+from iqpilot.common.api.base import BaseApi
+
+API_HOST = os.getenv('API_HOST', 'https://api-iqlabs.konn3kt.com')
+
+
+class CommaConnectApi(BaseApi):
+  def __init__(self, dongle_id):
+    super().__init__(dongle_id, API_HOST)
+    self.user_agent = "openpilot-"

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/atlas_alerts.py

@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from bisect import insort
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+from enum import IntEnum
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import car, log
+from iqpilot.common.realtime import DT_CTRL
+from iqpilot.system.hardware import HARDWARE
+
+AlertSize = log.SelfdriveState.AlertSize
+AlertStatus = log.SelfdriveState.AlertStatus
+VisualAlert = car.CarControl.HUDControl.VisualAlert
+AudibleAlert = car.CarControl.HUDControl.AudibleAlert
+
+
+def _frames_for(seconds: float) -> int:
+  return int(seconds / DT_CTRL)
+
+
+class Tier(IntEnum):
+  LOWEST = 0
+  LOWER = 1
+  LOW = 2
+  MID = 3
+  HIGH = 4
+  HIGHEST = 5
+
+
+class Tags:
+  ENABLE = "enable"
+  PRE_ENABLE = "preEnable"
+  OVERRIDE_LATERAL = "overrideLateral"
+  OVERRIDE_LONGITUDINAL = "overrideLongitudinal"
+  NO_ENTRY = "noEntry"
+  WARNING = "warning"
+  USER_DISABLE = "userDisable"
+  SOFT_DISABLE = "softDisable"
+  IMMEDIATE_DISABLE = "immediateDisable"
+  PERMANENT = "permanent"
+
+
+@dataclass(slots=True)
+class AlertCard:
+  alert_text_1: str
+  alert_text_2: str
+  alert_status: log.SelfdriveState.AlertStatus
+  alert_size: log.SelfdriveState.AlertSize
+  priority: Tier
+  visual_alert: car.CarControl.HUDControl.VisualAlert
+  audible_alert: car.CarControl.HUDControl.AudibleAlert
+  duration: int
+  creation_delay: float = 0.0
+  alert_type: str = field(default="", init=False)
+  event_type: str | None = field(default=None, init=False)
+
+  def __init__(self,
+               alert_text_1: str,
+               alert_text_2: str,
+               alert_status: log.SelfdriveState.AlertStatus,
+               alert_size: log.SelfdriveState.AlertSize,
+               priority: Tier,
+               visual_alert: car.CarControl.HUDControl.VisualAlert,
+               audible_alert: car.CarControl.HUDControl.AudibleAlert,
+               duration: float,
+               creation_delay: float = 0.0):
+    self.alert_text_1 = alert_text_1
+    self.alert_text_2 = alert_text_2
+    self.alert_status = alert_status
+    self.alert_size = alert_size
+    self.priority = priority
+    self.visual_alert = visual_alert
+    self.audible_alert = audible_alert
+    self.duration = _frames_for(duration)
+    self.creation_delay = creation_delay
+    self.alert_type = ""
+    self.event_type = None
+
+  def __str__(self) -> str:
+    return f"{self.alert_text_1}/{self.alert_text_2} {self.priority} {self.visual_alert} {self.audible_alert}"
+
+
+AlertFactory = Callable[[car.CarParams, car.CarState, messaging.SubMaster, bool, int, log.ControlsState], AlertCard]
+
+
+def car_mode_entry_alert(CP: car.CarParams, CS: car.CarState, sm: messaging.SubMaster, metric: bool, soft_disable_time: int, personality) -> AlertCard:
+  del CS, sm, metric, soft_disable_time, personality
+  headline = "Enable Adaptive Cruise to Engage"
+  if CP.brand == "honda":
+    headline = "Enable Main Switch to Engage"
+  return NoEntryCard(headline)
+
+
+class EventBook(ABC):
+  def __init__(self):
+    self._live_names: list[int] = []
+    self._latched_names: list[int] = []
+    self.event_counters: dict[int, int] = {}
+
+  @property
+  def events(self) -> list[int]:
+    return self._live_names
+
+  @events.setter
+  def events(self, values: list[int]) -> None:
+    self._live_names = values
+
+  @property
+  def static_events(self) -> list[int]:
+    return self._latched_names
+
+  @static_events.setter
+  def static_events(self, values: list[int]) -> None:
+    self._latched_names = values
+
+  @property
+  def names(self) -> list[int]:
+    return list(self._live_names)
+
+  def __len__(self) -> int:
+    return len(self._live_names)
+
+  def add(self, event_name: int, static: bool = False) -> None:
+    if static:
+      insort(self._latched_names, event_name)
+    insort(self._live_names, event_name)
+
+  def clear(self) -> None:
+    refreshed: dict[int, int] = {}
+    for event_name, frames_seen in self.event_counters.items():
+      refreshed[event_name] = frames_seen + 1 if event_name in self._live_names else 0
+    self.event_counters = refreshed
+    self._live_names = list(self._latched_names)
+
+  def contains(self, event_type: str) -> bool:
+    board = self.get_events_mapping()
+    return any(event_type in board.get(event_name, {}) for event_name in self._live_names)
+
+  def has(self, event_name: int) -> bool:
+    return event_name in self._live_names
+
+  def contains_in_list(self, events_list: list[int]) -> bool:
+    return any(event_name in self._live_names for event_name in events_list)
+
+  def remove(self, event_name: int, static: bool = False) -> None:
+    if static and event_name in self._latched_names:
+      self._latched_names.remove(event_name)
+
+    if event_name in self._live_names:
+      self.event_counters[event_name] = self.event_counters.get(event_name, 0) + 1
+      self._live_names.remove(event_name)
+
+  def add_from_msg(self, events: Iterable) -> None:
+    for event in events:
+      insort(self._live_names, event.name.raw)
+
+  def to_msg(self):
+    board = self.get_events_mapping()
+    outbound = []
+    for event_name in self._live_names:
+      msg = self.get_event_msg_type().new_message()
+      msg.name = event_name
+      for event_kind in board.get(event_name, {}):
+        setattr(msg, event_kind, True)
+      outbound.append(msg)
+    return outbound
+
+  def create_alerts(self, event_types: list[str], callback_args=None):
+    callback_args = [] if callback_args is None else callback_args
+    board = self.get_events_mapping()
+    spawned: list[AlertCard] = []
+    for event_name in self._live_names:
+      variants = board.get(event_name, {})
+      for event_type in event_types:
+        chosen = variants.get(event_type)
+        if chosen is None:
+          continue
+        alert = self._realize(chosen, callback_args)
+        age_frames = self.event_counters.get(event_name, 0) + 1
+        if age_frames * DT_CTRL < alert.creation_delay:
+          continue
+        alert.alert_type = f"{self.get_event_name(event_name)}/{event_type}"
+        alert.event_type = event_type
+        spawned.append(alert)
+    return spawned
+
+  @staticmethod
+  def _realize(candidate: AlertCard | AlertFactory, callback_args: list) -> AlertCard:
+    return candidate if isinstance(candidate, AlertCard) else candidate(*callback_args)
+
+  @abstractmethod
+  def get_events_mapping(self) -> dict[int, dict[str, AlertCard | AlertFactory]]:
+    raise NotImplementedError
+
+  @abstractmethod
+  def get_event_name(self, event: int) -> str:
+    raise NotImplementedError
+
+  @abstractmethod
+  def get_event_msg_type(self):
+    raise NotImplementedError
+
+
+def _mici_reframe(primary: str, secondary: str) -> tuple[str, str, log.SelfdriveState.AlertSize]:
+  if HARDWARE.get_device_type() == "mici":
+    return secondary, primary, AlertSize.small
+  return primary, secondary, AlertSize.mid
+
+
+class NoEntryCard(AlertCard):
+  def __init__(self,
+               alert_text_2: str,
+               alert_text_1: str = "IQ.Pilot Unavailable",
+               visual_alert: car.CarControl.HUDControl.VisualAlert = VisualAlert.none,
+               priority: Tier = Tier.LOW):
+    primary, secondary, size = _mici_reframe(alert_text_1, alert_text_2)
+    super().__init__(primary, secondary, AlertStatus.normal, size, priority, visual_alert, AudibleAlert.refuse, 3.0)
+
+
+class GentleDisableCard(AlertCard):
+  def __init__(self, alert_text_2: str):
+    super().__init__(
+      "TAKE CONTROL IMMEDIATELY",
+      alert_text_2,
+      AlertStatus.userPrompt,
+      AlertSize.full,
+      Tier.MID,
+      VisualAlert.steerRequired,
+      AudibleAlert.warningSoft,
+      2.0,
+    )
+
+
+class PendingDisableCard(GentleDisableCard):
+  def __init__(self, alert_text_2: str):
+    super().__init__(alert_text_2)
+    self.alert_text_1 = "IQ.Pilot will disengage"
+
+
+class HardDisableCard(AlertCard):
+  def __init__(self, alert_text_2: str):
+    super().__init__(
+      "TAKE CONTROL IMMEDIATELY",
+      alert_text_2,
+      AlertStatus.critical,
+      AlertSize.full,
+      Tier.HIGHEST,
+      VisualAlert.steerRequired,
+      AudibleAlert.warningImmediate,
+      4.0,
+    )
+
+
+class ChimeCard(AlertCard):
+  def __init__(self, audible_alert: car.CarControl.HUDControl.AudibleAlert):
+    super().__init__("", "", AlertStatus.normal, AlertSize.none, Tier.MID, VisualAlert.none, audible_alert, 0.2)
+
+
+class BannerCard(AlertCard):
+  def __init__(self, alert_text_1: str, alert_text_2: str = "", duration: float = 0.2, priority: Tier = Tier.LOWER, creation_delay: float = 0.0):
+    size = AlertSize.mid if alert_text_2 else AlertSize.small
+    super().__init__(alert_text_1, alert_text_2, AlertStatus.normal, size, priority, VisualAlert.none, AudibleAlert.none, duration, creation_delay)
+
+
+class BootCard(AlertCard):
+  def __init__(self, alert_text_1: str, alert_text_2: str = "Always keep hands on wheel and eyes on road", alert_status=AlertStatus.normal):
+    if HARDWARE.get_device_type() == "mici":
+      compact_secondary = "" if alert_text_2 == "Always keep hands on wheel and eyes on road" else alert_text_2
+      super().__init__(alert_text_1, compact_secondary, alert_status, AlertSize.small, Tier.LOWER, VisualAlert.none, AudibleAlert.none, 5.0)
+    else:
+      super().__init__(alert_text_1, alert_text_2, alert_status, AlertSize.mid, Tier.LOWER, VisualAlert.none, AudibleAlert.none, 5.0)
+
+
+class AlertBase(AlertCard):
+  pass
+
+
+NULL_ALERT = AlertCard("", "", AlertStatus.normal, AlertSize.none, Tier.LOWEST, VisualAlert.none, AudibleAlert.none, 0.0)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/basedir.py

@@ -0,0 +1,4 @@
+import os
+
+
+BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/constants.py

@@ -0,0 +1,23 @@
+import numpy as np
+
+# conversions
+class CV:
+  # Speed
+  MPH_TO_KPH = 1.609344
+  KPH_TO_MPH = 1. / MPH_TO_KPH
+  MS_TO_KPH = 3.6
+  KPH_TO_MS = 1. / MS_TO_KPH
+  MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
+  MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
+  MS_TO_KNOTS = 1.9438
+  KNOTS_TO_MS = 1. / MS_TO_KNOTS
+
+  # Angle
+  DEG_TO_RAD = np.pi / 180.
+  RAD_TO_DEG = 1. / DEG_TO_RAD
+
+  # Mass
+  LB_TO_KG = 0.453592
+
+
+ACCELERATION_DUE_TO_GRAVITY = 9.81  # m/s^2

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/file_chunker.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+import sys
+import math
+import os
+from pathlib import Path
+
+CHUNK_SIZE = 45 * 1024 * 1024  # 45MB, under GitHub's 50MB limit
+
+def get_chunk_name(name, idx, num_chunks):
+  return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
+
+def get_manifest_path(name):
+  return f"{name}.chunkmanifest"
+
+def _chunk_paths(path, num_chunks):
+  return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
+
+def get_chunk_targets(path, file_size):
+  num_chunks = math.ceil(file_size / CHUNK_SIZE)
+  return _chunk_paths(path, num_chunks)
+
+def chunk_file(path, targets):
+  manifest_path, *chunk_paths = targets
+  with open(path, 'rb') as f:
+    data = f.read()
+  actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
+  assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
+  for i, chunk_path in enumerate(chunk_paths):
+    with open(chunk_path, 'wb') as f:
+      f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
+  Path(manifest_path).write_text(str(len(chunk_paths)))
+  os.remove(path)
+
+def get_existing_chunks(path):
+  if os.path.isfile(path):
+    return [path]
+  if os.path.isfile(manifest := get_manifest_path(path)):
+    num_chunks = int(Path(manifest).read_text().strip())
+    return _chunk_paths(path, num_chunks)
+  raise FileNotFoundError(path)
+
+def read_file_chunked(path):
+  manifest_path = get_manifest_path(path)
+  if os.path.isfile(manifest_path):
+    num_chunks = int(Path(manifest_path).read_text().strip())
+    return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
+  if os.path.isfile(path):
+    return Path(path).read_bytes()
+  raise FileNotFoundError(path)
+
+
+if __name__ == "__main__":
+  path = sys.argv[1]
+  chunk_paths = get_chunk_targets(path, os.path.getsize(path))
+  chunk_file(path, chunk_paths)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/filter_simple.py

@@ -0,0 +1,71 @@
+from collections import deque
+
+import numpy as np
+
+
+class FirstOrderFilter:
+  def __init__(self, x0, rc, dt, initialized=True):
+    self.x = x0
+    self.dt = dt
+    self.update_alpha(rc)
+    self.initialized = initialized
+
+  def update_alpha(self, rc):
+    self.alpha = self.dt / (rc + self.dt)
+
+  def update(self, x):
+    if self.initialized:
+      self.x = (1. - self.alpha) * self.x + self.alpha * x
+    else:
+      self.initialized = True
+      self.x = x
+    return self.x
+
+
+class BounceFilter(FirstOrderFilter):
+  def __init__(self, x0, rc, dt, initialized=True, bounce=2):
+    self.velocity = FirstOrderFilter(0.0, 0.15, dt)
+    self.bounce = bounce
+    super().__init__(x0, rc, dt, initialized)
+
+  def update(self, x):
+    super().update(x)
+    scale = self.dt / (1.0 / 60.0)  # tuned at 60 fps
+    self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
+    self.velocity.update(0.0)
+    if abs(self.velocity.x) < 1e-5:
+      self.velocity.x = 0.0
+    self.x += self.velocity.x
+    return self.x
+
+
+class MyMovingAverage:
+  def __init__(self, window_size, value=None):
+    self.window_size = window_size
+    if value is not None:
+      self.values = deque([value] * window_size, maxlen=window_size)
+      self.sum = value * window_size
+      self.result = value
+    else:
+      self.values = deque(maxlen=window_size)
+      self.sum = 0
+      self.result = 0
+
+  def set(self, value):
+    self.values.clear()
+    self.values.append(value)
+    self.sum = value
+    self.result = value
+    return value
+
+  def set_all(self, value):
+    self.values = deque([value] * self.window_size, maxlen=self.window_size)
+    self.sum = value * self.window_size
+    self.result = value
+    return value
+
+  def process(self, value, median=False):
+    self.values.append(value)
+    self.sum = sum(self.values)
+    self.result = float(np.median(self.values)) if median else float(self.sum) / len(self.values)
+    return self.result

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/gpio.py

@@ -0,0 +1,89 @@
+import os
+import fcntl
+import ctypes
+from functools import cache
+
+def gpio_init(pin: int, output: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
+      f.write(b"out" if output else b"in")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} direction: {e}")
+
+def gpio_set(pin: int, high: bool) -> None:
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
+      f.write(b"1" if high else b"0")
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+def gpio_read(pin: int) -> bool | None:
+  val = None
+  try:
+    with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
+      val = bool(int(f.read().strip()))
+  except Exception as e:
+    print(f"Failed to set gpio {pin} value: {e}")
+
+  return val
+
+def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
+  try:
+    with open("/sys/class/gpio/export", 'w') as f:
+      f.write(str(pin))
+  except Exception:
+    print(f"Failed to export gpio {pin}")
+
+@cache
+def get_irq_action(irq: int) -> list[str]:
+  try:
+    with open(f"/sys/kernel/irq/{irq}/actions") as f:
+      actions = f.read().strip().split(',')
+      return actions
+  except FileNotFoundError:
+    return []
+
+def get_irqs_for_action(action: str) -> list[str]:
+  ret = []
+  with open("/proc/interrupts") as f:
+    for l in f.readlines():
+      irq = l.split(':')[0].strip()
+      if irq.isdigit() and action in get_irq_action(irq):
+        ret.append(irq)
+  return ret
+
+# *** gpiochip ***
+
+class gpioevent_data(ctypes.Structure):
+  _fields_ = [
+    ("timestamp", ctypes.c_uint64),
+    ("id", ctypes.c_uint32),
+  ]
+
+class gpioevent_request(ctypes.Structure):
+  _fields_ = [
+    ("lineoffset", ctypes.c_uint32),
+    ("handleflags", ctypes.c_uint32),
+    ("eventflags", ctypes.c_uint32),
+    ("label", ctypes.c_char * 32),
+    ("fd", ctypes.c_int)
+  ]
+
+def gpiochip_get_ro_value_fd(label: str, gpiochip_id: int, pin: int) -> int:
+  GPIOEVENT_REQUEST_BOTH_EDGES = 0x3
+  GPIOHANDLE_REQUEST_INPUT = 0x1
+  GPIO_GET_LINEEVENT_IOCTL = 0xc030b404
+
+  rq = gpioevent_request()
+  rq.lineoffset = pin
+  rq.handleflags = GPIOHANDLE_REQUEST_INPUT
+  rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES
+  rq.label = label.encode('utf-8')[:31] + b'\0'
+
+  fd = os.open(f"/dev/gpiochip{gpiochip_id}", os.O_RDONLY)
+  fcntl.ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, rq)
+  os.close(fd)
+  return int(rq.fd)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/i2c.py

@@ -0,0 +1,81 @@
+import os
+import fcntl
+import ctypes
+
+# I2C constants from /usr/include/linux/i2c-dev.h
+I2C_SLAVE = 0x0703
+I2C_SLAVE_FORCE = 0x0706
+I2C_SMBUS = 0x0720
+
+# SMBus transfer types
+I2C_SMBUS_READ = 1
+I2C_SMBUS_WRITE = 0
+I2C_SMBUS_BYTE_DATA = 2
+I2C_SMBUS_I2C_BLOCK_DATA = 8
+
+I2C_SMBUS_BLOCK_MAX = 32
+
+
+class _I2cSmbusData(ctypes.Union):
+  _fields_ = [
+    ("byte", ctypes.c_uint8),
+    ("word", ctypes.c_uint16),
+    ("block", ctypes.c_uint8 * (I2C_SMBUS_BLOCK_MAX + 2)),
+  ]
+
+
+class _I2cSmbusIoctlData(ctypes.Structure):
+  _fields_ = [
+    ("read_write", ctypes.c_uint8),
+    ("command", ctypes.c_uint8),
+    ("size", ctypes.c_uint32),
+    ("data", ctypes.POINTER(_I2cSmbusData)),
+  ]
+
+
+class SMBus:
+  def __init__(self, bus: int):
+    self._fd = os.open(f'/dev/i2c-{bus}', os.O_RDWR)
+
+  def __enter__(self) -> 'SMBus':
+    return self
+
+  def __exit__(self, *args) -> None:
+    self.close()
+
+  def close(self) -> None:
+    if hasattr(self, '_fd') and self._fd >= 0:
+      os.close(self._fd)
+      self._fd = -1
+
+  def _set_address(self, addr: int, force: bool = False) -> None:
+    ioctl_arg = I2C_SLAVE_FORCE if force else I2C_SLAVE
+    fcntl.ioctl(self._fd, ioctl_arg, addr)
+
+  def _smbus_access(self, read_write: int, command: int, size: int, data: _I2cSmbusData) -> None:
+    ioctl_data = _I2cSmbusIoctlData(read_write, command, size, ctypes.pointer(data))
+    fcntl.ioctl(self._fd, I2C_SMBUS, ioctl_data)
+
+  def read_byte_data(self, addr: int, register: int, force: bool = False) -> int:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_BYTE_DATA, data)
+    return int(data.byte)
+
+  def write_byte_data(self, addr: int, register: int, value: int, force: bool = False) -> None:
+    self._set_address(addr, force)
+    data = _I2cSmbusData()
+    data.byte = value & 0xFF
+    self._smbus_access(I2C_SMBUS_WRITE, register, I2C_SMBUS_BYTE_DATA, data)
+
+  def read_i2c_block_data(self, addr: int, register: int, length: int, force: bool = False) -> list[int]:
+    self._set_address(addr, force)
+    if not (0 <= length <= I2C_SMBUS_BLOCK_MAX):
+      raise ValueError(f"length must be 0..{I2C_SMBUS_BLOCK_MAX}")
+
+    data = _I2cSmbusData()
+    data.block[0] = length
+    self._smbus_access(I2C_SMBUS_READ, register, I2C_SMBUS_I2C_BLOCK_DATA, data)
+    read_len = int(data.block[0]) or length
+    read_len = min(read_len, length)
+    return [int(b) for b in data.block[1 : read_len + 1]]

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/iq_perf.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Any
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import custom
+from iqpilot.common.swaglog import cloudlog
+
+
+TRACE_SERVICE = "iqPerfTrace"
+MAX_TRACE_SAMPLES = 16
+_SHARED_PM: messaging.PubMaster | None = None
+
+
+@dataclass(slots=True)
+class PerfSample:
+  frame_id: int = 0
+  loop_dt_us: int = 0
+  update_us: int = 0
+  state_control_us: int = 0
+  publish_us: int = 0
+  tail_work_us: int = 0
+  rk_remaining_us: int = 0
+  stale_carcontrol_us: int = 0
+  stale_carcontrol_frames: int = 0
+  sendcan_gap_us: int = 0
+  model_eval_us: int = 0
+  model_dropped_frames: int = 0
+  model_backlog: int = 0
+  texture_decode_us: int = 0
+  texture_upload_us: int = 0
+  texture_unload_us: int = 0
+  texture_prune_us: int = 0
+  texture_consume_us: int = 0
+  texture_batch_size: int = 0
+  texture_bytes: int = 0
+  texture_cache_before: int = 0
+  texture_cache_after: int = 0
+  texture_unloaded: int = 0
+  memory_usage_percent: int = 0
+  gpu_usage_percent: int = 0
+  cpu_usage_percent: int = 0
+  flags: int = 0
+
+
+class PerfTraceRing:
+  def __init__(self, size: int = MAX_TRACE_SAMPLES):
+    self._samples: deque[PerfSample] = deque(maxlen=size)
+
+  def push(self, sample: PerfSample) -> None:
+    self._samples.append(sample)
+
+  def snapshot(self) -> list[PerfSample]:
+    return list(self._samples)
+
+
+class PerfTraceEmitter:
+  _SEVERITY_MAP = {
+    "info": custom.IQPerfTrace.Severity.info,
+    "warning": custom.IQPerfTrace.Severity.warning,
+    "error": custom.IQPerfTrace.Severity.error,
+    "critical": custom.IQPerfTrace.Severity.critical,
+  }
+
+  def __init__(self, process_name: str, pubmaster: messaging.PubMaster | None = None):
+    self.process_name = process_name
+    self._pm: messaging.PubMaster | None = pubmaster
+    self._last_emit_mono: dict[str, float] = {}
+    self._disabled = False
+
+  def _pubmaster(self) -> messaging.PubMaster:
+    global _SHARED_PM
+    if self._pm is not None:
+      return self._pm
+    if _SHARED_PM is None:
+      _SHARED_PM = messaging.PubMaster([TRACE_SERVICE])
+    self._pm = _SHARED_PM
+    return self._pm
+
+  @staticmethod
+  def _clamp_uint(value: int, bits: int) -> int:
+    return max(0, min(value, (1 << bits) - 1))
+
+  @staticmethod
+  def _clamp_int(value: int, bits: int) -> int:
+    lo = -(1 << (bits - 1))
+    hi = (1 << (bits - 1)) - 1
+    return max(lo, min(value, hi))
+
+  def emit(self, event_class: str, *,
+           severity: str = "warning",
+           frame_id: int = 0,
+           total_time_us: int = 0,
+           rk_remaining_us: int = 0,
+           batch_size: int = 0,
+           dropped_frames: int = 0,
+           backlog: int = 0,
+           flags: int = 0,
+           samples: list[PerfSample] | None = None,
+           missing_services: list[str] | None = None,
+           top_processes: list[str] | None = None,
+           detail: str = "",
+           min_interval_s: float = 0.0,
+           mirror_cloudlog: bool = True) -> bool:
+    if self._disabled:
+      return False
+    now = time.monotonic()
+    last_emit = self._last_emit_mono.get(event_class, 0.0)
+    if min_interval_s > 0.0 and (now - last_emit) < min_interval_s:
+      return False
+    self._last_emit_mono[event_class] = now
+
+    msg = messaging.new_message(TRACE_SERVICE)
+    trace = msg.iqPerfTrace
+    trace.process = self.process_name
+    trace.eventClass = event_class
+    trace.severity = self._SEVERITY_MAP.get(severity, custom.IQPerfTrace.Severity.warning)
+    trace.frameId = self._clamp_uint(int(frame_id), 32)
+    trace.totalTimeUs = self._clamp_uint(int(total_time_us), 32)
+    trace.rkRemainingUs = self._clamp_int(int(rk_remaining_us), 32)
+    trace.batchSize = self._clamp_uint(int(batch_size), 16)
+    trace.droppedFrames = self._clamp_uint(int(dropped_frames), 16)
+    trace.backlog = self._clamp_uint(int(backlog), 16)
+    trace.flags = self._clamp_uint(int(flags), 32)
+    trace.missingServices = list(missing_services or [])
+    trace.topProcesses = list(top_processes or [])
+    trace.detail = detail
+
+    trace_samples = samples or []
+    samples_builder = trace.init("samples", len(trace_samples))
+    for i, sample in enumerate(trace_samples):
+      builder = samples_builder[i]
+      builder.frameId = self._clamp_uint(int(sample.frame_id), 32)
+      builder.loopDtUs = self._clamp_uint(int(sample.loop_dt_us), 32)
+      builder.updateUs = self._clamp_uint(int(sample.update_us), 32)
+      builder.stateControlUs = self._clamp_uint(int(sample.state_control_us), 32)
+      builder.publishUs = self._clamp_uint(int(sample.publish_us), 32)
+      builder.tailWorkUs = self._clamp_uint(int(sample.tail_work_us), 32)
+      builder.rkRemainingUs = self._clamp_int(int(sample.rk_remaining_us), 32)
+      builder.staleCarControlUs = self._clamp_uint(int(sample.stale_carcontrol_us), 32)
+      builder.staleCarControlFrames = self._clamp_uint(int(sample.stale_carcontrol_frames), 16)
+      builder.sendcanGapUs = self._clamp_uint(int(sample.sendcan_gap_us), 32)
+      builder.modelEvalUs = self._clamp_uint(int(sample.model_eval_us), 32)
+      builder.modelDroppedFrames = self._clamp_uint(int(sample.model_dropped_frames), 16)
+      builder.modelBacklog = self._clamp_uint(int(sample.model_backlog), 16)
+      builder.textureDecodeUs = self._clamp_uint(int(sample.texture_decode_us), 32)
+      builder.textureUploadUs = self._clamp_uint(int(sample.texture_upload_us), 32)
+      builder.textureUnloadUs = self._clamp_uint(int(sample.texture_unload_us), 32)
+      builder.texturePruneUs = self._clamp_uint(int(sample.texture_prune_us), 32)
+      builder.textureConsumeUs = self._clamp_uint(int(sample.texture_consume_us), 32)
+      builder.textureBatchSize = self._clamp_uint(int(sample.texture_batch_size), 16)
+      builder.textureBytes = self._clamp_uint(int(sample.texture_bytes), 32)
+      builder.textureCacheBefore = self._clamp_uint(int(sample.texture_cache_before), 16)
+      builder.textureCacheAfter = self._clamp_uint(int(sample.texture_cache_after), 16)
+      builder.textureUnloaded = self._clamp_uint(int(sample.texture_unloaded), 16)
+      builder.memoryUsagePercent = self._clamp_uint(int(sample.memory_usage_percent), 16)
+      builder.gpuUsagePercent = self._clamp_uint(int(sample.gpu_usage_percent), 16)
+      builder.cpuUsagePercent = self._clamp_uint(int(sample.cpu_usage_percent), 16)
+      builder.flags = self._clamp_uint(int(sample.flags), 32)
+
+    try:
+      self._pubmaster().send(TRACE_SERVICE, msg)
+    except messaging.MultiplePublishersError:
+      self._disabled = True
+      cloudlog.error(f"iq_perf_trace disabled for {self.process_name}: duplicate publisher for {TRACE_SERVICE}")
+      return False
+    except Exception:
+      cloudlog.exception(f"iq_perf_trace publish failed for {self.process_name}")
+      return False
+
+    if mirror_cloudlog:
+      cloudlog.event(
+        "iq_perf_trace",
+        process=self.process_name,
+        event_class=event_class,
+        severity=severity,
+        frame_id=int(frame_id),
+        total_time_us=int(total_time_us),
+        dropped_frames=int(dropped_frames),
+        flags=int(flags),
+        detail=detail,
+      )
+    return True

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/issue_debug.py

@@ -0,0 +1,44 @@
+import os
+import threading
+import time
+from datetime import datetime
+from pathlib import Path
+
+from iqpilot.system.hardware import PC
+from iqpilot.system.hardware.hw import Paths
+
+
+DEBUG_FILENAME = "iqpilot_issue_debug.txt"
+DEBUG_PATH = Path(Paths.comma_home()) / "community" / DEBUG_FILENAME if PC else Path("/data/community") / DEBUG_FILENAME
+
+_lock = threading.Lock()
+_last_log_times: dict[str, float] = {}
+
+
+def log_issue(tag: str, message: str) -> None:
+  try:
+    DEBUG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with _lock:
+      with open(DEBUG_PATH, "a", encoding="utf-8") as f:
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+        f.write(f"[{timestamp}] [{tag}] {message}\n")
+  except OSError:
+    pass
+
+
+def log_issue_limited(key: str, tag: str, message: str, interval_sec: float = 1.0) -> None:
+  now = time.monotonic()
+  with _lock:
+    last = _last_log_times.get(key, 0.0)
+    if now - last < interval_sec:
+      return
+    _last_log_times[key] = now
+
+  log_issue(tag, message)
+
+
+def clear_issue_debug_log() -> None:
+  try:
+    os.remove(DEBUG_PATH)
+  except OSError:
+    pass

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/logging_extra.py

@@ -0,0 +1,249 @@
+import io
+import os
+import sys
+import copy
+import json
+import time
+import uuid
+import socket
+import logging
+import traceback
+import numpy as np
+from threading import local
+from collections import OrderedDict
+from contextlib import contextmanager
+
+LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
+
+def json_handler(obj):
+  if isinstance(obj, np.bool_):
+    return bool(obj)
+  # if isinstance(obj, (datetime.date, datetime.time)):
+  #   return obj.isoformat()
+  return repr(obj)
+
+def json_robust_dumps(obj):
+  return json.dumps(obj, default=json_handler)
+
+class NiceOrderedDict(OrderedDict):
+  def __str__(self):
+    return json_robust_dumps(self)
+
+class SwagFormatter(logging.Formatter):
+  def __init__(self, swaglogger):
+    logging.Formatter.__init__(self, None, '%a %b %d %H:%M:%S %Z %Y')
+
+    self.swaglogger = swaglogger
+    self.host = socket.gethostname()
+
+  def format_dict(self, record):
+    record_dict = NiceOrderedDict()
+
+    if isinstance(record.msg, dict):
+      record_dict['msg'] = record.msg
+    else:
+      try:
+        record_dict['msg'] = record.getMessage()
+      except (ValueError, TypeError):
+        record_dict['msg'] = [record.msg]+record.args
+
+    record_dict['ctx'] = self.swaglogger.get_ctx()
+
+    if record.exc_info:
+      record_dict['exc_info'] = self.formatException(record.exc_info)
+
+    record_dict['level'] = record.levelname
+    record_dict['levelnum'] = record.levelno
+    record_dict['name'] = record.name
+    record_dict['filename'] = record.filename
+    record_dict['lineno'] = record.lineno
+    record_dict['pathname'] = record.pathname
+    record_dict['module'] = record.module
+    record_dict['funcName'] = record.funcName
+    record_dict['host'] = self.host
+    record_dict['process'] = record.process
+    record_dict['thread'] = record.thread
+    record_dict['threadName'] = record.threadName
+    record_dict['created'] = record.created
+
+    return record_dict
+
+  def format(self, record):
+    if self.swaglogger is None:
+      raise Exception("must set swaglogger before calling format()")
+    return json_robust_dumps(self.format_dict(record))
+
+class SwagLogFileFormatter(SwagFormatter):
+  def fix_kv(self, k, v):
+    # append type to names to preserve legacy naming in logs
+    # avoids overlapping key namespaces with different types
+    # e.g. log.info() creates 'msg' -> 'msg$s'
+    #      log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
+    #      because overlapping namespace 'msg' caused problems
+    if isinstance(v, (str, bytes)):
+      k += "$s"
+    elif isinstance(v, float):
+      k += "$f"
+    elif isinstance(v, bool):
+      k += "$b"
+    elif isinstance(v, int):
+      k += "$i"
+    elif isinstance(v, dict):
+      nv = {}
+      for ik, iv in v.items():
+        ik, iv = self.fix_kv(ik, iv)
+        nv[ik] = iv
+      v = nv
+    elif isinstance(v, list):
+      k += "$a"
+    return k, v
+
+  def format(self, record):
+    if isinstance(record, str):
+      v = json.loads(record)
+    else:
+      v = self.format_dict(record)
+
+    mk, mv = self.fix_kv('msg', v['msg'])
+    del v['msg']
+    v[mk] = mv
+    v['id'] = uuid.uuid4().hex
+
+    return json_robust_dumps(v)
+
+class SwagErrorFilter(logging.Filter):
+  def filter(self, record):
+    return record.levelno < logging.ERROR
+
+def _tmpfunc():
+  return 0
+
+def _srcfile():
+  return os.path.normcase(_tmpfunc.__code__.co_filename)
+
+class SwagLogger(logging.Logger):
+  def __init__(self):
+    logging.Logger.__init__(self, "swaglog")
+
+    self.global_ctx = {}
+
+    self.log_local = local()
+    self.log_local.ctx = {}
+
+  def local_ctx(self):
+    try:
+      return self.log_local.ctx
+    except AttributeError:
+      self.log_local.ctx = {}
+      return self.log_local.ctx
+
+  def get_ctx(self):
+    return dict(self.local_ctx(), **self.global_ctx)
+
+  @contextmanager
+  def ctx(self, **kwargs):
+    old_ctx = self.local_ctx()
+    self.log_local.ctx = copy.copy(old_ctx) or {}
+    self.log_local.ctx.update(kwargs)
+    try:
+      yield
+    finally:
+      self.log_local.ctx = old_ctx
+
+  def bind(self, **kwargs):
+    self.local_ctx().update(kwargs)
+
+  def bind_global(self, **kwargs):
+    self.global_ctx.update(kwargs)
+
+  def event(self, event, *args, **kwargs):
+    evt = NiceOrderedDict()
+    evt['event'] = event
+    if args:
+      evt['args'] = args
+    evt.update(kwargs)
+    if 'error' in kwargs:
+      self.error(evt)
+    elif 'debug' in kwargs:
+      self.debug(evt)
+    else:
+      self.info(evt)
+
+  def timestamp(self, event_name):
+    if LOG_TIMESTAMPS:
+      t = time.monotonic()
+      tstp = NiceOrderedDict()
+      tstp['timestamp'] = NiceOrderedDict()
+      tstp['timestamp']["event"] = event_name
+      tstp['timestamp']["time"] = t*1e9
+      self.debug(tstp)
+
+  def findCaller(self, stack_info=False, stacklevel=1):
+    """
+    Find the stack frame of the caller so that we can note the source
+    file name, line number and function name.
+    """
+    f = sys._getframe(3)
+    #On some versions of IronPython, currentframe() returns None if
+    #IronPython isn't run with -X:Frames.
+    if f is not None:
+      f = f.f_back
+    orig_f = f
+    while f and stacklevel > 1:
+      f = f.f_back
+      stacklevel -= 1
+    if not f:
+      f = orig_f
+    rv = "(unknown file)", 0, "(unknown function)", None
+    while hasattr(f, "f_code"):
+      co = f.f_code
+      filename = os.path.normcase(co.co_filename)
+
+      if filename == _srcfile:
+        f = f.f_back
+        continue
+      sinfo = None
+      if stack_info:
+        sio = io.StringIO()
+        sio.write('Stack (most recent call last):\n')
+        traceback.print_stack(f, file=sio)
+        sinfo = sio.getvalue()
+        if sinfo[-1] == '\n':
+          sinfo = sinfo[:-1]
+        sio.close()
+      rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
+      break
+    return rv
+
+if __name__ == "__main__":
+  log = SwagLogger()
+
+  stdout_handler = logging.StreamHandler(sys.stdout)
+  stdout_handler.setLevel(logging.INFO)
+  stdout_handler.addFilter(SwagErrorFilter())
+  log.addHandler(stdout_handler)
+
+  stderr_handler = logging.StreamHandler(sys.stderr)
+  stderr_handler.setLevel(logging.ERROR)
+  log.addHandler(stderr_handler)
+
+  log.info("asdasd %s", "a")
+  log.info({'wut': 1})
+  log.warning("warning")
+  log.error("error")
+  log.critical("critical")
+  log.event("test", x="y")
+
+  with log.ctx():
+    stdout_handler.setFormatter(SwagFormatter(log))
+    stderr_handler.setFormatter(SwagFormatter(log))
+    log.bind(user="some user")
+    log.info("in req")
+    print("")
+    log.warning("warning")
+    print("")
+    log.error("error")
+    print("")
+    log.critical("critical")
+    print("")
+    log.event("do_req", a=1, b="c")

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/realtime.py

@@ -0,0 +1,134 @@
+"""Utilities for reading real time clocks and keeping soft real time constraints."""
+import gc
+import os
+import sys
+import time
+
+from setproctitle import getproctitle
+
+from iqpilot.common.utils import MovingAverage
+from iqpilot.system.hardware import PC
+
+
+# time step for each process
+DT_CTRL = 0.01  # controlsd
+DT_MDL = 0.05  # model
+DT_HW = 0.5  # hardwared and manager
+DT_DMON = 0.05  # driver monitoring
+
+
+class Priority:
+  # CORE 2
+  # - modeld = 55
+  # - camerad = 54
+  CTRL_LOW = 51 # plannerd & radard
+
+  # CORE 3
+  # - pandad = 55
+  CTRL_HIGH = 53
+
+
+def set_core_affinity(cores: list[int]) -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setaffinity(0, cores)
+
+
+def config_realtime_process(cores: int | list[int], priority: int) -> None:
+  gc.disable()
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(priority))
+  c = cores if isinstance(cores, list) else [cores, ]
+  set_core_affinity(c)
+
+
+def config_background_thread() -> None:
+  if sys.platform == 'linux' and not PC:
+    os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
+    set_core_affinity(list(range(os.cpu_count() or 1)))
+
+
+def lock_memory() -> None:
+  """mlockall this process so memory reclaim/compaction can't stall it. RT control
+  procs only (locking ui/modeld would worsen pressure). Best-effort."""
+  if sys.platform != 'linux' or PC:
+    return
+  try:
+    import ctypes
+    import resource
+    resource.setrlimit(resource.RLIMIT_MEMLOCK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+    MCL_CURRENT, MCL_FUTURE = 0x1, 0x2
+    libc = ctypes.CDLL("libc.so.6", use_errno=True)
+    if libc.mlockall(MCL_CURRENT | MCL_FUTURE) != 0:
+      raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno()))
+  except Exception as e:
+    try:
+      from iqpilot.common.swaglog import cloudlog
+      cloudlog.warning(f"lock_memory (mlockall) failed: {e}")
+    except Exception:
+      pass
+
+
+class Ratekeeper:
+  def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
+    """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
+    self._interval = 1. / rate
+    self._print_delay_threshold = print_delay_threshold
+    self._frame = 0
+    self._remaining = 0.0
+    self._process_name = getproctitle()
+    self._last_monitor_time = -1.
+    self._next_frame_time = -1.
+
+    self.avg_dt = MovingAverage(100)
+    self.avg_dt.add_value(self._interval)
+
+  def reset(self) -> None:
+    self._remaining = 0.0
+    self._last_monitor_time = -1.
+    self._next_frame_time = -1.
+    self.avg_dt = MovingAverage(100)
+    self.avg_dt.add_value(self._interval)
+
+  @property
+  def frame(self) -> int:
+    return self._frame
+
+  @property
+  def remaining(self) -> float:
+    return self._remaining
+
+  @property
+  def lag(self) -> float:
+    return max(0., -self._remaining)
+
+  @property
+  def lagging(self) -> bool:
+    expected_dt = self._interval * (1 / 0.9)
+    return self.avg_dt.get_average() > expected_dt
+
+  # Maintain loop rate by calling this at the end of each loop
+  def keep_time(self) -> bool:
+    lagged = self.monitor_time()
+    if self._remaining > 0:
+      time.sleep(self._remaining)
+    return lagged
+
+  # Monitors the cumulative lag, but does not enforce a rate
+  def monitor_time(self) -> bool:
+    if self._last_monitor_time < 0:
+      self._next_frame_time = time.monotonic() + self._interval
+      self._last_monitor_time = time.monotonic()
+
+    prev = self._last_monitor_time
+    self._last_monitor_time = time.monotonic()
+    self.avg_dt.add_value(self._last_monitor_time - prev)
+
+    lagged = False
+    remaining = self._next_frame_time - time.monotonic()
+    self._next_frame_time += self._interval
+    if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
+      print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
+      lagged = True
+    self._frame += 1
+    self._remaining = remaining
+    return lagged

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/spinner.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+from iqpilot.common.basedir import BASEDIR
+
+
+class Spinner:
+  def __init__(self):
+    try:
+      self.spinner_proc = subprocess.Popen(["./spinner.py"],
+                                           stdin=subprocess.PIPE,
+                                           cwd=os.path.join(BASEDIR, "iqpilot", "system", "ui"),
+                                           close_fds=True)
+    except OSError:
+      self.spinner_proc = None
+
+  def __enter__(self):
+    return self
+
+  def update(self, spinner_text: str):
+    if self.spinner_proc is not None:
+      self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
+      try:
+        self.spinner_proc.stdin.flush()
+      except BrokenPipeError:
+        pass
+
+  def update_progress(self, cur: float, total: float):
+    self.update(str(round(100 * cur / total)))
+
+  def close(self):
+    if self.spinner_proc is not None:
+      self.spinner_proc.kill()
+      try:
+        self.spinner_proc.communicate(timeout=2.)
+      except subprocess.TimeoutExpired:
+        print("WARNING: failed to kill spinner")
+      self.spinner_proc = None
+
+  def __del__(self):
+    self.close()
+
+  def __exit__(self, exc_type, exc_value, traceback):
+    self.close()
+
+
+if __name__ == "__main__":
+  import time
+  with Spinner() as s:
+    s.update("Spinner text")
+    time.sleep(5.0)
+  print("gone")
+  time.sleep(5.0)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/steer_delay.py

@@ -0,0 +1,60 @@
+"""
+Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos
+
+Chooses which steer-actuator delay the lateral controllers run with: the value the
+live estimator learned, or the driver's fixed software delay — gated by the
+"IQLiveSteerDelay" param. The pick is mirrored into "IQSteerDelayCache" so consumers that do
+not subscribe to lateralDelay can still read the current value.
+"""
+from iqpilot.cereal import car
+from iqpilot.common.params import Params
+
+_ENABLE_KEY = "IQLiveSteerDelay"
+_FIXED_KEY = "IQSoftwareSteerDelay"
+_CACHE_KEY = "IQSteerDelayCache"
+
+
+def fixed_steer_delay(params, stock_delay):
+  """The rack's own delay plus the driver's IQSoftwareSteerDelay offset, as the UI reports it."""
+  return stock_delay + float(params.get(_FIXED_KEY, return_default=True))
+
+
+def resolve_steer_delay(params, stock_delay):
+  """Learned lateral delay while live-learning is enabled, otherwise the driver's fixed delay."""
+  if not params.get_bool(_ENABLE_KEY):
+    return fixed_steer_delay(params, stock_delay)
+  return float(params.get(_CACHE_KEY, return_default=True))
+
+
+def lateral_action_delay(params, car_params, live_delay):
+  """Delay the lateral path should be planned against.
+
+  Angle cars honour the IQLiveSteerDelay toggle so that with live learning off the
+  estimate never reaches the path: lagd cross-correlates against localizer lateral
+  accel, so it reports whole-vehicle response (~0.36 s measured on VW MQB, 0.44 s on
+  Tesla) where the lookahead wants actuator delay (~0.10 s). Torque cars keep the
+  live estimate.
+  """
+  if car_params.steerControlType == car.CarParams.SteerControlType.angle:
+    return resolve_steer_delay(params, car_params.steerActuatorDelay)
+  return live_delay
+
+
+def cached_steer_delay():
+  """Last value SteerDelayPublisher mirrored into the param — usable without a
+  lateralDelay subscription (e.g. at process startup)."""
+  return Params().get(_CACHE_KEY, return_default=True)
+
+
+class SteerDelayPublisher:
+  """Refreshes IQSteerDelayCache every lag message: the learned live delay when the
+  toggle is on, else the actuator delay plus the driver's fixed software offset."""
+
+  def __init__(self, car_params):
+    self._params = Params()
+    self._actuator_delay = car_params.steerActuatorDelay
+
+  def update(self, lag_msg):
+    live = self._params.get_bool(_ENABLE_KEY)
+    value = lag_msg.lateralDelay.lateralDelay if live else fixed_steer_delay(self._params, self._actuator_delay)
+    self._params.put_nonblocking(_CACHE_KEY, value)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/swaglog.py

@@ -0,0 +1,165 @@
+import logging
+import os
+import sys
+import time
+import warnings
+from pathlib import Path
+from logging.handlers import BaseRotatingHandler
+
+import zmq
+
+from iqpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
+from iqpilot.system.hardware.hw import Paths
+
+
+def get_file_handler():
+  Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
+  base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
+  handler = SwaglogRotatingFileHandler(base_filename)
+  return handler
+
+class SwaglogRotatingFileHandler(BaseRotatingHandler):
+  def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
+    super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
+    self.base_filename = base_filename
+    self.interval = interval # seconds
+    self.max_bytes = max_bytes
+    self.backup_count = backup_count
+    self.log_files = self.get_existing_logfiles()
+    log_indexes = [f.split(".")[-1] for f in self.log_files]
+    self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
+    self.last_rollover = None
+    self.doRollover()
+
+  def _open(self):
+    self.last_rollover = time.monotonic()
+    self.last_file_idx += 1
+    next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
+    stream = open(next_filename, self.mode, encoding=self.encoding)
+    self.log_files.insert(0, next_filename)
+    return stream
+
+  def get_existing_logfiles(self):
+    log_files = list()
+    base_dir = os.path.dirname(self.base_filename)
+    for fn in os.listdir(base_dir):
+      fp = os.path.join(base_dir, fn)
+      if fp.startswith(self.base_filename) and os.path.isfile(fp):
+        log_files.append(fp)
+    return sorted(log_files)
+
+  def shouldRollover(self, record):
+    size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
+    time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
+    return size_exceeded or time_exceeded
+
+  def doRollover(self):
+    if self.stream:
+      self.stream.close()
+    self.stream = self._open()
+
+    if self.backup_count > 0:
+      while len(self.log_files) > self.backup_count:
+        to_delete = self.log_files.pop()
+        if os.path.exists(to_delete): # just being safe, should always exist
+          os.remove(to_delete)
+
+class UnixDomainSocketHandler(logging.Handler):
+  def __init__(self, formatter):
+    logging.Handler.__init__(self)
+    self.setFormatter(formatter)
+    self.pid = None
+
+    self.zctx = None
+    self.sock = None
+
+  def __del__(self):
+    self.close()
+
+  def close(self):
+    if self.sock is not None:
+      self.sock.close()
+    if self.zctx is not None:
+      self.zctx.term()
+
+  def connect(self):
+    self.zctx = zmq.Context()
+    self.sock = self.zctx.socket(zmq.PUSH)
+    self.sock.setsockopt(zmq.LINGER, 10)
+    self.sock.connect(Paths.swaglog_ipc())
+    self.pid = os.getpid()
+
+  def emit(self, record):
+    if os.getpid() != self.pid:
+      # TODO suppresses warning about forking proc with zmq socket, fix root cause
+      warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
+      self.connect()
+
+    msg = self.format(record).rstrip('\n')
+    # print("SEND".format(repr(msg)))
+    try:
+      s = chr(record.levelno)+msg
+      self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
+    except zmq.error.Again:
+      # drop :/
+      pass
+
+
+class ForwardingHandler(logging.Handler):
+  def __init__(self, target_logger):
+    super().__init__()
+    self.target_logger = target_logger
+
+  def emit(self, record):
+    self.target_logger.handle(record)
+
+
+def add_file_handler(log):
+  """
+  Function to add the file log handler to swaglog.
+  This can be used to store logs when logmessaged is not running.
+  """
+  handler = get_file_handler()
+  handler.setFormatter(SwagLogFileFormatter(log))
+  log.addHandler(handler)
+
+
+cloudlog = log = SwagLogger()
+log.setLevel(logging.DEBUG)
+
+
+class PrettyConsoleFormatter(logging.Formatter):
+  # StreamHandler writes to stderr, so tty-gate on that
+  _COLOR = sys.stderr.isatty() and os.environ.get('NO_COLOR') is None
+
+  def format(self, record):
+    msg = record.getMessage()
+    if not self._COLOR:
+      return f"{record.filename}: {msg}"
+    lvl = record.levelno
+    if   lvl >= 50: lc, ln = "\033[1;38;5;196m", "CRIT"
+    elif lvl >= 40: lc, ln = "\033[1;38;5;203m", " ERR"
+    elif lvl >= 30: lc, ln = "\033[38;5;214m", "WARN"
+    elif lvl >= 20: lc, ln = "\033[38;5;110m", "info"
+    else:           lc, ln = "\033[38;5;244m", " dbg"
+    body = f"\033[1;38;5;210m{msg}\033[0m" if lvl >= 40 else msg
+    src = "" if record.filename == "(unknown file)" else f"\033[2m{record.filename}\033[0m  "
+    return f"{lc}{ln:>4}\033[0m  {src}{body}"
+
+
+outhandler = logging.StreamHandler()
+outhandler.setFormatter(PrettyConsoleFormatter())
+
+print_level = os.environ.get('LOGPRINT', 'warning')
+if print_level == 'debug':
+  outhandler.setLevel(logging.DEBUG)
+elif print_level == 'info':
+  outhandler.setLevel(logging.INFO)
+elif print_level == 'warning':
+  outhandler.setLevel(logging.WARNING)
+
+ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
+
+log.addHandler(outhandler)
+# logs are sent through IPC before writing to disk to prevent disk I/O blocking
+log.addHandler(ipchandler)

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/time_helpers.py

@@ -0,0 +1,15 @@
+import datetime
+from pathlib import Path
+
+MIN_DATE = datetime.datetime(year=2025, month=2, day=21)
+
+def min_date():
+  # on systemd systems, the default time is the systemd build time
+  systemd_path = Path("/lib/systemd/systemd")
+  if systemd_path.exists():
+    d = datetime.datetime.fromtimestamp(systemd_path.stat().st_mtime)
+    return max(MIN_DATE, d + datetime.timedelta(days=1))
+  return MIN_DATE
+
+def system_time_valid():
+  return datetime.datetime.now() > min_date()

artifacts/iqpilot_emac_private/python/_iqclosure/iqpilot/common/utils.py

@@ -0,0 +1,271 @@
+import io
+import os
+import tempfile
+import contextlib
+import subprocess
+import time
+import functools
+from subprocess import Popen, PIPE, TimeoutExpired
+import zstandard as zstd
+
+LOG_COMPRESSION_LEVEL = 10  # little benefit up to level 15. level ~17 is a small step change
+
+class Timer:
+  """Simple lap timer for profiling sequential operations."""
+
+  def __init__(self):
+    self._start = self._lap = time.monotonic()
+    self._sections = {}
+
+  def lap(self, name):
+    now = time.monotonic()
+    self._sections[name] = now - self._lap
+    self._lap = now
+
+  @property
+  def total(self):
+    return time.monotonic() - self._start
+
+  def fmt(self, duration):
+    parts = ", ".join(f"{k}={v:.2f}s" + (f" ({duration/v:.0f}x)" if k == 'render' and v > 0 else "") for k, v in self._sections.items())
+    total = self.total
+    realtime = f"{duration/total:.1f}x realtime" if total > 0 else "N/A"
+    return f"{duration}s in {total:.1f}s ({realtime}) | {parts}"
+
+def sudo_write(val: str, path: str) -> None:
+  try:
+    with open(path, 'w') as f:
+      f.write(str(val))
+  except PermissionError:
+    os.system(f"sudo chmod a+w {path}")
+    try:
+      with open(path, 'w') as f:
+        f.write(str(val))
+    except PermissionError:
+      # fallback for debugfs files
+      os.system(f"sudo su -c 'echo {val} > {path}'")
+
+
+def sudo_read(path: str) -> str:
+  try:
+    return subprocess.check_output(f"sudo cat {path}", shell=True, encoding='utf8').strip()
+  except Exception:
+    return ""
+
+
+class MovingAverage:
+  def __init__(self, window_size: int):
+    self.window_size: int = window_size
+    self.buffer: list[float] = [0.0] * window_size
+    self.index: int = 0
+    self.count: int = 0
+    self.sum: float = 0.0
+
+  def add_value(self, new_value: float):
+    # Update the sum: subtract the value being replaced and add the new value
+    self.sum -= self.buffer[self.index]
+    self.buffer[self.index] = new_value
+    self.sum += new_value
+
+    # Update the index in a circular manner
+    self.index = (self.index + 1) % self.window_size
+
+    # Track the number of added values (for partial windows)
+    self.count = min(self.count + 1, self.window_size)
+
+  def get_average(self) -> float:
+    if self.count == 0:
+      return float('nan')
+    return self.sum / self.count
+
+
+class CallbackReader:
+  """Wraps a file, but overrides the read method to also
+  call a callback function with the number of bytes read so far."""
+
+  def __init__(self, f, callback, *args):
+    self.f = f
+    self.callback = callback
+    self.cb_args = args
+    self.total_read = 0
+
+  def __getattr__(self, attr):
+    return getattr(self.f, attr)
+
+  def read(self, *args, **kwargs):
+    chunk = self.f.read(*args, **kwargs)
+    self.total_read += len(chunk)
+    self.callback(*self.cb_args, self.total_read)
+    return chunk
+
+
+@contextlib.contextmanager
+def atomic_write(path: str, mode: str = 'w', buffering: int = -1, encoding: str | None = None, newline: str | None = None,
+                 overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
+
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
+
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)
+
+
+def get_upload_stream(filepath: str, should_compress: bool) -> tuple[io.BufferedIOBase, int]:
+  if not should_compress:
+    file_size = os.path.getsize(filepath)
+    file_stream = open(filepath, "rb")
+    return file_stream, file_size
+
+  # Compress the file on the fly
+  compressed_stream = io.BytesIO()
+  compressor = zstd.ZstdCompressor(level=LOG_COMPRESSION_LEVEL)
+
+  with open(filepath, "rb") as f:
+    compressor.copy_stream(f, compressed_stream)
+    compressed_size = compressed_stream.tell()
+    compressed_stream.seek(0)
+    return compressed_stream, compressed_size
+
+
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d
+
+
+def run_cmd(cmd: list[str], cwd=None, env=None) -> str:
+  return subprocess.check_output(cmd, encoding='utf8', cwd=cwd, env=env).strip()
+
+
+def run_cmd_default(cmd: list[str], default: str = "", cwd=None, env=None) -> str:
+  try:
+    return run_cmd(cmd, cwd=cwd, env=env)
+  except subprocess.CalledProcessError:
+    return default
+
+
+@contextlib.contextmanager
+def managed_proc(cmd: list[str], env: dict[str, str]):
+  proc = Popen(cmd, env=env, stdout=PIPE, stderr=PIPE)
+  try:
+    yield proc
+  finally:
+    if proc.poll() is None:
+      proc.terminate()
+    try:
+      proc.wait(timeout=5)
+    except TimeoutExpired:
+      proc.kill()
+
+
+def tabulate(tabular_data, headers=(), tablefmt="simple", floatfmt="g", stralign="left", numalign=None):
+  rows = [list(row) for row in tabular_data]
+
+  def fmt(val):
+    if isinstance(val, str):
+      return val
+    if isinstance(val, (bool, int)):
+      return str(val)
+    try:
+      return format(val, floatfmt)
+    except (TypeError, ValueError):
+      return str(val)
+
+  formatted = [[fmt(c) for c in row] for row in rows]
+  hdrs = [str(h) for h in headers] if headers else None
+
+  ncols = max((len(r) for r in formatted), default=0)
+  if hdrs:
+    ncols = max(ncols, len(hdrs))
+  if ncols == 0:
+    return ""
+
+  for r in formatted:
+    r.extend([""] * (ncols - len(r)))
+  if hdrs:
+    hdrs.extend([""] * (ncols - len(hdrs)))
+
+  widths = [0] * ncols
+  if hdrs:
+    for i in range(ncols):
+      widths[i] = len(hdrs[i])
+  for row in formatted:
+    for i in range(ncols):
+      widths[i] = max(widths[i], max(len(ln) for ln in row[i].split('\n')))
+
+  def _align(s, w):
+    if stralign == "center":
+      return s.center(w)
+    return s.ljust(w)
+
+  if tablefmt == "html":
+    parts = ["<table>"]
+    if hdrs:
+      parts.append("<thead>")
+      parts.append("<tr>" + "".join(f"<th>{h}</th>" for h in hdrs) + "</tr>")
+      parts.append("</thead>")
+    parts.append("<tbody>")
+    for row in formatted:
+      parts.append("<tr>" + "".join(f"<td>{c}</td>" for c in row) + "</tr>")
+    parts.append("</tbody>")
+    parts.append("</table>")
+    return "\n".join(parts)
+
+  if tablefmt == "simple_grid":
+    def _sep(left, mid, right):
+      return left + mid.join("─" * (w + 2) for w in widths) + right
+
+    top, mid_sep, bot = _sep("┌", "┬", "┐"), _sep("├", "┼", "┤"), _sep("└", "┴", "┘")
+
+    def _fmt_row(cells):
+      split = [c.split('\n') for c in cells]
+      nlines = max(len(s) for s in split)
+      for s in split:
+        s.extend([""] * (nlines - len(s)))
+      return ["│" + "│".join(f" {_align(split[i][li], widths[i])} " for i in range(ncols)) + "│" for li in range(nlines)]
+
+    lines = [top]
+    if hdrs:
+      lines.extend(_fmt_row(hdrs))
+      lines.append(mid_sep)
+    for ri, row in enumerate(formatted):
+      lines.extend(_fmt_row(row))
+      lines.append(mid_sep if ri < len(formatted) - 1 else bot)
+    return "\n".join(lines)
+
+  gap = "  "
+  lines = []
+  if hdrs:
+    lines.append(gap.join(h.ljust(w) for h, w in zip(hdrs, widths, strict=True)))
+    lines.append(gap.join("-" * w for w in widths))
+  for row in formatted:
+    lines.append(gap.join(_align(row[i], widths[i]) for i in range(ncols)))
+  return "\n".join(lines)
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          print(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        print(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator