IQ.Pilot Prebuilt Release @ 27f668a

iqpilot/selfdrive/dmonitoringmodeld/dmonitoringmodeld.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+import os
+from iqpilot.system.hardware import TICI
+os.environ['DEV'] = 'QCOM' if TICI else 'CPU'
+from tinygrad.tensor import Tensor
+from tinygrad.dtype import dtypes
+import time
+import pickle
+import numpy as np
+from pathlib import Path
+
+from iqpilot.cereal import messaging
+from iqpilot.cereal.messaging import PubMaster, SubMaster
+from iqpilot.cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionIpcClient, VisionBuf
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.common.realtime import config_realtime_process
+from iqpilot.common.transformations.model import dmonitoringmodel_intrinsics
+from iqpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
+from iqpilot.selfdrive.locationd.calibration_helpers import get_calibrated_rpy
+from iqpilot.selfdrive.dmonitoringmodeld.math import sigmoid, safe_exp
+from iqpilot.selfdrive.dmonitoringmodeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame
+from iqpilot.selfdrive.iqmodeld.runtime.tinygrad import qcom_tensor_from_opencl_address
+
+PROCESS_NAME = "selfdrive.dmonitoringmodeld.dmonitoringmodeld"
+SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
+MODEL_PKL_PATH = Path(__file__).parent / 'models/dmonitoring_model_tinygrad.pkl'
+METADATA_PATH = Path(__file__).parent / 'models/dmonitoring_model_metadata.pkl'
+
+
+class ModelState:
+  inputs: dict[str, np.ndarray]
+  output: np.ndarray
+
+  def __init__(self, cl_ctx):
+    with open(METADATA_PATH, 'rb') as f:
+      model_metadata = pickle.load(f)
+      self.input_shapes = model_metadata['input_shapes']
+      self.output_slices = model_metadata['output_slices']
+
+    self.frame = MonitoringModelFrame(cl_ctx)
+    self.numpy_inputs = {
+      'calib': np.zeros(self.input_shapes['calib'], dtype=np.float32),
+    }
+
+    self.tensor_inputs = {k: Tensor(v, device='NPY').realize() for k,v in self.numpy_inputs.items()}
+    with open(MODEL_PKL_PATH, "rb") as f:
+      self.model_run = pickle.load(f)
+
+  def run(self, buf: VisionBuf, calib: np.ndarray, transform: np.ndarray) -> tuple[np.ndarray, float]:
+    self.numpy_inputs['calib'][0,:] = calib
+
+    t1 = time.perf_counter()
+
+    input_img_cl = self.frame.prepare(buf, transform.flatten())
+    if TICI:
+      # The imgs tensors are backed by opencl memory, only need init once
+      if 'input_img' not in self.tensor_inputs:
+        self.tensor_inputs['input_img'] = qcom_tensor_from_opencl_address(input_img_cl.mem_address, self.input_shapes['input_img'], dtype=dtypes.uint8)
+    else:
+      self.tensor_inputs['input_img'] = Tensor(self.frame.buffer_from_cl(input_img_cl).reshape(self.input_shapes['input_img']), dtype=dtypes.uint8).realize()
+
+
+    output = self.model_run(**self.tensor_inputs).contiguous().realize().uop.base.buffer.numpy()
+
+    t2 = time.perf_counter()
+    return output, t2 - t1
+
+def slice_outputs(model_outputs, output_slices):
+  return  {k: model_outputs[np.newaxis, v] for k,v in output_slices.items()}
+
+def parse_model_output(model_output):
+  parsed = {}
+  parsed['wheel_on_right'] = sigmoid(model_output['wheel_on_right'])
+  for ds_suffix in ['lhd', 'rhd']:
+    face_descs = model_output[f'face_descs_{ds_suffix}']
+    parsed[f'face_descs_{ds_suffix}'] = face_descs[:, :-6]
+    parsed[f'face_descs_{ds_suffix}_std'] = safe_exp(face_descs[:, -6:])
+    for key in ['face_prob', 'left_eye_prob', 'right_eye_prob','left_blink_prob', 'right_blink_prob', 'sunglasses_prob', 'using_phone_prob']:
+      parsed[f'{key}_{ds_suffix}'] = sigmoid(model_output[f'{key}_{ds_suffix}'])
+  return parsed
+
+def fill_driver_data(msg, model_output, ds_suffix):
+  msg.faceOrientation = model_output[f'face_descs_{ds_suffix}'][0, :3].tolist()
+  msg.faceOrientationStd = model_output[f'face_descs_{ds_suffix}_std'][0, :3].tolist()
+  msg.facePosition = model_output[f'face_descs_{ds_suffix}'][0, 3:5].tolist()
+  msg.facePositionStd = model_output[f'face_descs_{ds_suffix}_std'][0, 3:5].tolist()
+  msg.faceProb = model_output[f'face_prob_{ds_suffix}'][0, 0].item()
+  msg.leftEyeProb = model_output[f'left_eye_prob_{ds_suffix}'][0, 0].item()
+  msg.rightEyeProb = model_output[f'right_eye_prob_{ds_suffix}'][0, 0].item()
+  msg.leftBlinkProb = model_output[f'left_blink_prob_{ds_suffix}'][0, 0].item()
+  msg.rightBlinkProb = model_output[f'right_blink_prob_{ds_suffix}'][0, 0].item()
+  msg.sunglassesProb = model_output[f'sunglasses_prob_{ds_suffix}'][0, 0].item()
+  msg.phoneProb = model_output[f'using_phone_prob_{ds_suffix}'][0, 0].item()
+
+def get_driverstate_packet(model_output, frame_id: int, location_ts: int, exec_time: float, gpu_exec_time: float):
+  msg = messaging.new_message('driverStateV2', valid=True)
+  ds = msg.driverStateV2
+  ds.frameId = frame_id
+  ds.modelExecutionTime = exec_time
+  ds.gpuExecutionTime = gpu_exec_time
+  ds.rawPredictions = model_output['raw_pred']
+  ds.wheelOnRightProb = model_output['wheel_on_right'][0, 0].item()
+  fill_driver_data(ds.leftDriverData, model_output, 'lhd')
+  fill_driver_data(ds.rightDriverData, model_output, 'rhd')
+  return msg
+
+
+def main():
+  config_realtime_process(7, 5)
+
+  # Set in the child, not at import: manager preimports every process module in the parent,
+  # so an import-time write lands in one shared env that all children inherit (and setdefault
+  # in a child is then a guaranteed no-op). tinygrad reads this lazily at QCOMDevice init.
+  # KGSL: lower value = higher priority. DM has no 50ms deadline; at the driving contexts'
+  # default 8 its kernels interleave with the warp and blow its submit tail 16ms -> 72ms p90.
+  os.environ['QCOM_PRIORITY'] = os.getenv('DMON_QCOM_PRIORITY', '12')
+
+  cl_context = CLContext()
+  model = ModelState(cl_context)
+  cloudlog.warning("models loaded, dmonitoringmodeld starting")
+
+  cloudlog.warning("connecting to driver stream")
+  vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context)
+  while not vipc_client.connect(False):
+    time.sleep(0.1)
+  assert vipc_client.is_connected()
+  cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}")
+
+  sm = SubMaster(["extrinsicsCalibration"])
+  pm = PubMaster(["driverStateV2"])
+
+  calib = np.zeros(model.numpy_inputs['calib'].size, dtype=np.float32)
+  model_transform = None
+
+  while True:
+    buf = vipc_client.recv()
+    if buf is None:
+      continue
+
+    if model_transform is None:
+      cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye
+      model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32)
+
+    sm.update(0)
+    if sm.updated["extrinsicsCalibration"]:
+      calib_rpy = get_calibrated_rpy(sm["extrinsicsCalibration"])
+      calib[:] = calib_rpy if calib_rpy is not None else np.zeros_like(calib)
+
+    t1 = time.perf_counter()
+    model_output, gpu_execution_time = model.run(buf, calib, model_transform)
+    t2 = time.perf_counter()
+    raw_pred = model_output.tobytes() if SEND_RAW_PRED else b''
+    model_output = slice_outputs(model_output, model.output_slices)
+    model_output = parse_model_output(model_output)
+    model_output['raw_pred'] = raw_pred
+    msg = get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time)
+    pm.send("driverStateV2", msg)
+
+
+if __name__ == "__main__":
+  try:
+    main()
+  except KeyboardInterrupt:
+    cloudlog.warning("got SIGINT")

iqpilot/selfdrive/dmonitoringmodeld/get_model_metadata.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import sys
+import pathlib
+import onnx
+import codecs
+import pickle
+from typing import Any
+
+def get_name_and_shape(value_info:onnx.ValueInfoProto) -> tuple[str, tuple[int,...]]:
+  shape = tuple([int(dim.dim_value) for dim in value_info.type.tensor_type.shape.dim])
+  name = value_info.name
+  return name, shape
+
+def get_metadata_value_by_name(model:onnx.ModelProto, name:str) -> str | Any:
+  for prop in model.metadata_props:
+    if prop.key == name:
+      return prop.value
+  return None
+
+if __name__ == "__main__":
+  model_path = pathlib.Path(sys.argv[1])
+  model = onnx.load(str(model_path))
+  output_slices = get_metadata_value_by_name(model, 'output_slices')
+  assert output_slices is not None, 'output_slices not found in metadata'
+
+  metadata = {
+    'model_checkpoint': get_metadata_value_by_name(model, 'model_checkpoint'),
+    'output_slices': pickle.loads(codecs.decode(output_slices.encode(), "base64")),
+    'input_shapes': dict([get_name_and_shape(x) for x in model.graph.input]),
+    'output_shapes': dict([get_name_and_shape(x) for x in model.graph.output])
+  }
+
+  metadata_path = model_path.parent / (model_path.stem + '_metadata.pkl')
+  with open(metadata_path, 'wb') as f:
+    pickle.dump(metadata, f)
+
+  print(f'saved metadata to {metadata_path}')

iqpilot/selfdrive/dmonitoringmodeld/math.py

@@ -0,0 +1,9 @@
+import numpy as np
+
+
+def safe_exp(values, out=None):
+  return np.exp(np.clip(values, -np.inf, 11), out=out)
+
+
+def sigmoid(values):
+  return 1.0 / (1.0 + safe_exp(-values))

iqpilot/selfdrive/dmonitoringmodeld/models/prebuilt_check.json

@@ -0,0 +1,9 @@
+{
+  "dmonitoring_model": {
+    "outputs": {
+      "dmonitoring_model_metadata.pkl": "31a86ab7a92dc0af088b15787a440dd3b210aa662e445a15145900e559a1b5c3",
+      "dmonitoring_model_tinygrad.pkl": "72757faf4828b7b574b9090c299fcad3e80d56ecfb4f5afa30d2355e556fb9c1"
+    },
+    "signature": "02f2a397f74dad0300afbbf9d4f68f253c9793c52d1464b56acad2a3c7a6d364"
+  }
+}

iqpilot/selfdrive/dmonitoringmodeld/prebuilt_models.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+import functools
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+
+MODELD_DIR = Path(__file__).resolve().parent
+MODELS_DIR = MODELD_DIR / 'models'
+BASEDIR = MODELD_DIR.parents[2]
+METADATA_SCRIPT = MODELD_DIR / 'get_model_metadata.py'
+PYPROJECT = BASEDIR / 'pyproject.toml'
+TINYGRAD_REVISION_FILE = BASEDIR / 'artifacts/package_sources/tinygrad/.iqpilot-revision'
+
+MODEL_NAMES = ['dmonitoring_model']
+
+
+def _hash_file(h, path: Path) -> None:
+  with open(path, 'rb') as f:
+    for chunk in iter(lambda: f.read(1024 * 1024), b''):
+      h.update(chunk)
+
+
+def _file_sha256(path: Path) -> str:
+  h = hashlib.sha256()
+  _hash_file(h, path)
+  return h.hexdigest()
+
+
+@functools.lru_cache(maxsize=1)
+def _tinygrad_revision() -> str:
+  match = re.search(r'"tinygrad @ git\+https://[^@]+@([0-9a-f]{40})"', PYPROJECT.read_text())
+  if match is not None:
+    return match.group(1)
+
+  try:
+    revision = TINYGRAD_REVISION_FILE.read_text().strip()
+  except OSError as e:
+    raise RuntimeError("missing pinned tinygrad revision") from e
+  if re.fullmatch(r'[0-9a-f]{40}', revision) is None:
+    raise RuntimeError("invalid pinned tinygrad revision")
+  return revision
+
+
+CHECK_PATH = MODELS_DIR / 'prebuilt_check.json'
+
+
+def _load_checks() -> dict:
+  try:
+    data = json.loads(CHECK_PATH.read_text())
+  except (json.JSONDecodeError, OSError):
+    return {}
+  return data if isinstance(data, dict) else {}
+
+
+def _output_names(model_name: str) -> list[str]:
+  return [f'{model_name}_tinygrad.pkl', f'{model_name}_metadata.pkl']
+
+
+def compute_signature(model_name: str, flags: str) -> str:
+  h = hashlib.sha256()
+  h.update(flags.encode())
+  h.update(_tinygrad_revision().encode())
+  _hash_file(h, METADATA_SCRIPT)
+  _hash_file(h, MODELS_DIR / f'{model_name}.onnx')
+  return h.hexdigest()
+
+
+def outputs_match(model_name: str) -> bool:
+  """The committed artifacts on disk are exactly the ones the check file pins."""
+  data = _load_checks().get(model_name, {})
+  outputs = data.get('outputs', {})
+  if set(outputs) != set(_output_names(model_name)):
+    return False
+  for fn, expected in outputs.items():
+    p = MODELS_DIR / fn
+    if not p.is_file() or _file_sha256(p) != expected:
+      return False
+  return True
+
+
+def packaged_prebuilt_matches(model_name: str) -> bool:
+  return not (MODELS_DIR / f'{model_name}.onnx').is_file() and outputs_match(model_name)
+
+
+def verify_prebuilt(model_name: str, flags: str) -> bool:
+  if not (MODELS_DIR / f'{model_name}.onnx').is_file():
+    return False
+  data = _load_checks().get(model_name, {})
+  if data.get('signature') != compute_signature(model_name, flags):
+    return False
+  return outputs_match(model_name)
+
+
+def verification_details(model_name: str, flags: str) -> list[str]:
+  data = _load_checks().get(model_name, {})
+  details = [
+    f'signature expected={data.get("signature", "missing")} actual={compute_signature(model_name, flags)}',
+    f'tinygrad={_tinygrad_revision()}',
+    f'onnx={_file_sha256(MODELS_DIR / f"{model_name}.onnx")}',
+    f'metadata_script={_file_sha256(METADATA_SCRIPT)}',
+  ]
+  outputs = data.get('outputs', {})
+  for fn in _output_names(model_name):
+    path = MODELS_DIR / fn
+    actual = _file_sha256(path) if path.is_file() else 'missing'
+    details.append(f'{fn} expected={outputs.get(fn, "missing")} actual={actual}')
+  return details
+
+
+def write_check(model_name: str, flags: str) -> None:
+  outputs = {}
+  for fn in _output_names(model_name):
+    p = MODELS_DIR / fn
+    if not p.is_file():
+      raise FileNotFoundError(f'missing build output: {p}')
+    outputs[fn] = _file_sha256(p)
+  checks = _load_checks()
+  checks[model_name] = {'signature': compute_signature(model_name, flags), 'outputs': outputs}
+  CHECK_PATH.write_text(json.dumps(checks, indent=2, sort_keys=True) + '\n')
+
+
+def _larch64_flags() -> str:
+  return "DEV=QCOM IMAGE=2 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
+
+
+if __name__ == '__main__':
+  mode = sys.argv[1] if len(sys.argv) > 1 else 'verify'
+  flags = _larch64_flags()
+  for name in MODEL_NAMES:
+    if mode == 'write':
+      write_check(name, flags)
+      print(f'{name}: check written')
+    else:
+      valid = verify_prebuilt(name, flags)
+      print(f'{name}: {"OK" if valid else "STALE"}')
+      if not valid:
+        for detail in verification_details(name, flags):
+          print(f'  {detail}')

iqpilot/selfdrive/dmonitoringmodeld/test_prebuilt_models.py

@@ -0,0 +1,65 @@
+import hashlib
+import json
+
+from iqpilot.selfdrive.dmonitoringmodeld import prebuilt_models
+
+
+def write_outputs(models_dir, check_path):
+  outputs = {}
+  for name, contents in {
+    'dmonitoring_model_tinygrad.pkl': b'tinygrad',
+    'dmonitoring_model_metadata.pkl': b'metadata',
+  }.items():
+    (models_dir / name).write_bytes(contents)
+    outputs[name] = hashlib.sha256(contents).hexdigest()
+  check_path.write_text(json.dumps({'dmonitoring_model': {'outputs': outputs}}))
+
+
+def test_packaged_prebuilt_without_onnx(tmp_path, monkeypatch):
+  models_dir = tmp_path / 'models'
+  models_dir.mkdir()
+  check_path = models_dir / 'prebuilt_check.json'
+  write_outputs(models_dir, check_path)
+  monkeypatch.setattr(prebuilt_models, 'MODELS_DIR', models_dir)
+  monkeypatch.setattr(prebuilt_models, 'CHECK_PATH', check_path)
+
+  assert prebuilt_models.packaged_prebuilt_matches('dmonitoring_model')
+  assert not prebuilt_models.verify_prebuilt('dmonitoring_model', 'flags')
+
+
+def test_packaged_prebuilt_rejects_corrupt_output(tmp_path, monkeypatch):
+  models_dir = tmp_path / 'models'
+  models_dir.mkdir()
+  check_path = models_dir / 'prebuilt_check.json'
+  write_outputs(models_dir, check_path)
+  (models_dir / 'dmonitoring_model_tinygrad.pkl').write_bytes(b'corrupt')
+  monkeypatch.setattr(prebuilt_models, 'MODELS_DIR', models_dir)
+  monkeypatch.setattr(prebuilt_models, 'CHECK_PATH', check_path)
+
+  assert not prebuilt_models.packaged_prebuilt_matches('dmonitoring_model')
+
+
+def test_source_checkout_is_not_packaged_prebuilt(tmp_path, monkeypatch):
+  models_dir = tmp_path / 'models'
+  models_dir.mkdir()
+  check_path = models_dir / 'prebuilt_check.json'
+  write_outputs(models_dir, check_path)
+  (models_dir / 'dmonitoring_model.onnx').write_bytes(b'onnx')
+  monkeypatch.setattr(prebuilt_models, 'MODELS_DIR', models_dir)
+  monkeypatch.setattr(prebuilt_models, 'CHECK_PATH', check_path)
+
+  assert not prebuilt_models.packaged_prebuilt_matches('dmonitoring_model')
+
+
+def test_vendored_tinygrad_revision(tmp_path, monkeypatch):
+  revision = '0123456789abcdef0123456789abcdef01234567'
+  pyproject = tmp_path / 'pyproject.toml'
+  revision_file = tmp_path / '.iqpilot-revision'
+  pyproject.write_text('dependencies = ["tinygrad"]\n')
+  revision_file.write_text(f'{revision}\n')
+  monkeypatch.setattr(prebuilt_models, 'PYPROJECT', pyproject)
+  monkeypatch.setattr(prebuilt_models, 'TINYGRAD_REVISION_FILE', revision_file)
+  prebuilt_models._tinygrad_revision.cache_clear()
+
+  assert prebuilt_models._tinygrad_revision() == revision
+  prebuilt_models._tinygrad_revision.cache_clear()

iqpilot/selfdrive/dmonitoringmodeld/transforms/transform.cl

@@ -0,0 +1,54 @@
+#define INTER_BITS 5
+#define INTER_TAB_SIZE (1 << INTER_BITS)
+#define INTER_SCALE 1.f / INTER_TAB_SIZE
+
+#define INTER_REMAP_COEF_BITS 15
+#define INTER_REMAP_COEF_SCALE (1 << INTER_REMAP_COEF_BITS)
+
+__kernel void warpPerspective(__global const uchar * src,
+                              int src_row_stride, int src_px_stride, int src_offset, int src_rows, int src_cols,
+                              __global uchar * dst,
+                              int dst_row_stride, int dst_offset, int dst_rows, int dst_cols,
+                              __constant float * M)
+{
+    int dx = get_global_id(0);
+    int dy = get_global_id(1);
+
+    if (dx < dst_cols && dy < dst_rows)
+    {
+        float X0 = M[0] * dx + M[1] * dy + M[2];
+        float Y0 = M[3] * dx + M[4] * dy + M[5];
+        float W = M[6] * dx + M[7] * dy + M[8];
+        W = W != 0.0f ? INTER_TAB_SIZE / W : 0.0f;
+        int X = rint(X0 * W), Y = rint(Y0 * W);
+
+        int sx = convert_short_sat(X >> INTER_BITS);
+        int sy = convert_short_sat(Y >> INTER_BITS);
+
+        short sx_clamp = clamp(sx, 0, src_cols - 1);
+        short sx_p1_clamp = clamp(sx + 1, 0, src_cols - 1);
+        short sy_clamp = clamp(sy, 0, src_rows - 1);
+        short sy_p1_clamp = clamp(sy + 1, 0, src_rows - 1);
+        int v0 = convert_int(src[mad24(sy_clamp, src_row_stride, src_offset + sx_clamp*src_px_stride)]);
+        int v1 = convert_int(src[mad24(sy_clamp, src_row_stride, src_offset + sx_p1_clamp*src_px_stride)]);
+        int v2 = convert_int(src[mad24(sy_p1_clamp, src_row_stride, src_offset + sx_clamp*src_px_stride)]);
+        int v3 = convert_int(src[mad24(sy_p1_clamp, src_row_stride, src_offset + sx_p1_clamp*src_px_stride)]);
+
+        short ay = (short)(Y & (INTER_TAB_SIZE - 1));
+        short ax = (short)(X & (INTER_TAB_SIZE - 1));
+        float taby = 1.f/INTER_TAB_SIZE*ay;
+        float tabx = 1.f/INTER_TAB_SIZE*ax;
+
+        int dst_index = mad24(dy, dst_row_stride, dst_offset + dx);
+
+        int itab0 = convert_short_sat_rte( (1.0f-taby)*(1.0f-tabx) * INTER_REMAP_COEF_SCALE );
+        int itab1 = convert_short_sat_rte( (1.0f-taby)*tabx * INTER_REMAP_COEF_SCALE );
+        int itab2 = convert_short_sat_rte( taby*(1.0f-tabx) * INTER_REMAP_COEF_SCALE );
+        int itab3 = convert_short_sat_rte( taby*tabx * INTER_REMAP_COEF_SCALE );
+
+        int val = v0 * itab0 +  v1 * itab1 + v2 * itab2 + v3 * itab3;
+
+        uchar pix = convert_uchar_sat((val + (1 << (INTER_REMAP_COEF_BITS-1))) >> INTER_REMAP_COEF_BITS);
+        dst[dst_index] = pix;
+    }
+}