superrob/IQ.Pilot
1
0
Fork 0
forked from IQ.Lvbs/IQ.Pilot
Code Pull Requests Activity

IQ.Pilot Release Commit @ f2a861c

Browse Source
This commit is contained in:
IQ.Lvbs CI [bot]
2026-09-02 15:07:09 -05:00
parent b42569dbca
commit e8748fd704
5497 changed files with 316070 additions and 179848 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
iqpilot/selfdrive/iqmodeld/models/runners/__init__.py
View File

@@ -1,3 +1,3 @@
"""
Runner interfaces used by iqmodeld model execution.
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
"""

64
iqpilot/selfdrive/iqmodeld/models/runners/model_runner.py
View File

@@ -7,20 +7,21 @@ from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any
import numpy as np
from cereal import custom
from openpilot.system.hardware import TICI
from openpilot.system.hardware.hw import Paths as _hw_paths
from openpilot.iqpilot.selfdrive.iqmodeld.models.helpers import get_active_bundle as _fetch_bundle
from openpilot.iqpilot.selfdrive.iqmodeld.models.combined_artifact import has_combined_split_artifact
from iqpilot.cereal import custom
from iqpilot.common.swaglog import cloudlog
from iqpilot.system.hardware import TICI
from iqpilot.system.hardware.hw import Paths as _hw_paths
from iqpilot.selfdrive.iqmodeld.models.helpers import get_active_bundle as _fetch_bundle
from iqpilot.selfdrive.iqmodeld.models.combined_artifact import has_combined_split_artifact
# ---- runtime type surface (native OpenCL/frame handles resolve to Any off-device) ----
if TYPE_CHECKING:
from openpilot.iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import GpuMemorySlot, RoadProjector
from iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import GpuMemorySlot, RoadProjector
else:
def _resolve_native_types() -> tuple[Any, Any]:
try:
from openpilot.iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import GpuMemorySlot as iq_clmem
from openpilot.iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import RoadProjector as iq_frame
from iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import GpuMemorySlot as iq_clmem
from iqpilot.selfdrive.iqmodeld.native.iqmodel_pyx import RoadProjector as iq_frame
return iq_clmem, iq_frame
except (ModuleNotFoundError, ImportError):
return Any, Any
@@ -59,11 +60,25 @@ def _configure_accelerator():
_configure_accelerator()
# real metadata pkls are a few KB; anything bigger is a model artifact wrongly
# referenced as metadata (pre-fix manifests self-referenced the artifact), and
# unpickling it here double-loads the model onto the GPU
_META_MAX_BYTES = 1 << 20
def load_artifact_metadata(metadata_filename):
"""Read one artifact's metadata pkl: (input shapes, output slices)."""
with open(os.path.join(CUSTOM_MODEL_PATH, metadata_filename), 'rb') as fh:
blob = _pk.load(fh)
return tuple(blob.get(field, {}) for field in _META_FIELDS)
try:
path = os.path.join(CUSTOM_MODEL_PATH, metadata_filename)
if os.path.getsize(path) > _META_MAX_BYTES:
cloudlog.error(f"metadata pkl {metadata_filename} is artifact-sized, refusing to unpickle it")
return tuple({} for _ in _META_FIELDS)
with open(path, 'rb') as fh:
blob = _pk.load(fh)
return tuple(blob.get(field, {}) for field in _META_FIELDS)
except Exception:
cloudlog.exception(f"unreadable metadata pkl {metadata_filename}, continuing without it")
return tuple({} for _ in _META_FIELDS)
@dataclass
@@ -114,7 +129,7 @@ class ModelRunner(RunnerRoot):
if not active:
raise ValueError("runner started without an active model bundle")
self.models = {spec.type.raw: ArtifactSpec(spec) for spec in active.models}
self.models = {spec.type.raw: ArtifactSpec(spec) for spec in _qcom_models(active)}
self.is_20hz_3d = False
self.is_20hz = active.is20hz
self.inputs = {}
@@ -165,8 +180,15 @@ class ModelRunner(RunnerRoot):
# ---- runner selection (which backend to build for the active bundle) ----------
def _qcom_models(bundle) -> list:
# usbeMac artifacts ride along in a bundle for the eGPU host; they are never
# loaded on QCOM and must not affect runner classification
return [m for m in bundle.models if m.type.raw != ModelType.usbeMac]
def _single_artifact_prefix(bundle, prefix: str) -> bool:
return len(bundle.models) == 1 and bundle.models[0].artifact.fileName.startswith(prefix)
models = _qcom_models(bundle)
return len(models) == 1 and models[0].artifact.fileName.startswith(prefix)
def _is_fused_bundle(bundle) -> bool:
@@ -178,7 +200,7 @@ def _is_supercombo_bundle(bundle) -> bool:
def _is_split_bundle(bundle) -> bool:
present = {m.type.raw for m in bundle.models}
present = {m.type.raw for m in _qcom_models(bundle)}
split_kinds = {ModelType.vision, ModelType.policy, ModelType.offPolicy, ModelType.onPolicy}
return not present.isdisjoint(split_kinds)
@@ -187,21 +209,23 @@ def get_model_runner() -> "ModelRunner":
"""Build the runner backend that fits the active bundle (supercombo / fused /
combined-split / split / single). Concrete runners are imported lazily so one
backend failing to load can't take down the others at import time."""
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.tinygrad_runner import (TinygradRunner,
from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.tinygrad_runner import (TinygradRunner,
TinygradSplitRunner)
bundle = _fetch_bundle()
if not (bundle and bundle.models):
# an eMac-only bundle (no QCOM-loadable models) runs the stock default on
# device; the big host serves the bundle's precompiled artifact
if not (bundle and bundle.models and _qcom_models(bundle)):
return TinygradRunner(ModelType.supercombo)
if _is_supercombo_bundle(bundle):
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.supercombo_runner import TinygradSupercomboRunner
from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.supercombo_runner import TinygradSupercomboRunner
return TinygradSupercomboRunner()
if _is_fused_bundle(bundle):
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.fused_runner import TinygradFusedRunner
from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.fused_runner import TinygradFusedRunner
return TinygradFusedRunner()
if _is_split_bundle(bundle) and has_combined_split_artifact(bundle):
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.combined_split_runner import TinygradCombinedSplitRunner
from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.combined_split_runner import TinygradCombinedSplitRunner
return TinygradCombinedSplitRunner()
if _is_split_bundle(bundle):
return TinygradSplitRunner()
return TinygradRunner(bundle.models[0].type.raw)
return TinygradRunner(_qcom_models(bundle)[0].type.raw)

3
iqpilot/selfdrive/iqmodeld/models/runners/onnx/__init__.py
View File

@@ -1,3 +0,0 @@
"""
ONNX runner support for iqmodeld.
"""

57
iqpilot/selfdrive/iqmodeld/models/runners/onnx/onnx_runner.py
View File

@@ -1,57 +0,0 @@
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
"""
from __future__ import annotations
import numpy as np
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import CLMemDict, FrameDict, ModelType, NumpyDict, ShapeDict
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from openpilot.iqpilot.selfdrive.iqmodeld import MODEL_PATH
from openpilot.iqpilot.selfdrive.iqmodeld.config import ModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.parser import ArchiveParser
from openpilot.iqpilot.selfdrive.iqmodeld.runtime.ort import ORT_TYPES_TO_NP_TYPES, make_onnx_cpu_runner
def _onnx_dtype_table(session) -> dict[str, np.dtype]:
return {
tensor_info.name: ORT_TYPES_TO_NP_TYPES[tensor_info.type]
for tensor_info in session.get_inputs()
}
class ONNXRunner(ModelRunner):
def __init__(self):
super().__init__()
self.runner = make_onnx_cpu_runner(MODEL_PATH)
self._constants = ModelConstants
self._model_data = self.models.get(ModelType.supercombo)
self._input_dtypes = _onnx_dtype_table(self.runner)
self._parser = ArchiveParser()
self.parser_method_dict[ModelType.supercombo] = self._parser.parse_outputs
@property
def input_shapes(self) -> ShapeDict:
return {tensor_info.name: tensor_info.shape for tensor_info in self.runner.get_inputs()}
def _frame_as_numpy(self, stream_name: str, imgs_cl: CLMemDict, frames: FrameDict) -> np.ndarray:
flattened = frames[stream_name].as_numpy(imgs_cl[stream_name])
shaped = flattened.reshape(self.input_shapes[stream_name])
return shaped.astype(self._input_dtypes[stream_name])
def prepare_inputs(self, imgs_cl: CLMemDict, numpy_inputs: NumpyDict, frames: FrameDict) -> dict:
staged_inputs = dict(numpy_inputs)
for stream_name in imgs_cl:
staged_inputs[stream_name] = self._frame_as_numpy(stream_name, imgs_cl, frames)
self.inputs = staged_inputs
return staged_inputs
def _parse_outputs(self, model_outputs: np.ndarray) -> NumpyDict:
if self._model_data is None:
raise ValueError("Model data is not available. Ensure the model is loaded correctly.")
return self.parser_method_dict[self._model_data.model.type.raw](self._slice_outputs(model_outputs))
def _run_model(self) -> NumpyDict:
combined = self.runner.run(None, self.inputs)[0].reshape(-1)
return self._parse_outputs(combined)

4
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/__init__.py
View File

@@ -1,3 +1,3 @@
"""
Tinygrad runner support for iqmodeld.
"""
Copyright © IQ.Lvbs, apart of Project Teal Lvbs, All Rights Reserved, licensed under https://konn3kt.com/tos/
"""

12
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/combined_split_runner.py
View File

@@ -11,12 +11,12 @@ from typing import Any
import numpy as np
from openpilot.iqpilot.selfdrive.iqmodeld.models.combined_artifact import resolve_combined_split_artifact
from openpilot.iqpilot.selfdrive.iqmodeld.models.helpers import get_active_bundle
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import NumpyDict, ShapeDict, SliceDict
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from openpilot.iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.parser import PhaseParser
from iqpilot.selfdrive.iqmodeld.models.combined_artifact import resolve_combined_split_artifact
from iqpilot.selfdrive.iqmodeld.models.helpers import get_active_bundle
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import NumpyDict, ShapeDict, SliceDict
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
def _tinygrad_imports():

41
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/fused_runner.py
View File

@@ -9,12 +9,12 @@ from typing import Any
import numpy as np
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import (
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import (
CUSTOM_MODEL_PATH, NumpyDict, ShapeDict, SliceDict,
)
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from openpilot.iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.parser import PhaseParser
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
def _tinygrad_imports():
@@ -27,8 +27,6 @@ WARP_DEV = os.getenv('WARP_DEV')
class TinygradFusedRunner(ModelRunner):
"""Runs a fused warp+vision+policy pkl. Bundle ships one `driving_fused_*` artifact."""
uses_opencl_warp: bool = False
def __init__(self):
@@ -110,19 +108,30 @@ class TinygradFusedRunner(ModelRunner):
'feat_q': zeros_f32((self._frame_skip * (fb[1] - 1) + 1, fb[0], fb[2])),
'desire_q': zeros_f32((self._frame_skip * dp[1], dp[0], dp[2])),
}
# shapes must match the captured run_policy JIT inputs
on_shapes = self._on_meta['input_shapes']
captured = self._run_policy.captured
jit_shapes = {
name: tuple(int(s) for s in view.shape)
for name, (view, _vars, _dtype, _device) in zip(captured.expected_names, captured.expected_input_info)
}
def policy_input_shape(name):
shape = on_shapes.get(name, jit_shapes.get(name))
if shape is None:
raise ValueError(f"fused pkl declares no shape for policy input {name}")
return shape
self._npy_buffers = {
'desire': np.zeros(dp[2], dtype=np.float32),
'traffic_convention': np.zeros(on_shapes['traffic_convention'], dtype=np.float32),
'action_t': np.zeros(on_shapes['action_t'], dtype=np.float32),
'traffic_convention': np.zeros(policy_input_shape('traffic_convention'), dtype=np.float32),
'tfm': np.zeros((3, 3), dtype=np.float32),
'big_tfm': np.zeros((3, 3), dtype=np.float32),
}
if 'action_t' in jit_shapes:
self._npy_buffers['action_t'] = np.zeros(policy_input_shape('action_t'), dtype=np.float32)
self._cam_resolution = (cam_w, cam_h)
def run_fused(self, bufs: dict, transforms: dict[str, np.ndarray], numpy_inputs: NumpyDict) -> NumpyDict:
"""warp + vision + policy in one pass from raw NV12 bufs + transform matrices."""
Tensor, Device = _tinygrad_imports()
main_buf = bufs['img']
@@ -134,14 +143,13 @@ class TinygradFusedRunner(ModelRunner):
self._npy_buffers['desire'][:] = numpy_inputs[desire_key]
if 'traffic_convention' in numpy_inputs:
self._npy_buffers['traffic_convention'][:] = numpy_inputs['traffic_convention']
if 'action_t' in numpy_inputs:
if 'action_t' in numpy_inputs and 'action_t' in self._npy_buffers:
self._npy_buffers['action_t'][:] = numpy_inputs['action_t']
self._npy_buffers['tfm'][:] = transforms['img']
self._npy_buffers['big_tfm'][:] = transforms['big_img']
npy = lambda key: Tensor(self._npy_buffers[key], device='NPY')
# frames go on the compute device to match the captured warp JIT
frame = self._frame_tensor('img', bufs['img'])
big_frame = self._frame_tensor('big_img', bufs['big_img'])
@@ -149,12 +157,13 @@ class TinygradFusedRunner(ModelRunner):
img, big_img = warp_jit(img_q=self._queues['img_q'], big_img_q=self._queues['big_img_q'],
tfm=npy('tfm'), big_tfm=npy('big_tfm'), frame=frame, big_frame=big_frame)
vision_out_t, on_out_t, off_out_t = self._run_policy(
policy_inputs = dict(
img=img, big_img=big_img, feat_q=self._queues['feat_q'], desire_q=self._queues['desire_q'],
desire=npy('desire'), traffic_convention=npy('traffic_convention'), action_t=npy('action_t'))
desire=npy('desire'), traffic_convention=npy('traffic_convention'))
if 'action_t' in self._npy_buffers:
policy_inputs['action_t'] = npy('action_t')
vision_out_t, on_out_t, off_out_t = self._run_policy(**policy_inputs)
# parse each model's output on its own sliced dict; parsing a merged dict
# would run parse_dynamic_outputs twice and double-parse plan/lead
def _slice(tensor_out, meta) -> NumpyDict:
flat = tensor_out.numpy().flatten()
return {k: flat[np.newaxis, sl] for k, sl in meta['output_slices'].items() if k != 'pad'}

6
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/model_types.py
View File

@@ -9,9 +9,9 @@ from collections.abc import Callable
import numpy as np
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelType, NumpyDict
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import RunnerRoot
from openpilot.iqpilot.selfdrive.iqmodeld.parser import ArchiveParser, PhaseParser
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelType, NumpyDict
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import RunnerRoot
from iqpilot.selfdrive.iqmodeld.parser import ArchiveParser, PhaseParser
class _ParserRole(RunnerRoot, ABC):

17
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/supercombo_runner.py
View File

@@ -12,11 +12,11 @@ from typing import Any
import numpy as np
from openpilot.common.params import Params
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import CUSTOM_MODEL_PATH, NumpyDict, ShapeDict, SliceDict
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from openpilot.iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.parser import PhaseParser
from iqpilot.common.params import Params
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import CUSTOM_MODEL_PATH, NumpyDict, ShapeDict, SliceDict
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from iqpilot.selfdrive.iqmodeld.parser import PhaseParser
def _tinygrad_imports():
@@ -68,8 +68,6 @@ def _is_jit_arg_mismatch(err: BaseException) -> bool:
class TinygradSupercomboRunner(ModelRunner):
"""Runs a single combined supercombo pkl. Bundle ships one `driving_supercombo_*` artifact."""
uses_opencl_warp: bool = False
def __init__(self):
@@ -282,7 +280,6 @@ class TinygradSupercomboRunner(ModelRunner):
zeros_u8 = lambda s: Tensor(np.zeros(s, dtype=np.uint8), device=Device.DEFAULT).contiguous().realize()
zeros_f32 = lambda s: Tensor(np.zeros(s, dtype=np.float32), device=Device.DEFAULT).contiguous().realize()
# packed npy block (single NPY tensor, mutated in place via views): order matches run_policy.split
shapes = {'desire': (dp[2],), 'traffic_convention': tuple(tc), 'action_t': tuple(at), 'prev_feat': (fb[0], fb[2])}
sizes = [math.prod(s) for s in shapes.values()]
packed = np.zeros(sum(sizes), dtype=np.float32)
@@ -318,7 +315,6 @@ class TinygradSupercomboRunner(ModelRunner):
self._npy['traffic_convention'][:] = numpy_inputs['traffic_convention']
if 'action_t' in numpy_inputs:
self._npy['action_t'][:] = numpy_inputs['action_t']
# self._npy['prev_feat'] holds last frame's hidden_state (zeros on the first frame)
frame = self._frame_tensor('img', bufs['img'])
big_frame = self._frame_tensor('big_img', bufs['big_img'])
@@ -334,11 +330,10 @@ class TinygradSupercomboRunner(ModelRunner):
raise
flat = out.numpy().flatten()
# feed hidden_state back as prev_feat for the next frame
self._npy['prev_feat'][:] = flat[self._hidden_slice].reshape(self._npy['prev_feat'].shape)
sliced = {k: flat[np.newaxis, sl] for k, sl in self._slices.items()}
return self._parser.parse_vision_outputs(sliced) # single-pass; parse_outputs double-parses a combined dict
return self._parser.parse_vision_outputs(sliced)
def _run_model(self) -> NumpyDict:
raise RuntimeError("supercombo path goes through run_fused(), not _run_model()")

18
iqpilot/selfdrive/iqmodeld/models/runners/tinygrad/tinygrad_runner.py
View File

@@ -8,9 +8,10 @@ import pickle
from dataclasses import dataclass
import numpy as np
from tinygrad.dtype import dtypes
from tinygrad.tensor import Tensor
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import (
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import (
CLMemDict,
CUSTOM_MODEL_PATH,
FrameDict,
@@ -19,18 +20,18 @@ from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import (
ShapeDict,
SliceDict,
)
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from openpilot.iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.model_types import (
from iqpilot.selfdrive.iqmodeld.models.runners.model_runner import ModelRunner
from iqpilot.selfdrive.iqmodeld.models.runners.tinygrad.model_types import (
OffPolicyTinygrad,
OnPolicyTinygrad,
PolicyTinygrad,
SupercomboTinygrad,
VisionTinygrad,
)
from openpilot.iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.config import ModelConstants
from openpilot.iqpilot.selfdrive.iqmodeld.runtime.tinygrad import qcom_tensor_from_opencl_address
from openpilot.system.hardware import TICI
from iqpilot.selfdrive.iqmodeld.models.split_model_constants import SplitModelConstants
from iqpilot.selfdrive.iqmodeld.config import ModelConstants
from iqpilot.selfdrive.iqmodeld.runtime.tinygrad import qcom_tensor_from_opencl_address
from iqpilot.system.hardware import TICI
@dataclass(frozen=True)
@@ -84,6 +85,9 @@ class TinygradRunner(ModelRunner, SupercomboTinygrad, PolicyTinygrad, VisionTiny
self.model_run = _load_program_blob(asset_name)
self._input_plan = _compile_input_plan(self.model_run.captured)
for name, spec in self._input_plan.items():
if "img" in name and spec.dtype is not dtypes.uint8:
raise ValueError(f"{asset_name}: image input {name} expects {spec.dtype}, incompatible with uint8 warp buffer")
self.input_to_dtype = {name: spec.dtype for name, spec in self._input_plan.items()}
self.input_to_device = {name: spec.device for name, spec in self._input_plan.items()}