Files
IQ.Pilot/iqpilot/selfdrive/ui/mici/onroad/model_renderer.py
2026-08-22 23:42:41 -05:00

503 lines
19 KiB
Python

import colorsys
import numpy as np
import pyray as rl
from iqpilot.cereal import messaging, car
from dataclasses import dataclass, field
from iqpilot.common.params import Params
from iqpilot.common.filter_simple import FirstOrderFilter
from iqpilot.selfdrive.locationd.calibrationd import HEIGHT_INIT
from iqpilot.ui.onroad.hud_overlays import ChevronMetrics
from iqpilot.ui.onroad.lead_confidence import driving_confidence
from iqpilot.selfdrive.locationd.calibration_helpers import get_render_path_height
from iqpilot.selfdrive.ui.ui_state import ui_state, UIStatus
from iqpilot.selfdrive.ui.mici.onroad import blend_colors
from iqpilot.system.ui.lib.application import gui_app
from iqpilot.system.ui.lib.shader_polygon import draw_polygon, Gradient
from iqpilot.system.ui.widgets import Widget
CLIP_MARGIN = 500
MIN_DRAW_DISTANCE = 10.0
MAX_DRAW_DISTANCE = 100.0
THROTTLE_COLORS = [
rl.Color(13, 248, 122, 102), # HSLF(148/360, 0.94, 0.51, 0.4)
rl.Color(114, 255, 92, 89), # HSLF(112/360, 1.0, 0.68, 0.35)
rl.Color(114, 255, 92, 0), # HSLF(112/360, 1.0, 0.68, 0.0)
]
NO_THROTTLE_COLORS = [
rl.Color(242, 242, 242, 102), # HSLF(148/360, 0.0, 0.95, 0.4)
rl.Color(242, 242, 242, 89), # HSLF(112/360, 0.0, 0.95, 0.35)
rl.Color(242, 242, 242, 0), # HSLF(112/360, 0.0, 0.95, 0.0)
]
LANE_LINE_COLORS = {
UIStatus.DISENGAGED: rl.Color(200, 200, 200, 255),
UIStatus.OVERRIDE: rl.Color(255, 255, 255, 255),
UIStatus.ENGAGED: rl.Color(0, 255, 64, 255),
}
@dataclass
class ModelPoints:
raw_points: np.ndarray = field(default_factory=lambda: np.empty((0, 3), dtype=np.float32))
projected_points: np.ndarray = field(default_factory=lambda: np.empty((0, 2), dtype=np.float32))
@dataclass
class LeadVehicle:
center: tuple[float, float] | None = None
radius: float = 0.0
sz: float = 0.0
fill_alpha: int = 0
class ModelRenderer(Widget):
def __init__(self):
super().__init__()
self.chevron_metrics = ChevronMetrics()
self._lead_orb = gui_app.texture("icons/lead_orb.png", 256, 256)
self._longitudinal_control = False
self._experimental_mode = False
self._blend_filter = FirstOrderFilter(1.0, 0.25, 1 / gui_app.target_fps)
self._prev_allow_throttle = True
self._lane_line_probs = np.zeros(4, dtype=np.float32)
self._road_edge_stds = np.zeros(2, dtype=np.float32)
self._lead_vehicles = [LeadVehicle(), LeadVehicle()]
self._path_offset_z = HEIGHT_INIT[0]
# Initialize ModelPoints objects
self._path = ModelPoints()
self._lane_lines = [ModelPoints() for _ in range(4)]
self._road_edges = [ModelPoints() for _ in range(2)]
self._acceleration_x = np.empty((0,), dtype=np.float32)
self._acceleration_x_filter = FirstOrderFilter(0.0, 0.1, 1 / gui_app.target_fps)
self._acceleration_x_filter2 = FirstOrderFilter(0.0, 1, 1 / gui_app.target_fps)
self._torque_filter = FirstOrderFilter(0, 0.1, 1 / gui_app.target_fps)
self._ll_color_filter = FirstOrderFilter(0.0, 0.1, 1 / gui_app.target_fps)
# Transform matrix (3x3 for car space to screen space)
self._car_space_transform = np.zeros((3, 3), dtype=np.float32)
self._transform_dirty = True
self._clip_region = None
self._counter = -1
self._camera_offset = ui_state.params.get("CameraOffset", return_default=True) if ui_state.active_bundle else 0.0
self._exp_gradient = Gradient(
start=(0.0, 1.0), # Bottom of path
end=(0.0, 0.0), # Top of path
colors=[],
stops=[],
)
# Get longitudinal control setting from car parameters
if car_params := Params().get("CarParams"):
cp = messaging.log_from_bytes(car_params, car.CarParams)
self._longitudinal_control = cp.openpilotLongitudinalControl
def set_transform(self, transform: np.ndarray):
self._car_space_transform = transform.astype(np.float32)
self._transform_dirty = True
def _render(self, rect: rl.Rectangle):
sm = ui_state.sm
driving_confidence.update()
if self._counter % 180 == 0: # This runs at 60fps, so we query every 3 seconds
self._camera_offset = ui_state.params.get("CameraOffset", return_default=True) if ui_state.active_bundle else 0.0
self._counter += 1
self._torque_filter.update(-ui_state.sm['carOutput'].actuatorsOutput.torque)
# Check if data is up-to-date
if (sm.recv_frame["extrinsicsCalibration"] < ui_state.started_frame or
sm.recv_frame["modelV2"] < ui_state.started_frame):
return
# Set up clipping region
self._clip_region = rl.Rectangle(
rect.x - CLIP_MARGIN, rect.y - CLIP_MARGIN, rect.width + 2 * CLIP_MARGIN, rect.height + 2 * CLIP_MARGIN
)
# Update state
self._experimental_mode = sm['selfdriveState'].experimentalMode
live_calib = sm['extrinsicsCalibration']
self._path_offset_z = get_render_path_height(live_calib)
if sm.updated['carParams']:
self._longitudinal_control = sm['carParams'].openpilotLongitudinalControl
model = sm['modelV2']
radar_state = sm['radarState'] if sm.valid['radarState'] else None
lead_one = radar_state.leadOne if radar_state else None
render_lead_indicator = self._longitudinal_control and radar_state is not None
# Update model data when needed
model_updated = sm.updated['modelV2']
if model_updated or sm.updated['radarState'] or self._transform_dirty:
if model_updated:
self._update_raw_points(model)
path_x_array = self._path.raw_points[:, 0]
if path_x_array.size == 0:
return
self._update_model(lead_one, path_x_array)
if render_lead_indicator:
self._update_leads(radar_state, path_x_array)
self._transform_dirty = False
# Draw elements (hide when disengaged)
if ui_state.status != UIStatus.DISENGAGED:
self._draw_lane_lines()
self._draw_path(sm)
if render_lead_indicator and radar_state:
self._draw_lead_indicator()
self.chevron_metrics.draw_lead_status(sm, radar_state, self._rect, self._lead_vehicles)
def _update_raw_points(self, model):
"""Update raw 3D points from model data"""
self._path.raw_points = np.array([model.position.x, np.array(model.position.y) + self._camera_offset, model.position.z], dtype=np.float32).T
for i, lane_line in enumerate(model.laneLines):
self._lane_lines[i].raw_points = np.array([lane_line.x, np.array(lane_line.y) + self._camera_offset, lane_line.z], dtype=np.float32).T
for i, road_edge in enumerate(model.roadEdges):
self._road_edges[i].raw_points = np.array([road_edge.x, np.array(road_edge.y) + self._camera_offset, road_edge.z], dtype=np.float32).T
self._lane_line_probs = np.array(model.laneLineProbs, dtype=np.float32)
self._road_edge_stds = np.array(model.roadEdgeStds, dtype=np.float32)
self._acceleration_x = np.array(model.acceleration.x, dtype=np.float32)
def _update_leads(self, radar_state, path_x_array):
"""Update positions of lead vehicles"""
self._lead_vehicles = [LeadVehicle(), LeadVehicle()]
leads = [radar_state.leadOne, radar_state.leadTwo]
for i, lead_data in enumerate(leads):
if lead_data and lead_data.status:
d_rel, y_rel, v_rel = lead_data.dRel, lead_data.yRel, lead_data.vRel
idx = self._get_path_length_idx(path_x_array, d_rel)
# Get z-coordinate from path at the lead vehicle position
z = self._path.raw_points[idx, 2] if idx < len(self._path.raw_points) else 0.0
point = self._map_to_screen(d_rel, -y_rel + self._camera_offset, z + self._path_offset_z)
if point:
self._lead_vehicles[i] = self._update_lead_vehicle(d_rel, v_rel, point, self._rect)
def _update_model(self, lead, path_x_array):
"""Update model visualization data based on model message"""
max_distance = np.clip(path_x_array[-1], MIN_DRAW_DISTANCE, MAX_DRAW_DISTANCE)
max_idx = self._get_path_length_idx(self._lane_lines[0].raw_points[:, 0], max_distance)
# Update lane lines using raw points
line_width_factor = 0.12
for i, lane_line in enumerate(self._lane_lines):
if i in (1, 2):
line_width_factor = 0.16
lane_line.projected_points = self._map_line_to_polygon(
lane_line.raw_points, line_width_factor * self._lane_line_probs[i], 0.0, max_idx
)
# Update road edges using raw points
for road_edge in self._road_edges:
road_edge.projected_points = self._map_line_to_polygon(road_edge.raw_points, line_width_factor, 0.0, max_idx)
# Update path using raw points
if lead and lead.status:
lead_d = lead.dRel * 2.0
max_distance = np.clip(lead_d - min(lead_d * 0.35, 10.0), 0.0, max_distance)
soon_acceleration = self._acceleration_x[len(self._acceleration_x) // 4] if len(self._acceleration_x) > 0 else 0
self._acceleration_x_filter.update(soon_acceleration)
self._acceleration_x_filter2.update(soon_acceleration)
# make path width wider/thinner when initially braking/accelerating
if self._experimental_mode and False:
high_pass_acceleration = self._acceleration_x_filter.x - self._acceleration_x_filter2.x
y_off = np.interp(high_pass_acceleration, [-1, 0, 1], [0.9 * 2, 0.9, 0.9 / 2])
else:
y_off = 0.9
max_idx = self._get_path_length_idx(path_x_array, max_distance)
self._path.projected_points = self._map_line_to_polygon(
self._path.raw_points, y_off, self._path_offset_z, max_idx, allow_invert=False
)
self._update_experimental_gradient()
def _update_experimental_gradient(self):
"""Pre-calculate experimental mode gradient colors"""
if not self._experimental_mode:
return
# reconstruct absolute (screen) points so the rect-space cull below stays correct
path_pts = self._path.projected_points + np.array([self._rect.x, self._rect.y], dtype=np.float32)
max_len = min(len(path_pts) // 2, len(self._acceleration_x))
segment_colors = []
gradient_stops = []
i = 0
while i < max_len:
# Some points (screen space) are out of frame (rect space)
track_y = path_pts[i][1]
if track_y < self._rect.y or track_y > (self._rect.y + self._rect.height):
i += 1
continue
# Calculate color based on acceleration (0 is bottom, 1 is top)
lin_grad_point = 1 - (track_y - self._rect.y) / self._rect.height
# speed up: 120, slow down: 0
path_hue = np.clip(60 + self._acceleration_x[i] * 35, 0, 120)
saturation = min(abs(self._acceleration_x[i] * 1.5), 1)
lightness = np.interp(saturation, [0.0, 1.0], [0.95, 0.62])
alpha = np.interp(lin_grad_point, [0.75 / 2.0, 0.75], [0.4, 0.0])
# Use HSL to RGB conversion
color = self._hsla_to_color(path_hue / 360.0, saturation, lightness, alpha)
gradient_stops.append(lin_grad_point)
segment_colors.append(color)
# Skip a point, unless next is last
i += 1 + (1 if (i + 2) < max_len else 0)
# Store the gradient in the path object
self._exp_gradient.colors = segment_colors
self._exp_gradient.stops = gradient_stops
def _update_lead_vehicle(self, d_rel, v_rel, point, rect):
speed_buff, lead_buff = 10.0, 40.0
# Calculate fill alpha
fill_alpha = 0
if d_rel < lead_buff:
fill_alpha = 255 * (1.0 - (d_rel / lead_buff))
if v_rel < 0:
fill_alpha += 255 * (-1 * (v_rel / speed_buff))
fill_alpha = min(fill_alpha, 255)
# Calculate size and position. Distance-scaled orb radius (closer lead -> bigger orb).
sz = np.clip((25 * 30) / (d_rel / 3 + 30), 15.0, 30.0) * 1
radius = sz * 1.1
# point is in absolute screen coords; clamp against the rect's absolute bounds so the orb stays
# fully on-screen (rect-relative bounds mis-placed it when the camera pane is offset, e.g. split nav)
x = np.clip(point[0], rect.x + radius, rect.x + rect.width - radius)
y = np.clip(point[1], rect.y + radius, rect.y + rect.height - radius)
return LeadVehicle(center=(float(x), float(y)), radius=float(radius), sz=float(sz), fill_alpha=int(fill_alpha))
def _get_ll_color(self, prob: float, adjacent: bool, left: bool):
alpha = np.clip(prob, 0.0, 0.7)
if adjacent:
_base_color = LANE_LINE_COLORS.get(ui_state.status, LANE_LINE_COLORS[UIStatus.DISENGAGED])
color = rl.Color(_base_color.r, _base_color.g, _base_color.b, int(alpha * 255))
# turn adjacent lls orange if torque is high
torque = self._torque_filter.x
high_torque = abs(torque) > 0.6
if high_torque and (left == (torque > 0)):
color = blend_colors(
color,
rl.Color(255, 115, 0, int(alpha * 255)), # orange
np.interp(abs(torque), [0.6, 0.8], [0.0, 1.0])
)
else:
color = rl.Color(255, 255, 255, int(alpha * 255))
if ui_state.status == UIStatus.DISENGAGED:
color = rl.Color(0, 0, 0, int(alpha * 255))
return color
def _draw_lane_lines(self):
"""Draw lane lines and road edges"""
"""Two closest lines should be green (lane line or road edges)"""
# projected_points are origin-relative (rect.x/y kept out of the transform so it stays cached);
# translate to the view's screen position here.
offset = np.array([self._rect.x, self._rect.y], dtype=np.float32)
for i, lane_line in enumerate(self._lane_lines):
if lane_line.projected_points.size == 0:
continue
color = self._get_ll_color(float(self._lane_line_probs[i]), i in (1, 2), i in (0, 1))
draw_polygon(self._rect, lane_line.projected_points + offset, color)
for i, road_edge in enumerate(self._road_edges):
if road_edge.projected_points.size == 0:
continue
# if closest lane lines are not confident, make road edges green
color = self._get_ll_color(float(1.0 - self._road_edge_stds[i]), float(self._lane_line_probs[i + 1]) < 0.25, i == 0)
draw_polygon(self._rect, road_edge.projected_points + offset, color)
def _draw_path(self, sm):
"""Draw path with dynamic coloring based on mode and throttle state."""
if not self._path.projected_points.size:
return
# projected_points are origin-relative; translate to the view's screen position
path_pts = self._path.projected_points + np.array([self._rect.x, self._rect.y], dtype=np.float32)
allow_throttle = sm['longitudinalPlan'].allowThrottle or not self._longitudinal_control
self._blend_filter.update(int(allow_throttle))
if self._experimental_mode:
# Draw with acceleration coloring
if ui_state.status == UIStatus.DISENGAGED:
draw_polygon(self._rect, path_pts, rl.Color(0, 0, 0, 90))
elif len(self._exp_gradient.colors) > 1:
draw_polygon(self._rect, path_pts, gradient=self._exp_gradient)
else:
draw_polygon(self._rect, path_pts, rl.Color(255, 255, 255, 30))
else:
# Blend throttle/no throttle colors based on transition
blend_factor = round(self._blend_filter.x * 100) / 100
blended_colors = self._blend_colors(NO_THROTTLE_COLORS, THROTTLE_COLORS, blend_factor)
gradient = Gradient(
start=(0.0, 1.0), # Bottom of path
end=(0.0, 0.0), # Top of path
colors=blended_colors,
stops=[0.0, 0.5, 1.0],
)
if ui_state.status == UIStatus.DISENGAGED:
draw_polygon(self._rect, path_pts, rl.Color(0, 0, 0, 90))
else:
draw_polygon(self._rect, path_pts, gradient=gradient)
def _draw_lead_indicator(self):
tint, _ = driving_confidence.colors()
src = rl.Rectangle(0, 0, self._lead_orb.width, self._lead_orb.height)
for lead in self._lead_vehicles:
if lead.center is None:
continue
cx, cy = lead.center
r = lead.radius
alpha = int(np.clip(140 + 115 * (lead.fill_alpha / 255.0), 0, 255))
dest = rl.Rectangle(cx, cy, r * 2.0, r * 2.0)
rl.draw_texture_pro(self._lead_orb, src, dest, rl.Vector2(r, r), 0.0, rl.Color(tint.r, tint.g, tint.b, alpha))
@staticmethod
def _get_path_length_idx(pos_x_array: np.ndarray, path_height: float) -> int:
"""Get the index corresponding to the given path height"""
if len(pos_x_array) == 0:
return 0
indices = np.where(pos_x_array <= path_height)[0]
return indices[-1] if indices.size > 0 else 0
def _map_to_screen(self, in_x, in_y, in_z):
"""Project a point in car space to screen space"""
input_pt = np.array([in_x, in_y, in_z])
pt = self._car_space_transform @ input_pt
if abs(pt[2]) < 1e-6:
return None
x, y = pt[0] / pt[2], pt[1] / pt[2]
clip = self._clip_region
if not (clip.x <= x <= clip.x + clip.width and clip.y <= y <= clip.y + clip.height):
return None
return (x, y)
def _map_line_to_polygon(self, line: np.ndarray, y_off: float, z_off: float, max_idx: int, allow_invert: bool = True) -> np.ndarray:
"""Convert 3D line to 2D polygon for rendering."""
if line.shape[0] == 0:
return np.empty((0, 2), dtype=np.float32)
# Slice points and filter non-negative x-coordinates
points = line[:max_idx + 1]
points = points[points[:, 0] >= 0]
if points.shape[0] == 0:
return np.empty((0, 2), dtype=np.float32)
N = points.shape[0]
# Generate left and right 3D points in one array using broadcasting
offsets = np.array([[0, -y_off, z_off], [0, y_off, z_off]], dtype=np.float32)
points_3d = points[None, :, :] + offsets[:, None, :] # Shape: 2xNx3
points_3d = points_3d.reshape(2 * N, 3) # Shape: (2*N)x3
# Transform all points to projected space in one operation
proj = self._car_space_transform @ points_3d.T # Shape: 3x(2*N)
proj = proj.reshape(3, 2, N)
left_proj = proj[:, 0, :]
right_proj = proj[:, 1, :]
# Filter points where z is sufficiently large
valid_proj = (np.abs(left_proj[2]) >= 1e-6) & (np.abs(right_proj[2]) >= 1e-6)
if not np.any(valid_proj):
return np.empty((0, 2), dtype=np.float32)
# Compute screen coordinates
left_screen = left_proj[:2, valid_proj] / left_proj[2, valid_proj][None, :]
right_screen = right_proj[:2, valid_proj] / right_proj[2, valid_proj][None, :]
# Define clip region bounds
clip = self._clip_region
x_min, x_max = clip.x, clip.x + clip.width
y_min, y_max = clip.y, clip.y + clip.height
# Filter points within clip region
left_in_clip = (
(left_screen[0] >= x_min) & (left_screen[0] <= x_max) &
(left_screen[1] >= y_min) & (left_screen[1] <= y_max)
)
right_in_clip = (
(right_screen[0] >= x_min) & (right_screen[0] <= x_max) &
(right_screen[1] >= y_min) & (right_screen[1] <= y_max)
)
both_in_clip = left_in_clip & right_in_clip
if not np.any(both_in_clip):
return np.empty((0, 2), dtype=np.float32)
# Select valid and clipped points
left_screen = left_screen[:, both_in_clip]
right_screen = right_screen[:, both_in_clip]
# Handle Y-coordinate inversion on hills
if not allow_invert and left_screen.shape[1] > 1:
y = left_screen[1, :] # y-coordinates
keep = y == np.minimum.accumulate(y)
if not np.any(keep):
return np.empty((0, 2), dtype=np.float32)
left_screen = left_screen[:, keep]
right_screen = right_screen[:, keep]
return np.vstack((left_screen.T, right_screen[:, ::-1].T)).astype(np.float32)
@staticmethod
def _hsla_to_color(h, s, l, a):
rgb = colorsys.hls_to_rgb(h, l, s)
return rl.Color(
int(rgb[0] * 255),
int(rgb[1] * 255),
int(rgb[2] * 255),
int(a * 255)
)
@staticmethod
def _blend_colors(begin_colors, end_colors, t):
if t >= 1.0:
return end_colors
if t <= 0.0:
return begin_colors
inv_t = 1.0 - t
return [rl.Color(
int(inv_t * start.r + t * end.r),
int(inv_t * start.g + t * end.g),
int(inv_t * start.b + t * end.b),
int(inv_t * start.a + t * end.a)
) for start, end in zip(begin_colors, end_colors, strict=True)]