IQ.Pilot Release Commit @ 4fcea4d

system/webrtc/device/audio.py

@@ -0,0 +1,105 @@
+import asyncio
+import fractions
+
+import aiortc
+import av
+import numpy as np
+
+from cereal import messaging
+
+
+WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
+WEBRTC_AUDIO_PTIME = 0.020
+
+
+class AudioInputStreamTrack(aiortc.mediastreams.AudioStreamTrack):
+  """Device microphone -> WebRTC, sourced from micd's `rawAudioData` cereal stream.
+
+  micd owns the ALSA capture device, so opening it again via PyAudio fails with a host error
+  ('audio in use', PortAudio errno -9999). Instead we consume micd's already-published int16 mono
+  PCM and repacketize it into WebRTC audio frames — no device contention, and it works whenever micd
+  is running. Reading one message per recv() paces playout to micd's real-time publish rate.
+  """
+  def __init__(self, rate: int = 16000, channels: int = 1):
+    super().__init__()
+    self.rate = rate
+    self.channels = channels
+    # conflate=False: keep audio continuous (don't drop buffered samples) for clean playback.
+    self._sock = messaging.sub_sock("rawAudioData", conflate=False)
+    self._start: float | None = None
+    self.pts = 0
+    self.enabled = True
+    self._audio_buffer = bytearray()
+
+  def enable(self, enabled: bool) -> None:
+    self.enabled = enabled
+
+  async def _fill_audio_buffer(self, target_bytes: int) -> None:
+    deadline = asyncio.get_running_loop().time() + WEBRTC_AUDIO_PTIME
+    while len(self._audio_buffer) < target_bytes:
+      msg = messaging.recv_one_or_none(self._sock)
+      if msg is not None:
+        audio = msg.rawAudioData
+        rate = int(audio.sampleRate) or self.rate
+        if rate != self.rate:
+          self.rate = rate
+          self._audio_buffer.clear()
+          self._start = None
+          self.pts = 0
+        self._audio_buffer.extend(bytes(audio.data))
+        continue
+      if asyncio.get_running_loop().time() >= deadline:
+        break
+      await asyncio.sleep(0.005)
+
+  async def _next_audio_data(self) -> tuple[bytes, int]:
+    samples = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
+    target_bytes = samples * 2
+    await self._fill_audio_buffer(target_bytes)
+
+    if len(self._audio_buffer) >= target_bytes:
+      data = bytes(self._audio_buffer[:target_bytes])
+      del self._audio_buffer[:target_bytes]
+    else:
+      data = bytes(self._audio_buffer)
+      self._audio_buffer.clear()
+      data += bytes(target_bytes - len(data))
+
+    return data, self.rate
+
+  async def _pace(self, pts: int, sample_rate: int) -> None:
+    if self._start is None:
+      self._start = asyncio.get_running_loop().time()
+      return
+
+    wait = self._start + (pts / sample_rate) - asyncio.get_running_loop().time()
+    if wait > 0:
+      await asyncio.sleep(wait)
+
+  async def recv(self):
+    while True:
+      if not self.enabled:
+        break
+      data, sample_rate = await self._next_audio_data()
+      if data:
+        samples = len(data) // 2
+        pts = self.pts
+        self.pts += samples
+        await self._pace(pts, sample_rate)
+
+        frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
+        frame.planes[0].update(data)
+        frame.pts = pts
+        frame.sample_rate = sample_rate
+        frame.time_base = fractions.Fraction(1, sample_rate)
+        return frame
+
+    samples_per_frame = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
+    samples = np.zeros((1, samples_per_frame), dtype=np.int16)
+    frame = av.AudioFrame.from_ndarray(samples, format='s16', layout='mono')
+    frame.sample_rate = self.rate
+    frame.time_base = fractions.Fraction(1, self.rate)
+    frame.pts = self.pts
+    self.pts += frame.samples
+    await self._pace(frame.pts, self.rate)
+    return frame

system/webrtc/device/video.py

@@ -0,0 +1,270 @@
+import asyncio
+import logging
+import struct
+import time
+
+import av
+from teleoprtc.tracks import TiciVideoStreamTrack
+
+from cereal import messaging
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_MDL, DT_DMON
+
+# Arbitrary 16-byte UUID identifying konn3kt frame-timing SEI messages. When timing
+# telemetry is enabled, each frame carries a user_data_unregistered SEI NAL with four
+# big-endian doubles (ms): encode duration, IPC/queue delay, host transit, and the
+# device wall clock. The client decodes these to compute true glass-to-glass latency.
+TIMING_SEI_UUID = bytes([
+  0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e,
+  0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc,
+])
+# Annex-B start code + SEI NAL (type 6) + user_data_unregistered (type 5) + payload size
+# (0x30 = 48 bytes = 16 UUID + 32 data). Trailing 0x80 is the RBSP stop bit.
+_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID
+
+
+class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
+  livestream_camera_to_sock_mapping = {
+    "driver": "livestreamDriverEncodeData",
+    "wideRoad": "livestreamWideRoadEncodeData",
+    "road": "livestreamRoadEncodeData",
+  }
+  main_camera_to_sock_mapping = {
+    "driver": "driverEncodeData",
+    "wideRoad": "wideRoadEncodeData",
+    "road": "roadEncodeData",
+  }
+
+  # Number of live tracks still waiting for their first keyframe. The on-demand
+  # keyframe request (LivestreamRequestKeyframe) is a single global param honored by
+  # every encoder, so with multiple concurrent tracks (dual-camera PiP) we must not
+  # clear it until *all* tracks have received an IDR — otherwise the first track to
+  # get its keyframe clears the request and starves the others (black feed).
+  _kf_pending_count = 0
+
+  def __init__(self, camera_type: str):
+    dt = DT_DMON if camera_type == "driver" else DT_MDL
+    super().__init__(camera_type, dt)
+
+    self._params = Params()
+    self._camera_type = camera_type
+    self._candidate_topics = [
+      self.main_camera_to_sock_mapping[camera_type],
+      self.livestream_camera_to_sock_mapping[camera_type],
+    ]
+    self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
+    self._active_topic = self._preferred_topics()[0]
+    self._pts = 0
+    self._t0_ns = time.monotonic_ns()
+    self._cached_header: bytes = b""
+    self._sent_keyframe = False
+    self._kf_requested = False  # whether this track counts toward _kf_pending_count
+    self._frame_count = 0
+    self._last_frame_time = 0.0
+    self._last_preference_refresh = 0.0
+    # Tracks how long the H264 livestream feed has been silent, to gate the last-resort main-feed
+    # fallback (see recv) without flapping between sources frame-by-frame.
+    self._live_silent_since: float | None = None
+    # Opt-in glass-to-glass latency telemetry (toggled by the client over the data channel).
+    self.timing_sei_enabled = False
+    self._logger = logging.getLogger("LiveStreamVideoStreamTrack")
+
+    # Ask the encoder for an immediate IDR so the stream starts fast instead of waiting up to a full
+    # GOP for the next periodic keyframe (encoderd honors LivestreamRequestKeyframe per-frame).
+    self._mark_keyframe_needed()
+
+  def _request_keyframe(self, enabled: bool) -> None:
+    try:
+      self._params.put_bool("LivestreamRequestKeyframe", enabled)
+    except Exception:
+      self._logger.exception("failed to set LivestreamRequestKeyframe")
+
+  def _mark_keyframe_needed(self) -> None:
+    """This track needs (another) keyframe: keep the global request asserted."""
+    if not self._kf_requested:
+      LiveStreamVideoStreamTrack._kf_pending_count += 1
+      self._kf_requested = True
+    self._request_keyframe(True)
+
+  def _mark_keyframe_received(self) -> None:
+    """This track got its keyframe; only clear the global request once no track needs one."""
+    if self._kf_requested:
+      self._kf_requested = False
+      LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
+    if LiveStreamVideoStreamTrack._kf_pending_count == 0:
+      self._request_keyframe(False)
+
+  def stop(self):
+    # Release our pending-keyframe hold so a torn-down track that never received an
+    # IDR doesn't pin LivestreamRequestKeyframe True forever (continuous keyframes).
+    if getattr(self, "_kf_requested", False):
+      self._kf_requested = False
+      LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
+    try:
+      super().stop()
+    except Exception:
+      pass
+
+  def switch_camera(self, camera_type: str) -> None:
+    """Repoint this track at a different camera without renegotiating the peer connection.
+
+    Lets a single video track back the whole Live View — the client flips cameras over the
+    data channel and we swap the source here, instead of uplinking every camera at once."""
+    if camera_type not in self.livestream_camera_to_sock_mapping:
+      self._logger.warning("[%s] ignoring switch to unknown camera %s", self._id, camera_type)
+      return
+    if camera_type == self._camera_type:
+      return
+    self._logger.info("[%s] switching camera %s -> %s", self._id, self._camera_type, camera_type)
+    self._camera_type = camera_type
+    self._candidate_topics = [
+      self.main_camera_to_sock_mapping[camera_type],
+      self.livestream_camera_to_sock_mapping[camera_type],
+    ]
+    self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
+    self._active_topic = self._preferred_topics()[0]
+    # Force a fresh keyframe/header before emitting frames from the new source, and ask the encoder
+    # for an immediate IDR so the camera switch isn't stalled waiting for the next periodic keyframe.
+    self._cached_header = b""
+    self._sent_keyframe = False
+    self._last_preference_refresh = 0.0
+    self._live_silent_since = None
+    self._mark_keyframe_needed()
+
+  def _preferred_topics(self) -> list[str]:
+    # WebRTC currently forces H.264. The dedicated livestream topics are the H.264 feeds,
+    # while the main encode topics are the full-resolution HEVC recordings. Prefer the
+    # livestream feeds both onroad and offroad, and keep the main topics only as fallback.
+    return [
+      self.livestream_camera_to_sock_mapping[self._camera_type],
+      self.main_camera_to_sock_mapping[self._camera_type],
+    ]
+
+  def _reset_decoder_state(self, topic: str) -> None:
+    if topic == self._active_topic:
+      return
+    self._logger.info("[%s] switching video source from %s to %s", self._id, self._active_topic, topic)
+    self._active_topic = topic
+    self._cached_header = b""
+    self._sent_keyframe = False
+
+  def _timing_sei(self, evta, log_mono_time: int) -> bytes:
+    """Build a timing SEI NAL from encode metadata, or empty bytes when disabled."""
+    if not self.timing_sei_enabled:
+      return b""
+    idx = evta.idx
+    return _SEI_PREFIX + struct.pack(
+      '>4d',
+      (idx.timestampEof - idx.timestampSof) / 1e6,   # encode duration (ms)
+      (log_mono_time - idx.timestampEof) / 1e6,       # IPC/queue delay (ms)
+      (time.monotonic_ns() - log_mono_time) / 1e6,    # host transit so far (ms)
+      time.time() * 1000,                             # device wall clock (ms)  # noqa: TID251
+    ) + b'\x80'
+
+  def _is_keyframe(self, data: bytes) -> bool:
+    """Check if H.264 NAL unit contains an IDR keyframe (NAL type 5)."""
+    i = 0
+    while i < len(data) - 4:
+      # Look for Annex B start codes: 0x000001 or 0x00000001
+      if data[i:i+3] == b'\x00\x00\x01':
+        nal_type = data[i+3] & 0x1f
+        if nal_type == 5:  # IDR slice
+          return True
+        i += 3
+      elif data[i:i+4] == b'\x00\x00\x00\x01':
+        nal_type = data[i+4] & 0x1f
+        if nal_type == 5:  # IDR slice
+          return True
+        i += 4
+      else:
+        i += 1
+    return False
+
+  async def recv(self):
+    while True:
+      now = time.monotonic()
+      # Resolve topics each iteration: a camera switch (different async task) can rebuild self._socks
+      # across the await below, so a value cached before the loop would index a stale key (KeyError).
+      live_topic = self.livestream_camera_to_sock_mapping[self._camera_type]
+      main_topic = self.main_camera_to_sock_mapping[self._camera_type]
+      # Lock onto the dedicated H264 livestream feed. Onroad the HEVC main feed also publishes at
+      # 20fps; eagerly preferring whichever socket had a frame ready raced frame-by-frame, reset the
+      # decoder every frame, and (the track is negotiated H264) shoved HEVC garbage into the stream —
+      # the onroad choppiness. Only fall back to the main feed as a last resort after a long
+      # livestream silence (e.g. stream_encoderd still spinning up), and snap back when it returns.
+      msg = messaging.recv_one_or_none(self._socks[live_topic])
+      if msg is not None:
+        self._reset_decoder_state(live_topic)
+        self._last_frame_time = now
+        self._live_silent_since = None
+        break
+
+      if self._live_silent_since is None:
+        self._live_silent_since = now
+      elif now - self._live_silent_since > 3.0:
+        maybe_msg = messaging.recv_one_or_none(self._socks[main_topic])
+        if maybe_msg is not None:
+          self._reset_decoder_state(main_topic)
+          self._last_frame_time = now
+          msg = maybe_msg
+          break
+
+      await asyncio.sleep(0.005)
+
+    evta = getattr(msg, msg.which())
+
+    header = bytes(evta.header)
+    data = bytes(evta.data)
+    self._frame_count += 1
+
+    # Cache SPS/PPS header when it arrives
+    if header:
+      self._cached_header = header
+      self._logger.debug(f"[{self._id}] cached SPS/PPS header ({len(header)} bytes)")
+
+    # CRITICAL: Cannot decode without SPS/PPS. Wait for it.
+    if not self._cached_header:
+      self._logger.debug(f"[{self._id}] frame {self._frame_count}: no SPS/PPS yet, skipping")
+      return await self.recv()
+
+    is_keyframe = self._is_keyframe(data)
+
+    # Wait for first keyframe before sending any frames
+    # Browser decoder needs IDR to initialize properly
+    if not self._sent_keyframe:
+      if not is_keyframe:
+        self._logger.debug(f"[{self._id}] frame {self._frame_count}: waiting for keyframe")
+        return await self.recv()
+      self._sent_keyframe = True
+      # Got the IDR we asked for — stop nagging the encoder, but only once every
+      # concurrent track has its keyframe (multi-track PiP shares the global param).
+      self._mark_keyframe_received()
+      self._logger.info(f"[{self._id}] first keyframe received, starting stream")
+
+    # Optional timing SEI NAL, inserted before the slice data (and after SPS/PPS on keyframes).
+    sei_nal = self._timing_sei(evta, msg.logMonoTime)
+
+    # Prepend SPS/PPS header to keyframes (required by some decoders)
+    # For non-keyframes, header is optional but safe to include
+    if is_keyframe:
+      payload = self._cached_header + sei_nal + data
+    else:
+      payload = sei_nal + data
+
+    self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
+
+    packet = av.Packet(payload)
+    packet.time_base = self._time_base
+    packet.pts = int(self._pts)
+    packet.dts = int(self._pts)
+    packet.duration = int(self._dt * self._clock_rate)
+
+    if is_keyframe:
+      packet.is_keyframe = True
+
+    self.log_debug("track sending frame %s (keyframe=%s, size=%d)", self._pts, is_keyframe, len(payload))
+
+    return packet
+
+  def codec_preference(self) -> str | None:
+    return "H264"

system/webrtc/schema.py

@@ -0,0 +1,43 @@
+import capnp
+from typing import Any
+
+
+def generate_type(type_walker, schema_walker) -> str | list[Any] | dict[str, Any]:
+  data_type = next(type_walker)
+  if data_type.which() == 'struct':
+    return generate_struct(next(schema_walker))
+  elif data_type.which() == 'list':
+    _ = next(schema_walker)
+    return [generate_type(type_walker, schema_walker)]
+  elif data_type.which() == 'enum':
+    return "text"
+  else:
+    return str(data_type.which())
+
+
+def generate_struct(schema: capnp.lib.capnp._StructSchema) -> dict[str, Any]:
+  return {field: generate_field(schema.fields[field]) for field in schema.fields if not field.endswith("DEPRECATED")}
+
+
+def generate_field(field: capnp.lib.capnp._StructSchemaField) -> str | list[Any] | dict[str, Any]:
+  def schema_walker(field):
+    yield field.schema
+
+    s = field.schema
+    while hasattr(s, 'elementType'):
+      s = s.elementType
+      yield s
+
+  def type_walker(field):
+    yield field.proto.slot.type
+
+    t = field.proto.slot.type
+    while hasattr(getattr(t, t.which()), 'elementType'):
+      t = getattr(t, t.which()).elementType
+      yield t
+
+  if field.proto.which() == "slot":
+    schema_gen, type_gen = schema_walker(field), type_walker(field)
+    return generate_type(type_gen, schema_gen)
+  else:
+    return generate_struct(field.schema)

system/webrtc/tests/test_stream_session.py

@@ -0,0 +1,104 @@
+import asyncio
+import json
+import time
+# for aiortc and its dependencies
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+from aiortc import RTCDataChannel
+from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
+import capnp
+import pyaudio
+from cereal import messaging, log
+
+from openpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
+from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
+from openpilot.system.webrtc.device.audio import AudioInputStreamTrack
+
+
+class TestStreamSession:
+  def setup_method(self):
+    self.loop = asyncio.new_event_loop()
+
+  def teardown_method(self):
+    self.loop.stop()
+    self.loop.close()
+
+  def test_outgoing_proxy(self, mocker):
+    test_msg = log.Event.new_message()
+    test_msg.logMonoTime = 123
+    test_msg.valid = True
+    test_msg.customReservedRawData0 = b"test"
+    expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
+    expected_json = json.dumps(expected_dict).encode()
+
+    channel = mocker.Mock(spec=RTCDataChannel)
+    mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
+    def mocked_update(t):
+      mocked_submaster.update_msgs(0, [test_msg])
+
+    mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
+    proxy = CerealOutgoingMessageProxy(mocked_submaster)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    channel.send.assert_called_once_with(expected_json)
+
+  def test_incoming_proxy(self, mocker):
+    tested_msgs = [
+      {"type": "customReservedRawData0", "data": "test"}, # primitive
+      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
+      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
+    ]
+
+    mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
+
+    proxy = CerealIncomingMessageProxy(mocked_pubmaster)
+
+    for msg in tested_msgs:
+      proxy.send(json.dumps(msg).encode())
+
+      mocked_pubmaster.send.assert_called_once()
+      mt, md = mocked_pubmaster.send.call_args.args
+      assert mt == msg["type"]
+      assert isinstance(md, capnp._DynamicStructBuilder)
+      assert hasattr(md, msg["type"])
+
+      mocked_pubmaster.reset_mock()
+
+  def test_livestream_track(self, mocker):
+    fake_msg = messaging.new_message("livestreamDriverEncodeData")
+
+    config = {"receive.return_value": fake_msg.to_bytes()}
+    mocker.patch("msgq.SubSocket", spec=True, **config)
+    track = LiveStreamVideoStreamTrack("driver")
+
+    assert track.id.startswith("driver")
+    assert track.codec_preference() == "H264"
+
+    for i in range(5):
+      packet = self.loop.run_until_complete(track.recv())
+      assert packet.time_base == VIDEO_TIME_BASE
+      if i == 0:
+        start_ns = time.monotonic_ns()
+        start_pts = packet.pts
+      assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
+      assert packet.size == 0
+
+  def test_input_audio_track(self, mocker):
+    packet_time, rate = 0.02, 16000
+    sample_count = int(packet_time * rate)
+    mocked_stream = mocker.MagicMock(spec=pyaudio.Stream)
+    mocked_stream.read.return_value = b"\x00" * 2 * sample_count
+
+    config = {"open.side_effect": lambda *args, **kwargs: mocked_stream}
+    mocker.patch("pyaudio.PyAudio", spec=True, **config)
+    track = AudioInputStreamTrack(audio_format=pyaudio.paInt16, packet_time=packet_time, rate=rate)
+
+    for i in range(5):
+      frame = self.loop.run_until_complete(track.recv())
+      assert frame.rate == rate
+      assert frame.samples == sample_count
+      assert frame.pts == i * sample_count

system/webrtc/tests/test_ui_stream.py

@@ -0,0 +1,301 @@
+import json
+import math
+
+from cereal import log, messaging
+from openpilot.system.webrtc.ui_stream import (
+  UI_STREAM_SERVICES,
+  UIStreamMessageProxy,
+  compute_ui_status,
+  frame_to_str,
+  MAX_BUFFERED_BYTES,
+)
+
+OpenpilotState = log.SelfdriveState.OpenpilotState
+
+
+def make_readers(**overrides):
+  readers = {}
+  for service in UI_STREAM_SERVICES:
+    if service == "onroadEvents":
+      msg = messaging.new_message(service, 0)
+    else:
+      msg = messaging.new_message(service)
+    readers[service] = msg
+  readers.update(overrides)
+  return {s: getattr(m, s) for s, m in readers.items()}
+
+
+class FakeSubMaster:
+  def __init__(self, readers, updated=None, valid=None):
+    self.readers = readers
+    self.updated = updated or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.valid = valid or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.logMonoTime = dict.fromkeys(UI_STREAM_SERVICES, 42)
+    self.update_calls = 0
+
+  def __getitem__(self, service):
+    return self.readers[service]
+
+  def update(self, timeout):
+    self.update_calls += 1
+
+
+class FakeChannel:
+  def __init__(self, buffered_amount=0):
+    self.bufferedAmount = buffered_amount
+    self.sent = []
+
+  def send(self, data):
+    self.sent.append(data)
+
+
+def make_proxy(sm, **kwargs):
+  return UIStreamMessageProxy(sm=sm, **kwargs)
+
+
+class TestComputeUiStatus:
+  def _msgs(self):
+    ss = messaging.new_message("selfdriveState")
+    iq = messaging.new_message("iqState")
+    ev = messaging.new_message("onroadEvents", 0)
+    return ss.selfdriveState, iq.iqState, ev.onroadEvents
+
+  def test_disengaged(self):
+    ss, iq, ev = self._msgs()
+    assert compute_ui_status(ss, iq, ev) == "disengaged"
+
+  def test_engaged_no_guidance(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+  def test_pre_enabled_is_override(self):
+    ss, iq, ev = self._msgs()
+    ss.state = OpenpilotState.preEnabled
+    assert compute_ui_status(ss, iq, ev) == "override"
+
+  def test_lat_only(self):
+    ss, iq, ev = self._msgs()
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "lat_only"
+
+  def test_long_only(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    assert compute_ui_status(ss, iq, ev) == "long_only"
+
+  def test_both_engaged(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+
+class TestUIStreamFrame:
+  def test_frame_shape_and_json(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) for i in range(33)]
+    model.position.y = [0.123456] * 33
+    model.position.z = [0.0] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [1.0, 2.0]
+      lane.y = [0.1, 0.2]
+      lane.z = [0.0, 0.0]
+    model.laneLineProbs = [0.9, 0.8, 0.7, 0.6]
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [1.0]
+      edge.y = [2.0]
+      edge.z = [0.0]
+    model.roadEdgeStds = [0.1, 0.2]
+    model.acceleration.x = [0.5] * 33
+
+    cs_msg = messaging.new_message("carState")
+    cs_msg.carState.vEgo = 12.345
+    cs_msg.carState.leftBlinker = True
+
+    readers = make_readers(modelV2=model_msg, carState=cs_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"
+    data = frame["data"]
+    assert len(data["modelV2"]["position"]["x"]) == 33
+    assert data["modelV2"]["position"]["y"][0] == 0.12
+    assert len(data["modelV2"]["laneLines"]) == 4
+    assert data["carState"]["vEgo"] == 12.35
+    assert data["carState"]["leftBlinker"] is True
+    assert data["uiStatus"] == "disengaged"
+    assert data["selfdriveState"]["alertSize"] == "none"
+    assert "hasLongitudinalControl" in data["init"]
+    assert "cameraOffset" in data["init"]
+    assert "isMetric" in data["init"]
+
+  def test_nan_scrubbed(self):
+    model_msg = messaging.new_message("modelV2")
+    model_msg.modelV2.position.x = [math.nan, math.inf, 1.0]
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    raw = channel.sent[0]
+    assert "NaN" not in raw and "Infinity" not in raw
+    frame = json.loads(raw)
+    assert frame["data"]["modelV2"]["position"]["x"] == [0.0, 0.0, 1.0]
+
+  def test_backpressure_drops_frames(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel(buffered_amount=MAX_BUFFERED_BYTES + 1)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert proxy.dropped_frames == 1
+
+  def test_no_send_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    proxy._last_emit_time = float("inf")
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert sm.update_calls == 1
+
+  def test_heartbeat_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["data"]["modelV2"] is None
+
+  def test_low_bandwidth_decimation(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 500_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 2
+
+  def test_full_rate_at_high_bitrate(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 5_000_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 4
+
+  def test_sticky_status_when_engaged_like(self):
+    ss_msg = messaging.new_message("selfdriveState")
+    ss_msg.selfdriveState.enabled = True
+    iq_msg = messaging.new_message("iqState")
+    iq_msg.iqState.aol.available = True
+    iq_msg.iqState.aol.enabled = True
+
+    readers = make_readers(selfdriveState=ss_msg, iqState=iq_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+    iq_msg.iqState.aol.available = False
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+  def test_frame_size_budget(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) * 3.03 for i in range(33)]
+    model.position.y = [1.234567] * 33
+    model.position.z = [0.456789] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [float(i) * 3.03 for i in range(33)]
+      lane.y = [1.234567] * 33
+      lane.z = [0.456789] * 33
+    model.laneLineProbs = [0.9] * 4
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [float(i) * 3.03 for i in range(33)]
+      edge.y = [1.234567] * 33
+      edge.z = [0.456789] * 33
+    model.acceleration.x = [1.23] * 33
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    frame = proxy._build_frame()
+    encoded = frame_to_str(frame)
+    assert len(encoded) < 8 * 1024
+
+
+class TestSessionWiring:
+  def test_set_ui_stream_control_message(self):
+    import asyncio
+    import logging
+    from types import SimpleNamespace
+    from openpilot.system.webrtc.webrtcd import StreamSession
+
+    session = StreamSession.__new__(StreamSession)
+    session.logger = logging.getLogger("webrtcd")
+    session.ui_stream_runner = None
+    session.bitrate_controller = None
+    session.incoming_bridge = None
+    channel = FakeChannel()
+    session.stream = SimpleNamespace(
+      has_messaging_channel=lambda: True,
+      get_messaging_channel=lambda: channel,
+    )
+
+    async def go():
+      await session.message_handler(b'{"type":"setUiStream","enabled":true}')
+      assert session.ui_stream_runner is not None
+      await asyncio.sleep(0.05)
+      await session.message_handler(b'{"type":"setUiStream","enabled":false}')
+      assert session.ui_stream_runner is None
+
+    asyncio.run(go())
+
+    assert len(channel.sent) >= 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"

system/webrtc/tests/test_webrtcd.py

@@ -0,0 +1,65 @@
+import pytest
+import asyncio
+import json
+# for aiortc and its dependencies
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+from openpilot.system.webrtc.webrtcd import get_stream
+
+import aiortc
+from teleoprtc import WebRTCOfferBuilder
+from parameterized import parameterized_class
+
+
+@parameterized_class(("in_services", "out_services"), [
+  (["testJoystick"], ["carState"]),
+  ([], ["carState"]),
+  (["testJoystick"], []),
+  ([], []),
+])
+@pytest.mark.asyncio
+class TestWebrtcdProc:
+  async def assertCompletesWithTimeout(self, awaitable, timeout=1):
+    try:
+      async with asyncio.timeout(timeout):
+        await awaitable
+    except TimeoutError:
+      pytest.fail("Timeout while waiting for awaitable to complete")
+
+  async def test_webrtcd(self, mocker):
+    mock_request = mocker.MagicMock()
+    async def connect(offer):
+      body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
+      mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
+      response = await get_stream(mock_request)
+      response_json = json.loads(response.text)
+      return aiortc.RTCSessionDescription(**response_json)
+
+    builder = WebRTCOfferBuilder(connect)
+    builder.offer_to_receive_video_stream("road")
+    builder.offer_to_receive_audio_stream()
+    if len(self.in_services) > 0 or len(self.out_services) > 0:
+      builder.add_messaging()
+
+    stream = builder.stream()
+
+    await self.assertCompletesWithTimeout(stream.start())
+    await self.assertCompletesWithTimeout(stream.wait_for_connection())
+
+    assert stream.has_incoming_video_track("road")
+    assert stream.has_incoming_audio_track()
+    assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
+
+    video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
+    await self.assertCompletesWithTimeout(video_track.recv())
+    await self.assertCompletesWithTimeout(audio_track.recv())
+
+    await self.assertCompletesWithTimeout(stream.stop())
+
+    # cleanup, very implementation specific, test may break if it changes
+    assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
+    _, session = mock_request.app["streams"].__setitem__.call_args.args
+    await self.assertCompletesWithTimeout(session.post_run_cleanup())
+

system/webrtc/ui_stream.py

@@ -0,0 +1,269 @@
+import json
+import math
+import time
+
+import numpy as np
+
+from cereal import car, log, custom, messaging
+from openpilot.common.params import Params
+
+OpenpilotState = log.SelfdriveState.OpenpilotState
+GuidanceState = custom.AlwaysOnLateral.AlwaysOnLateralState
+
+UI_STREAM_SERVICES = [
+  "modelV2", "carState", "selfdriveState", "controlsState", "liveCalibration",
+  "radarState", "longitudinalPlan", "deviceState", "roadCameraState",
+  "iqState", "onroadEvents",
+]
+
+# Above this the viewer is not draining the channel; drop frames instead of queueing,
+# telemetry is newest-wins and unbounded SCTP buffering is how webrtcd leaked before.
+MAX_BUFFERED_BYTES = 256 * 1024
+
+# Bitrate at/below which modelV2 frames are decimated to half rate to leave
+# headroom for video on a struggling uplink.
+LOW_BANDWIDTH_BITRATE = 500_000
+
+HEARTBEAT_INTERVAL = 1.0
+
+
+def _round_list(vals, decimals: int) -> list[float]:
+  arr = np.asarray(vals, dtype=np.float64)
+  if arr.size == 0:
+    return []
+  arr = np.round(np.where(np.isfinite(arr), arr, 0.0), decimals)
+  return arr.tolist()
+
+
+def _round_float(val, decimals: int = 3, default: float = 0.0) -> float:
+  try:
+    v = float(val)
+  except (TypeError, ValueError):
+    return default
+  return round(v, decimals) if math.isfinite(v) else default
+
+
+def _xyz(line, decimals: int = 2) -> dict[str, list[float]]:
+  return {
+    "x": _round_list(line.x, decimals),
+    "y": _round_list(line.y, decimals),
+    "z": _round_list(line.z, decimals),
+  }
+
+
+def _lead(lead) -> dict:
+  return {
+    "status": bool(lead.status),
+    "dRel": _round_float(lead.dRel, 2),
+    "yRel": _round_float(lead.yRel, 2),
+    "vRel": _round_float(lead.vRel, 2),
+  }
+
+
+def compute_ui_status(ss, iq_state, onroad_events) -> str:
+  # Mirrors IQUIState.update_status; that module pulls in the raylib UI stack,
+  # which must not be imported into webrtcd.
+  guidance = iq_state.aol
+  guidance_state = guidance.state
+
+  if ss.state == OpenpilotState.preEnabled:
+    return "override"
+
+  if ss.state == OpenpilotState.overriding:
+    if not guidance.available:
+      return "override"
+    if any(e.overrideLongitudinal for e in onroad_events):
+      return "override"
+
+  if guidance_state in (GuidanceState.paused, GuidanceState.overriding):
+    return "override"
+
+  if not guidance.available:
+    return "engaged" if ss.enabled else "disengaged"
+
+  if not guidance.enabled and not ss.enabled:
+    return "disengaged"
+
+  if guidance.enabled and ss.enabled:
+    return "engaged"
+
+  if guidance.enabled:
+    return "lat_only"
+
+  if ss.enabled:
+    return "long_only"
+
+  return "disengaged"
+
+
+def build_init_payload(params: Params | None = None) -> dict:
+  params = params or Params()
+
+  has_longitudinal_control = False
+  cp_bytes = params.get("CarParamsPersistent")
+  if cp_bytes is not None:
+    try:
+      cp = messaging.log_from_bytes(cp_bytes, car.CarParams)
+      if cp.alphaLongitudinalAvailable:
+        has_longitudinal_control = params.get_bool("AlphaLongitudinalEnabled")
+      else:
+        has_longitudinal_control = bool(cp.openpilotLongitudinalControl)
+    except Exception:
+      pass
+
+  camera_offset = 0.0
+  if params.get("ModelManager_ActiveBundle"):
+    try:
+      camera_offset = float(params.get("CameraOffset", return_default=True) or 0.0)
+    except (TypeError, ValueError):
+      camera_offset = 0.0
+
+  return {
+    "hasLongitudinalControl": has_longitudinal_control,
+    "cameraOffset": _round_float(camera_offset, 3),
+    "isMetric": params.get_bool("IsMetric"),
+  }
+
+
+class UIStreamMessageProxy:
+  """Sends a trimmed, HUD-only JSON projection of UI state over the session data
+  channel, clocked by modelV2 (~20Hz). Payload stays a few KB per frame; anything
+  the client renderers don't read is not serialized."""
+
+  def __init__(self, sm: messaging.SubMaster | None = None, bitrate_getter=None):
+    self.sm = sm if sm is not None else messaging.SubMaster(UI_STREAM_SERVICES)
+    self.channels = []
+    self.bitrate_getter = bitrate_getter
+    self.dropped_frames = 0
+    self._last_non_disengaged = "disengaged"
+    self._last_emit_time = 0.0
+    self._decimate_flip = False
+    self._init_payload = build_init_payload()
+
+  def add_channel(self, channel):
+    self.channels.append(channel)
+
+  def update(self):
+    self.sm.update(0)
+
+    model_updated = self.sm.updated["modelV2"]
+    now = time.monotonic()
+    if not model_updated:
+      if now - self._last_emit_time < HEARTBEAT_INTERVAL:
+        return
+    elif self._low_bandwidth():
+      self._decimate_flip = not self._decimate_flip
+      if self._decimate_flip:
+        return
+
+    # Send as a text frame: react-native-webrtc surfaces binary frames as
+    # ArrayBuffers that Hermes cannot reliably decode without TextDecoder.
+    frame = self._build_frame(include_model=model_updated)
+    encoded = frame_to_str(frame)
+    self._last_emit_time = now
+    for channel in self.channels:
+      if channel.bufferedAmount > MAX_BUFFERED_BYTES:
+        self.dropped_frames += 1
+        continue
+      channel.send(encoded)
+
+  def _low_bandwidth(self) -> bool:
+    if self.bitrate_getter is None:
+      return False
+    try:
+      bitrate = self.bitrate_getter()
+    except Exception:
+      return False
+    return bitrate is not None and bitrate <= LOW_BANDWIDTH_BITRATE
+
+  def _ui_status(self) -> str:
+    sm = self.sm
+    ss = sm["selfdriveState"]
+    iq_state = sm["iqState"]
+    status = compute_ui_status(ss, iq_state, sm["onroadEvents"])
+
+    # Same stickiness as UIState._update_status: while still engaged-like, a
+    # transient disengaged classification keeps the last non-disengaged status.
+    if status != "disengaged":
+      self._last_non_disengaged = status
+      return status
+
+    if ss.enabled or iq_state.aol.enabled:
+      if self._last_non_disengaged != "disengaged":
+        return self._last_non_disengaged
+      return "engaged" if ss.enabled else "disengaged"
+
+    self._last_non_disengaged = "disengaged"
+    return "disengaged"
+
+  def _build_frame(self, include_model: bool = True) -> dict:
+    sm = self.sm
+    cs = sm["carState"]
+    ss = sm["selfdriveState"]
+    calib = sm["liveCalibration"]
+    radar = sm["radarState"]
+    device_state = sm["deviceState"]
+
+    model_data = None
+    if include_model:
+      model = sm["modelV2"]
+      model_data = {
+        "position": _xyz(model.position),
+        "laneLines": [_xyz(line) for line in model.laneLines],
+        "laneLineProbs": _round_list(model.laneLineProbs, 3),
+        "roadEdges": [_xyz(edge) for edge in model.roadEdges],
+        "roadEdgeStds": _round_list(model.roadEdgeStds, 3),
+        "acceleration": {"x": _round_list(model.acceleration.x, 2)},
+      }
+
+    data = {
+      "modelV2": model_data,
+      "carState": {
+        "vEgo": _round_float(cs.vEgo, 2),
+        "vEgoCluster": _round_float(cs.vEgoCluster, 2),
+        "vCruiseCluster": _round_float(cs.vCruiseCluster, 2),
+        "leftBlinker": bool(cs.leftBlinker),
+        "rightBlinker": bool(cs.rightBlinker),
+      },
+      "selfdriveState": {
+        "enabled": bool(ss.enabled),
+        "experimentalMode": bool(ss.experimentalMode),
+        "state": str(ss.state),
+        "alertText1": str(ss.alertText1),
+        "alertText2": str(ss.alertText2),
+        "alertSize": str(ss.alertSize),
+        "alertStatus": str(ss.alertStatus),
+      },
+      "controlsState": {
+        "vCruiseDEPRECATED": _round_float(sm["controlsState"].vCruiseDEPRECATED, 2),
+      },
+      "liveCalibration": {
+        "calStatus": str(calib.calStatus),
+        "rpyCalib": _round_list(calib.rpyCalib, 5),
+        "wideFromDeviceEuler": _round_list(calib.wideFromDeviceEuler, 5),
+        "height": _round_list(calib.height, 3),
+      },
+      "radarState": {
+        "valid": bool(sm.valid["radarState"]),
+        "leadOne": _lead(radar.leadOne),
+        "leadTwo": _lead(radar.leadTwo),
+      },
+      "longitudinalPlan": {
+        "allowThrottle": bool(sm["longitudinalPlan"].allowThrottle),
+      },
+      "deviceState": {
+        "deviceType": str(device_state.deviceType),
+        "started": bool(device_state.started),
+      },
+      "roadCameraState": {
+        "sensor": str(sm["roadCameraState"].sensor),
+      },
+      "uiStatus": self._ui_status(),
+      "init": self._init_payload,
+    }
+
+    return {"type": "uiStream", "logMonoTime": sm.logMonoTime["modelV2"], "data": data}
+
+
+def frame_to_str(frame: dict) -> str:
+  return json.dumps(frame, separators=(",", ":"))

system/webrtc/webrtcd.py

@@ -0,0 +1,658 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import json
+import os
+import uuid
+import logging
+from dataclasses import dataclass, field
+from typing import Any, TYPE_CHECKING
+
+from openpilot.common.params import Params
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning)
+
+import capnp
+import aiortc.rtcrtpsender
+from aiohttp import web
+from aiortc.rtp import RTCP_PSFB_APP, RtcpPsfbPacket, unpack_remb_fci
+if TYPE_CHECKING:
+  from aiortc.rtcdatachannel import RTCDataChannel
+
+from openpilot.system.webrtc.schema import generate_field
+from cereal import messaging, log
+
+
+_handle_rtcp_packet = aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet
+
+
+async def _handle_rtcp_packet_with_remb(self, packet):
+  if isinstance(packet, RtcpPsfbPacket) and packet.fmt == RTCP_PSFB_APP:
+    try:
+      bitrate, ssrcs = unpack_remb_fci(packet.fci)
+      if getattr(self, "_ssrc", None) in ssrcs:
+        self._remb_bitrate = bitrate
+    except ValueError:
+      pass
+  return await _handle_rtcp_packet(self, packet)
+
+
+aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet = _handle_rtcp_packet_with_remb
+
+
+class CerealOutgoingMessageProxy:
+  def __init__(self, sm: messaging.SubMaster):
+    self.sm = sm
+    self.channels: list[RTCDataChannel] = []
+
+  def add_channel(self, channel: 'RTCDataChannel'):
+    self.channels.append(channel)
+
+  def to_json(self, msg_content: Any):
+    if isinstance(msg_content, capnp._DynamicStructReader):
+      msg_dict = msg_content.to_dict()
+    elif isinstance(msg_content, capnp._DynamicListReader):
+      msg_dict = [self.to_json(msg) for msg in msg_content]
+    elif isinstance(msg_content, bytes):
+      msg_dict = msg_content.decode()
+    else:
+      msg_dict = msg_content
+
+    return msg_dict
+
+  def update(self):
+    # this is blocking in async context...
+    self.sm.update(0)
+    for service, updated in self.sm.updated.items():
+      if not updated:
+        continue
+      msg_dict = self.to_json(self.sm[service])
+      mono_time, valid = self.sm.logMonoTime[service], self.sm.valid[service]
+      outgoing_msg = {"type": service, "logMonoTime": mono_time, "valid": valid, "data": msg_dict}
+      encoded_msg = json.dumps(outgoing_msg).encode()
+      for channel in self.channels:
+        channel.send(encoded_msg)
+
+
+class CerealIncomingMessageProxy:
+  def __init__(self, pm: messaging.PubMaster):
+    self.pm = pm
+
+  def send(self, message: bytes):
+    msg_json = json.loads(message)
+    msg_type, msg_data = msg_json["type"], msg_json["data"]
+    size = None
+    if not isinstance(msg_data, dict):
+      size = len(msg_data)
+
+    msg = messaging.new_message(msg_type, size=size)
+    setattr(msg, msg_type, msg_data)
+    self.pm.send(msg_type, msg)
+
+
+class AsyncTaskRunner:
+  def __init__(self):
+    self.task: asyncio.Task | None = None
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self):
+    if self.task is None:
+      self.task = asyncio.create_task(self.run())
+
+  async def stop(self):
+    if self.task is None:
+      return
+    if not self.task.done():
+      self.task.cancel()
+      try:
+        await self.task
+      except asyncio.CancelledError:
+        pass
+    self.task = None
+
+
+class IncomingAudioCerealProxy(AsyncTaskRunner):
+  def __init__(self, track: Any):
+    super().__init__()
+    from av.audio.resampler import AudioResampler
+    from openpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
+    from openpilot.system.webrtc.device.audio import WEBRTC_AUDIO_SERVICE
+
+    self.track = track
+    self.service = WEBRTC_AUDIO_SERVICE
+    self.pm = messaging.PubMaster([self.service])
+    self.resampler = AudioResampler(format="s16", layout="mono", rate=SOUND_SAMPLE_RATE)
+
+  def _publish(self, frame: Any) -> None:
+    data = frame.to_ndarray().tobytes()
+    if not data:
+      return
+
+    msg = messaging.new_message(self.service, valid=True)
+    msg.webrtcAudioData.data = data
+    msg.webrtcAudioData.sampleRate = frame.sample_rate
+    self.pm.send(self.service, msg)
+
+  async def run(self):
+    from aiortc.mediastreams import MediaStreamError
+
+    while True:
+      try:
+        frame = await self.track.recv()
+        for resampled_frame in self.resampler.resample(frame):
+          self._publish(resampled_frame)
+      except MediaStreamError:
+        break
+      except Exception:
+        self.logger.exception("Incoming audio cereal proxy failure")
+        await asyncio.sleep(0.1)
+
+
+class CerealProxyRunner:
+  def __init__(self, proxy: CerealOutgoingMessageProxy):
+    self.proxy = proxy
+    self.is_running = False
+    self.task = None
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self):
+    assert self.task is None
+    self.task = asyncio.create_task(self.run())
+
+  def stop(self):
+    if self.task is None or self.task.done():
+      return
+    self.task.cancel()
+    self.task = None
+
+  async def run(self):
+    from aiortc.exceptions import InvalidStateError
+
+    while True:
+      try:
+        self.proxy.update()
+      except InvalidStateError:
+        self.logger.warning("Cereal outgoing proxy invalid state (connection closed)")
+        break
+      except Exception:
+        self.logger.exception("Cereal outgoing proxy failure")
+      await asyncio.sleep(0.01)
+
+
+class LivestreamBitrateController:
+  """Adaptive bitrate for the livestream encoder using browser REMB feedback."""
+
+  # Match comma's rung choices more closely. A steadier capped stream tends to look better than
+  # an occasionally-higher bitrate stream that induces queueing, jitter, and frame pacing swings.
+  bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
+  label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[-1]}
+
+  sample_interval = 1.0
+  lower_factor = 0.9
+  probe_after = 10
+  settle_samples = 3
+
+  def __init__(self, peer_connection: Any):
+    self.pc = peer_connection
+    self.params = Params()
+    self.task: asyncio.Task | None = None
+
+    # Start conservative and probe UP only when REMB proves headroom. Previously this started at
+    # the top rung (5 Mbps); with no REMB feedback (e.g. transport-cc-only receivers, or a flaky
+    # uplink that never delivers RTCP), _bandwidth_estimate() returns None and run() hits
+    # `if estimate is None: continue` — so the level never moves and the encoder stays pinned at
+    # 5 Mbps, flooding a marginal cellular uplink until webrtcd's send buffer balloons and trips the
+    # device's lowMemory soft-disable. The med rung is carriable on typical cellular; healthy links
+    # with working REMB still probe up to high within ~probe_after seconds.
+    self.level = min(1, len(self.bitrates) - 1)
+    self.stable = 0
+    self.settle = 0
+    self._auto = True
+    self.current_bitrate = self.bitrates[self.level]
+    self._publish(self.bitrates[self.level])
+
+  def start(self):
+    if self.task is None:
+      self.task = asyncio.create_task(self.run())
+
+  def stop(self):
+    if self.task is not None and not self.task.done():
+      self.task.cancel()
+    self.task = None
+
+  async def run(self):
+    while True:
+      await asyncio.sleep(self.sample_interval)
+      if not self._auto:
+        continue
+      estimate = self._bandwidth_estimate()
+      if estimate is None:
+        continue
+
+      if self.settle > 0:
+        self.settle -= 1
+        continue
+
+      if estimate < self.bitrates[self.level] * self.lower_factor:
+        while self.level > 0 and estimate < self.bitrates[self.level] * self.lower_factor:
+          self.level -= 1
+        self.stable = 0
+        self._publish(self.bitrates[self.level])
+      elif self.level < len(self.bitrates) - 1:
+        self.stable += 1
+        if self.stable >= self.probe_after:
+          self.level += 1
+          self.stable = 0
+          self.settle = self.settle_samples
+          self._publish(self.bitrates[self.level])
+      else:
+        self.stable = 0
+
+  def _bandwidth_estimate(self) -> int | None:
+    estimate = None
+    for sender in self.pc.getSenders():
+      bitrate = getattr(sender, "_remb_bitrate", None)
+      if bitrate is not None:
+        estimate = bitrate if estimate is None else min(estimate, bitrate)
+    return estimate
+
+  def set_quality(self, quality: str):
+    if quality in self.label_to_bitrate:
+      self._auto = False
+      self._publish(self.label_to_bitrate[quality])
+    elif quality == "auto":
+      self._auto = True
+
+  def _publish(self, bitrate: int):
+    # Param is registered as INT — must pass a Python int, not str. Passing str throws
+    # TypeError in Params.put (type mismatch) and crashes StreamSession.__init__ → HTTP 500.
+    self.current_bitrate = int(bitrate)
+    self.params.put("LivestreamEncoderBitrate", int(bitrate))
+
+
+class DynamicPubMaster(messaging.PubMaster):
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.lock = asyncio.Lock()
+
+  async def add_services_if_needed(self, services):
+    async with self.lock:
+      for service in services:
+        if service not in self.sock:
+          self.sock[service] = messaging.pub_sock(service)
+
+
+class StreamSession:
+  shared_pub_master = DynamicPubMaster([])
+
+  def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
+               ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
+    from aiortc.mediastreams import VideoStreamTrack, AudioStreamTrack
+    from openpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
+    from openpilot.system.webrtc.device.audio import AudioInputStreamTrack
+    from teleoprtc import WebRTCAnswerBuilder
+    from teleoprtc.info import parse_info_from_offer
+
+    config = parse_info_from_offer(sdp)
+    builder = WebRTCAnswerBuilder(sdp, ice_servers=ice_servers or [])
+
+    assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
+    self.video_tracks: list[LiveStreamVideoStreamTrack] = []
+    for cam in cameras:
+      track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()
+      if isinstance(track, LiveStreamVideoStreamTrack):
+        self.video_tracks.append(track)
+      builder.add_video_stream(cam, track)
+    # Audio init may fail if openpilot is using the audio subsystem - skip gracefully
+    if config.expected_audio_track:
+      try:
+        self.audio_input_track = AudioInputStreamTrack() if not debug_mode else AudioStreamTrack()
+        builder.add_audio_stream(self.audio_input_track)
+        self.audio_send_enabled = True
+      except Exception as e:
+        logging.warning(f"Could not init audio input (audio in use?): {e}")
+    if config.incoming_audio_track:
+      builder.offer_to_receive_audio_stream()
+
+    self.stream = builder.stream()
+    self.identifier = str(uuid.uuid4())
+
+    self.incoming_bridge: CerealIncomingMessageProxy | None = None
+    self.incoming_bridge_services = incoming_services
+    self.outgoing_bridge: CerealOutgoingMessageProxy | None = None
+    self.outgoing_bridge_runner: CerealProxyRunner | None = None
+    if len(incoming_services) > 0:
+      self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master)
+    if len(outgoing_services) > 0:
+      self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services))
+      self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)
+
+    self.ui_stream_requested = ui_stream
+    self.ui_stream_runner: CerealProxyRunner | None = None
+
+    self.incoming_audio_proxy: IncomingAudioCerealProxy | None = None
+    self.audio_input_track: AudioInputStreamTrack | AudioStreamTrack | None = None
+    self.audio_send_enabled = False
+    self.audio_recv_requested = bool(config.incoming_audio_track)
+    self.audio_send_requested = bool(config.expected_audio_track)
+    self.run_task: asyncio.Task | None = None
+    # Adaptive bitrate controller for the livestream encoder (no-op in debug mode).
+    self.bitrate_controller: LivestreamBitrateController | None = None
+    if not debug_mode and len(self.video_tracks) > 0:
+      self.bitrate_controller = LivestreamBitrateController(self.stream.peer_connection)
+    self.logger = logging.getLogger("webrtcd")
+    self.logger.info("New stream session (%s), cameras %s, audio in %s out %s, incoming services %s, outgoing services %s",
+                      self.identifier, cameras, config.incoming_audio_track, config.expected_audio_track, incoming_services, outgoing_services)
+
+  def start(self):
+    self.run_task = asyncio.create_task(self.run())
+
+  async def stop_async(self):
+    if self.run_task is not None and not self.run_task.done():
+      self.run_task.cancel()
+      try:
+        await self.run_task
+      except asyncio.CancelledError:
+        pass
+      except Exception:
+        self.logger.exception("Stream session stop task failure")
+    self.run_task = None
+    await self.post_run_cleanup()
+
+  def stop(self):
+    # Backwards-compatible sync wrapper. Prefer `await stop_async()` from async contexts.
+    try:
+      loop = asyncio.get_running_loop()
+      # If we're already in an event loop, schedule async shutdown and return.
+      loop.create_task(self.stop_async())
+      return
+    except RuntimeError:
+      pass
+    asyncio.run(self.stop_async())
+
+  async def get_answer(self):
+    return await self.stream.start()
+
+  async def message_handler(self, message: bytes):
+    # Control messages are handled in-process and don't require an incoming cereal bridge.
+    try:
+      payload = json.loads(message) if isinstance(message, (bytes, str)) else None
+    except (ValueError, TypeError):
+      payload = None
+    if isinstance(payload, dict) and payload.get("type") == "timingSei":
+      enabled = bool(payload.get("enabled", False))
+      for track in self.video_tracks:
+        track.timing_sei_enabled = enabled
+      self.logger.info("timing SEI %s", "enabled" if enabled else "disabled")
+      return
+    if isinstance(payload, dict) and payload.get("type") == "setQuality":
+      if self.bitrate_controller is not None:
+        quality = str(payload.get("quality", "auto"))
+        self.bitrate_controller.set_quality(quality)
+        self.logger.info("livestream quality set to %s", quality)
+      return
+    if isinstance(payload, dict) and payload.get("type") == "setAudioEnabled":
+      enabled = bool(payload.get("enabled", True))
+      if hasattr(self.audio_input_track, "enable"):
+        self.audio_input_track.enable(enabled)
+      self.audio_send_enabled = enabled
+      self.logger.info("livestream audio send %s", "enabled" if enabled else "disabled")
+      return
+    if isinstance(payload, dict) and payload.get("type") == "setUiStream":
+      enabled = bool(payload.get("enabled", False))
+      self.set_ui_stream(enabled)
+      self.logger.info("ui stream %s", "enabled" if enabled else "disabled")
+      return
+    if isinstance(payload, dict) and payload.get("type") == "switchCamera":
+      camera = str(payload.get("camera", ""))
+      # Single-track model: repoint the (one) video track at the requested camera.
+      for track in self.video_tracks:
+        track.switch_camera(camera)
+      return
+
+    if self.incoming_bridge is None:
+      return
+    try:
+      self.incoming_bridge.send(message)
+    except Exception:
+      self.logger.exception("Cereal incoming proxy failure")
+
+  def set_ui_stream(self, enabled: bool):
+    if enabled:
+      if self.ui_stream_runner is not None or not self.stream.has_messaging_channel():
+        return
+      from openpilot.system.webrtc.ui_stream import UIStreamMessageProxy
+      bitrate_getter = None
+      if self.bitrate_controller is not None:
+        controller = self.bitrate_controller
+
+        def bitrate_getter():
+          return controller.current_bitrate
+      proxy = UIStreamMessageProxy(bitrate_getter=bitrate_getter)
+      proxy.add_channel(self.stream.get_messaging_channel())
+      self.ui_stream_runner = CerealProxyRunner(proxy)
+      self.ui_stream_runner.start()
+    elif self.ui_stream_runner is not None:
+      self.ui_stream_runner.stop()
+      self.ui_stream_runner = None
+
+  async def add_ice_candidate(self, cand: Any):
+    """Add a trickled ICE candidate from the client to the live peer connection."""
+    if not isinstance(cand, dict):
+      return
+    cand_str = cand.get("candidate") or ""
+    if not cand_str:
+      return  # end-of-candidates marker; aiortc needs no explicit signal
+    try:
+      from aiortc.sdp import candidate_from_sdp
+      sdp_str = cand_str.split(":", 1)[-1] if cand_str.startswith("candidate:") else cand_str
+      ice = candidate_from_sdp(sdp_str)
+      ice.sdpMid = cand.get("sdpMid")
+      ice.sdpMLineIndex = cand.get("sdpMLineIndex")
+      await self.stream.peer_connection.addIceCandidate(ice)
+    except Exception:
+      self.logger.exception("Failed to add ICE candidate")
+
+  async def run(self):
+    try:
+      await self.stream.wait_for_connection()
+      if self.stream.has_messaging_channel():
+        # Always install the handler so control messages (e.g. timing SEI toggle) work
+        # even when no incoming cereal bridge service was requested.
+        self.stream.set_message_handler(self.message_handler)
+        if self.incoming_bridge is not None:
+          await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
+        if self.outgoing_bridge_runner is not None:
+          channel = self.stream.get_messaging_channel()
+          self.outgoing_bridge_runner.proxy.add_channel(channel)
+          self.outgoing_bridge_runner.start()
+        if self.ui_stream_requested:
+          self.set_ui_stream(True)
+      if self.audio_recv_requested and self.stream.has_incoming_audio_track():
+        track = self.stream.get_incoming_audio_track(buffered=False)
+        self.incoming_audio_proxy = IncomingAudioCerealProxy(track)
+        self.incoming_audio_proxy.start()
+        self.logger.info("Stream session (%s) incoming audio proxy started", self.identifier)
+      else:
+        self.logger.info("Stream session (%s) no incoming audio track from client", self.identifier)
+      if self.bitrate_controller is not None:
+        self.bitrate_controller.start()
+      self.logger.info(
+        "Stream session (%s) audio state send_requested=%s send_enabled=%s recv_requested=%s recv_active=%s",
+        self.identifier,
+        self.audio_send_requested,
+        self.audio_send_enabled,
+        self.audio_recv_requested,
+        self.incoming_audio_proxy is not None,
+      )
+      self.logger.info("Stream session (%s) connected", self.identifier)
+
+      await self.stream.wait_for_disconnection()
+      await self.post_run_cleanup()
+
+      self.logger.info("Stream session (%s) ended", self.identifier)
+    except Exception:
+      self.logger.exception("Stream session failure")
+
+  async def post_run_cleanup(self):
+    if self.bitrate_controller is not None:
+      self.bitrate_controller.stop()
+    await self.stream.stop()
+    if self.ui_stream_runner is not None:
+      self.ui_stream_runner.stop()
+      self.ui_stream_runner = None
+    if self.outgoing_bridge is not None:
+      self.outgoing_bridge_runner.stop()
+    if self.incoming_audio_proxy is not None:
+      await self.incoming_audio_proxy.stop()
+
+
+def _is_retryable_stream_error(e: Exception) -> bool:
+  # Transient failures seen during answer generation: SDP/candidate parse issues
+  # (typically browser mDNS .local host candidates aiortc can't resolve) and
+  # socket-level hiccups while gathering. Anything else is a real error.
+  return isinstance(e, (ValueError, OSError))
+
+
+async def _cleanup_failed_session(session: 'StreamSession | None', logger: logging.Logger) -> None:
+  if session is None:
+    return
+  try:
+    await session.stop_async()
+  except Exception:
+    logger.exception("Failed to clean up failed stream session")
+
+
+def _strip_mdns_host_candidates(sdp: str) -> tuple[str, int]:
+  lines = sdp.split("\r\n")
+  kept = [line for line in lines if not (line.startswith("a=candidate:") and ".local" in line)]
+  return "\r\n".join(kept), len(lines) - len(kept)
+
+
+@dataclass
+class StreamRequestBody:
+  sdp: str
+  cameras: list[str]
+  bridge_services_in: list[str] = field(default_factory=list)
+  bridge_services_out: list[str] = field(default_factory=list)
+  iceServers: list[dict[str, Any]] = field(default_factory=list)
+  ui_stream: bool = False
+
+
+async def get_stream(request: 'web.Request'):
+  stream_dict, debug_mode = request.app['streams'], request.app['debug']
+  logger = logging.getLogger("webrtcd")
+  session: StreamSession | None = None
+  try:
+    raw_body = await request.json()
+    body = StreamRequestBody(**raw_body)
+    offer_sdp = body.sdp
+
+    # Single active session on the device: tear down any prior session before starting a new
+    # one. webrtcd is long-lived (manager-owned), so without this, repeated offers would leak
+    # sessions and contend for the same livestream topics.
+    for prev in list(stream_dict.values()):
+      try:
+        await prev.stop_async()
+      except Exception:
+        logger.exception("Failed to stop previous stream session")
+    stream_dict.clear()
+
+    session = StreamSession(offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
+                            ui_stream=body.ui_stream)
+    # Creating an answer can occasionally stall (ICE gathering, codec negotiation, etc).
+    # Bound it so the HTTP request doesn't hang forever and athena can surface a useful error.
+    try:
+      answer = await asyncio.wait_for(session.get_answer(), timeout=15.0)
+    except Exception as e:
+      if not _is_retryable_stream_error(e):
+        raise
+
+      logger.warning("Transient stream creation error (%s); retrying once with a fresh session", e)
+      await _cleanup_failed_session(session, logger)
+      retry_offer_sdp, removed_mdns = _strip_mdns_host_candidates(offer_sdp)
+      if removed_mdns > 0:
+        logger.info("Retrying with SDP sanitized; removed %d mDNS host ICE candidate(s)", removed_mdns)
+      else:
+        logger.info("Retrying with fresh session and original SDP (no mDNS host candidates removed)")
+
+      session = StreamSession(retry_offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
+                              ui_stream=body.ui_stream)
+      answer = await asyncio.wait_for(session.get_answer(), timeout=15.0)
+
+    session.start()
+
+    stream_dict[session.identifier] = session
+
+    return web.json_response({"sdp": answer.sdp, "type": answer.type})
+  except TimeoutError:
+    await _cleanup_failed_session(session, logger)
+    logger.exception("Timed out generating WebRTC answer")
+    return web.json_response({"error": "answer_timeout", "message": "Timed out generating WebRTC answer"}, status=504)
+  except Exception as e:
+    await _cleanup_failed_session(session, logger)
+    logger.exception("Failed to create WebRTC stream session")
+    return web.json_response({"error": "stream_create_failed", "message": str(e)}, status=500)
+
+
+async def add_ice(request: 'web.Request'):
+  stream_dict = request.app['streams']
+  try:
+    body = await request.json()
+  except Exception:
+    return web.json_response({"error": "bad_request"}, status=400)
+  cand = body.get("candidate")
+  # Single active session on the device; apply to whatever is live.
+  for session in list(stream_dict.values()):
+    await session.add_ice_candidate(cand)
+  return web.json_response({"ok": True})
+
+
+async def get_schema(request: 'web.Request'):
+  services = request.query["services"].split(",")
+  services = [s for s in services if s]
+  assert all(s in log.Event.schema.fields and not s.endswith("DEPRECATED") for s in services), "Invalid service name"
+  schema_dict = {s: generate_field(log.Event.schema.fields[s]) for s in services}
+  return web.json_response(schema_dict)
+
+
+async def on_shutdown(app: 'web.Application'):
+  for session in app['streams'].values():
+    await session.stop_async()
+  del app['streams']
+
+
+def webrtcd_thread(host: str, port: int, debug: bool):
+  logging.basicConfig(level=logging.CRITICAL, handlers=[logging.StreamHandler()])
+  logging_level = logging.DEBUG if debug else logging.INFO
+  logging.getLogger("WebRTCStream").setLevel(logging_level)
+  logging.getLogger("webrtcd").setLevel(logging_level)
+  logging.getLogger("LiveStreamVideoStreamTrack").setLevel(logging_level)
+
+  app = web.Application()
+
+  app['streams'] = dict()
+  app['debug'] = debug
+  app.on_shutdown.append(on_shutdown)
+  app.router.add_post("/stream", get_stream)
+  app.router.add_post("/ice", add_ice)
+  app.router.add_get("/schema", get_schema)
+
+  web.run_app(app, host=host, port=port)
+
+
+def main():
+  parser = argparse.ArgumentParser(description="WebRTC daemon")
+  parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to listen on")
+  parser.add_argument("--port", type=int, default=5001, help="Port to listen on")
+  parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+  args = parser.parse_args()
+
+  webrtcd_thread(args.host, args.port, args.debug)
+
+
+if __name__=="__main__":
+  main()