IQ.Pilot Release Commit @ b6534c0

iqpilot/system/webrtc/device/native_audio.py

@@ -0,0 +1,135 @@
+import asyncio
+import contextlib
+from collections import deque
+from fractions import Fraction
+
+import av
+
+from iqpilot.cereal import messaging
+from iqpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
+
+
+WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
+WEBRTC_AUDIO_PTIME = 0.020
+
+
+class AudioInputOpusProducer:
+  def __init__(self) -> None:
+    self._sock = messaging.sub_sock("rawAudioData", conflate=False)
+    self._pcm = bytearray()
+    self._source_rate = 16_000
+    self._next_pts = 0
+    self._packet_pts = 0
+    self._pending: deque[tuple[bytes, int]] = deque()
+    self._enabled = True
+    self._resampler = av.AudioResampler(format="fltp", layout="mono", rate=48_000)
+    self._encoder = av.CodecContext.create("libopus", "w")
+    self._encoder.sample_rate = 48_000
+    self._encoder.layout = "mono"
+    self._encoder.format = "fltp"
+    self._encoder.open()
+
+  def enable(self, enabled: bool) -> None:
+    self._enabled = enabled
+
+  async def _read_pcm_frame(self) -> av.AudioFrame:
+    while True:
+      samples = max(1, int(WEBRTC_AUDIO_PTIME * self._source_rate))
+      target_bytes = samples * 2
+      while len(self._pcm) < target_bytes:
+        msg = messaging.recv_one_or_none(self._sock)
+        if msg is None:
+          await asyncio.sleep(0.002)
+          continue
+        audio = msg.rawAudioData
+        rate = int(audio.sampleRate) or self._source_rate
+        if rate != self._source_rate:
+          self._source_rate = rate
+          self._pcm.clear()
+          continue
+        self._pcm.extend(bytes(audio.data))
+
+      data = bytes(self._pcm[:target_bytes])
+      del self._pcm[:target_bytes]
+      frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
+      frame.planes[0].update(data)
+      frame.sample_rate = self._source_rate
+      return frame
+
+  async def recv(self) -> tuple[bytes, int] | None:
+    if not self._enabled:
+      await asyncio.sleep(WEBRTC_AUDIO_PTIME)
+      return None
+    while not self._pending:
+      source_frame = await self._read_pcm_frame()
+      for frame in self._resampler.resample(source_frame):
+        frame.pts = self._next_pts
+        frame.time_base = Fraction(1, 48_000)
+        self._next_pts += frame.samples
+        for packet in self._encoder.encode(frame):
+          self._pending.append((bytes(packet), self._packet_pts))
+          self._packet_pts += int(packet.duration or frame.samples)
+    return self._pending.popleft()
+
+
+class DebugAudioOpusProducer(AudioInputOpusProducer):
+  async def _read_pcm_frame(self) -> av.AudioFrame:
+    samples = int(WEBRTC_AUDIO_PTIME * self._source_rate)
+    await asyncio.sleep(WEBRTC_AUDIO_PTIME)
+    frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
+    frame.planes[0].update(bytes(samples * 2))
+    frame.sample_rate = self._source_rate
+    return frame
+
+
+class IncomingOpusCerealProxy:
+  def __init__(self, track) -> None:
+    self._loop = asyncio.get_running_loop()
+    self._queue: asyncio.Queue[bytes] = asyncio.Queue(maxsize=32)
+    self._pm = messaging.PubMaster([WEBRTC_AUDIO_SERVICE])
+    self._decoder = av.CodecContext.create("opus", "r")
+    self._resampler = av.AudioResampler(format="s16", layout="mono", rate=SOUND_SAMPLE_RATE)
+    self._task: asyncio.Task | None = None
+    track.on_frame(self._on_frame)
+
+  def _on_frame(self, payload: bytes, _info) -> None:
+    def enqueue() -> None:
+      if self._queue.full():
+        with contextlib.suppress(asyncio.QueueEmpty):
+          self._queue.get_nowait()
+      self._queue.put_nowait(bytes(payload))
+    self._loop.call_soon_threadsafe(enqueue)
+
+  def start(self) -> None:
+    if self._task is None:
+      self._task = asyncio.create_task(self.run())
+
+  async def stop(self) -> None:
+    if self._task is None:
+      return
+    self._task.cancel()
+    try:
+      await self._task
+    except asyncio.CancelledError:
+      pass
+    self._task = None
+
+  def _publish(self, frame: av.AudioFrame) -> None:
+    data = frame.to_ndarray().tobytes()
+    if not data:
+      return
+    msg = messaging.new_message(WEBRTC_AUDIO_SERVICE, valid=True)
+    msg.webrtcAudioData.data = data
+    msg.webrtcAudioData.sampleRate = frame.sample_rate
+    self._pm.send(WEBRTC_AUDIO_SERVICE, msg)
+
+  async def run(self) -> None:
+    while True:
+      payload = await self._queue.get()
+      try:
+        for decoded in self._decoder.decode(av.Packet(payload)):
+          for frame in self._resampler.resample(decoded):
+            self._publish(frame)
+      except Exception:
+        # A malformed or stale packet must not end the video/control session.
+        continue

iqpilot/system/webrtc/device/native_video.py

@@ -0,0 +1,311 @@
+import asyncio
+import logging
+import struct
+import time
+
+import av
+from iqpilot.system.webrtc.rtc.tracks import TiciVideoStreamTrack
+
+from iqpilot.cereal import messaging
+from iqpilot.common.params import Params
+from iqpilot.common.realtime import DT_MDL, DT_DMON
+
+# Arbitrary 16-byte UUID identifying konn3kt frame-timing SEI messages. When timing
+# telemetry is enabled, each frame carries a user_data_unregistered SEI NAL with four
+# big-endian doubles (ms): encode duration, IPC/queue delay, host transit, and the
+# device wall clock. The client decodes these to compute true glass-to-glass latency.
+TIMING_SEI_UUID = bytes([
+  0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e,
+  0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc,
+])
+# Annex-B start code + SEI NAL (type 6) + user_data_unregistered (type 5) + payload size
+# (0x30 = 48 bytes = 16 UUID + 32 data). Trailing 0x80 is the RBSP stop bit.
+_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID
+
+
+class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
+  livestream_camera_to_sock_mapping = {
+    "driver": "livestreamDriverEncodeData",
+    "wideRoad": "livestreamWideRoadEncodeData",
+    "road": "livestreamRoadEncodeData",
+  }
+  main_camera_to_sock_mapping = {
+    "driver": "driverEncodeData",
+    "wideRoad": "wideRoadEncodeData",
+    "road": "roadEncodeData",
+  }
+
+  # Number of live tracks still waiting for their first keyframe. The on-demand
+  # keyframe request (LivestreamRequestKeyframe) is a single global param honored by
+  # every encoder, so with multiple concurrent tracks (dual-camera PiP) we must not
+  # clear it until *all* tracks have received an IDR — otherwise the first track to
+  # get its keyframe clears the request and starves the others (black feed).
+  _kf_pending_count = 0
+
+  def __init__(self, camera_type: str):
+    dt = DT_DMON if camera_type == "driver" else DT_MDL
+    super().__init__(camera_type, dt)
+
+    self._params = Params()
+    self._camera_type = camera_type
+    self._candidate_topics = [
+      self.main_camera_to_sock_mapping[camera_type],
+      self.livestream_camera_to_sock_mapping[camera_type],
+    ]
+    self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
+    self._active_topic = self._preferred_topics()[0]
+    self._pts = 0
+    self._t0_ns = time.monotonic_ns()
+    self._cached_header: bytes = b""
+    self._sent_keyframe = False
+    self._kf_requested = False  # whether this track counts toward _kf_pending_count
+    self._frame_count = 0
+    self._last_frame_time = 0.0
+    self._last_preference_refresh = 0.0
+    # Tracks how long the H264 livestream feed has been silent, to gate the last-resort main-feed
+    # fallback (see recv) without flapping between sources frame-by-frame.
+    self._live_silent_since: float | None = None
+    # Opt-in glass-to-glass latency telemetry (toggled by the client over the data channel).
+    self.timing_sei_enabled = False
+    self._logger = logging.getLogger("LiveStreamVideoStreamTrack")
+
+    # Ask the encoder for an immediate IDR so the stream starts fast instead of waiting up to a full
+    # GOP for the next periodic keyframe (encoderd honors LivestreamRequestKeyframe per-frame).
+    self._mark_keyframe_needed()
+
+  def _request_keyframe(self, enabled: bool) -> None:
+    try:
+      self._params.put_bool("LivestreamRequestKeyframe", enabled)
+    except Exception:
+      self._logger.exception("failed to set LivestreamRequestKeyframe")
+
+  def _mark_keyframe_needed(self) -> None:
+    """This track needs (another) keyframe: keep the global request asserted."""
+    if not self._kf_requested:
+      LiveStreamVideoStreamTrack._kf_pending_count += 1
+      self._kf_requested = True
+    self._request_keyframe(True)
+
+  def request_keyframe(self) -> None:
+    """RTCP PLI hook used by libdatachannel's native H.264 packetizer."""
+    self._mark_keyframe_needed()
+
+  def _mark_keyframe_received(self) -> None:
+    """This track got its keyframe; only clear the global request once no track needs one."""
+    if self._kf_requested:
+      self._kf_requested = False
+      LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
+    if LiveStreamVideoStreamTrack._kf_pending_count == 0:
+      self._request_keyframe(False)
+
+  def stop(self):
+    # Release our pending-keyframe hold so a torn-down track that never received an
+    # IDR doesn't pin LivestreamRequestKeyframe True forever (continuous keyframes).
+    if getattr(self, "_kf_requested", False):
+      self._kf_requested = False
+      LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
+    try:
+      super().stop()
+    except Exception:
+      pass
+
+  def switch_camera(self, camera_type: str) -> None:
+    """Repoint this track at a different camera without renegotiating the peer connection.
+
+    Lets a single video track back the whole Live View — the client flips cameras over the
+    data channel and we swap the source here, instead of uplinking every camera at once."""
+    if camera_type not in self.livestream_camera_to_sock_mapping:
+      self._logger.warning("[%s] ignoring switch to unknown camera %s", self._id, camera_type)
+      return
+    if camera_type == self._camera_type:
+      return
+    self._logger.info("[%s] switching camera %s -> %s", self._id, self._camera_type, camera_type)
+    self._camera_type = camera_type
+    self._candidate_topics = [
+      self.main_camera_to_sock_mapping[camera_type],
+      self.livestream_camera_to_sock_mapping[camera_type],
+    ]
+    self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
+    self._active_topic = self._preferred_topics()[0]
+    # Force a fresh keyframe/header before emitting frames from the new source, and ask the encoder
+    # for an immediate IDR so the camera switch isn't stalled waiting for the next periodic keyframe.
+    self._cached_header = b""
+    self._sent_keyframe = False
+    self._last_preference_refresh = 0.0
+    self._live_silent_since = None
+    self._mark_keyframe_needed()
+
+  def _preferred_topics(self) -> list[str]:
+    # WebRTC currently forces H.264. The dedicated livestream topics are the H.264 feeds,
+    # while the main encode topics are the full-resolution HEVC recordings. Prefer the
+    # livestream feeds both onroad and offroad, and keep the main topics only as fallback.
+    return [
+      self.livestream_camera_to_sock_mapping[self._camera_type],
+      self.main_camera_to_sock_mapping[self._camera_type],
+    ]
+
+  def _reset_decoder_state(self, topic: str) -> None:
+    if topic == self._active_topic:
+      return
+    self._logger.info("[%s] switching video source from %s to %s", self._id, self._active_topic, topic)
+    self._active_topic = topic
+    self._cached_header = b""
+    self._sent_keyframe = False
+
+  def _timing_sei(self, evta, log_mono_time: int) -> bytes:
+    """Build a timing SEI NAL from encode metadata, or empty bytes when disabled."""
+    if not self.timing_sei_enabled:
+      return b""
+    idx = evta.idx
+    return _SEI_PREFIX + struct.pack(
+      '>4d',
+      (idx.timestampEof - idx.timestampSof) / 1e6,   # encode duration (ms)
+      (log_mono_time - idx.timestampEof) / 1e6,       # IPC/queue delay (ms)
+      (time.monotonic_ns() - log_mono_time) / 1e6,    # host transit so far (ms)
+      time.time() * 1000,                             # device wall clock (ms)  # noqa: TID251
+    ) + b'\x80'
+
+  def _is_keyframe(self, data: bytes) -> bool:
+    """Check if H.264 NAL unit contains an IDR keyframe (NAL type 5)."""
+    i = 0
+    while i < len(data) - 4:
+      # Look for Annex B start codes: 0x000001 or 0x00000001
+      if data[i:i+3] == b'\x00\x00\x01':
+        nal_type = data[i+3] & 0x1f
+        if nal_type == 5:  # IDR slice
+          return True
+        i += 3
+      elif data[i:i+4] == b'\x00\x00\x00\x01':
+        nal_type = data[i+4] & 0x1f
+        if nal_type == 5:  # IDR slice
+          return True
+        i += 4
+      else:
+        i += 1
+    return False
+
+  async def recv(self):
+    while True:
+      now = time.monotonic()
+      # Resolve topics each iteration: a camera switch (different async task) can rebuild self._socks
+      # across the await below, so a value cached before the loop would index a stale key (KeyError).
+      live_topic = self.livestream_camera_to_sock_mapping[self._camera_type]
+      main_topic = self.main_camera_to_sock_mapping[self._camera_type]
+      # Lock onto the dedicated H264 livestream feed. Onroad the HEVC main feed also publishes at
+      # 20fps; eagerly preferring whichever socket had a frame ready raced frame-by-frame, reset the
+      # decoder every frame, and (the track is negotiated H264) shoved HEVC garbage into the stream —
+      # the onroad choppiness. Only fall back to the main feed as a last resort after a long
+      # livestream silence (e.g. stream_encoderd still spinning up), and snap back when it returns.
+      msg = messaging.recv_one_or_none(self._socks[live_topic])
+      if msg is not None:
+        self._reset_decoder_state(live_topic)
+        self._last_frame_time = now
+        self._live_silent_since = None
+        break
+
+      if self._live_silent_since is None:
+        self._live_silent_since = now
+      elif now - self._live_silent_since > 3.0:
+        maybe_msg = messaging.recv_one_or_none(self._socks[main_topic])
+        if maybe_msg is not None:
+          self._reset_decoder_state(main_topic)
+          self._last_frame_time = now
+          msg = maybe_msg
+          break
+
+      await asyncio.sleep(0.005)
+
+    evta = getattr(msg, msg.which())
+
+    header = bytes(evta.header)
+    data = bytes(evta.data)
+    self._frame_count += 1
+
+    # Cache SPS/PPS header when it arrives
+    if header:
+      self._cached_header = header
+      self._logger.debug(f"[{self._id}] cached SPS/PPS header ({len(header)} bytes)")
+
+    # CRITICAL: Cannot decode without SPS/PPS. Wait for it.
+    if not self._cached_header:
+      self._logger.debug(f"[{self._id}] frame {self._frame_count}: no SPS/PPS yet, skipping")
+      return await self.recv()
+
+    is_keyframe = self._is_keyframe(data)
+
+    # Wait for first keyframe before sending any frames
+    # Browser decoder needs IDR to initialize properly
+    if not self._sent_keyframe:
+      if not is_keyframe:
+        self._logger.debug(f"[{self._id}] frame {self._frame_count}: waiting for keyframe")
+        return await self.recv()
+      self._sent_keyframe = True
+      # Got the IDR we asked for — stop nagging the encoder, but only once every
+      # concurrent track has its keyframe (multi-track PiP shares the global param).
+      self._mark_keyframe_received()
+      self._logger.info(f"[{self._id}] first keyframe received, starting stream")
+
+    # Optional timing SEI NAL, inserted before the slice data (and after SPS/PPS on keyframes).
+    sei_nal = self._timing_sei(evta, msg.logMonoTime)
+
+    # Prepend SPS/PPS header to keyframes (required by some decoders)
+    # For non-keyframes, header is optional but safe to include
+    if is_keyframe:
+      payload = self._cached_header + sei_nal + data
+    else:
+      payload = sei_nal + data
+
+    self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
+
+    packet = av.Packet(payload)
+    packet.time_base = self._time_base
+    packet.pts = int(self._pts)
+    packet.dts = int(self._pts)
+    packet.duration = int(self._dt * self._clock_rate)
+
+    if is_keyframe:
+      packet.is_keyframe = True
+
+    self.log_debug("track sending frame %s (keyframe=%s, size=%d)", self._pts, is_keyframe, len(payload))
+
+    return packet
+
+  def codec_preference(self) -> str | None:
+    return "H264"
+
+
+class DebugVideoStreamTrack(TiciVideoStreamTrack):
+  def __init__(self, camera_type: str):
+    super().__init__(camera_type, 0.05)
+    self._codec = av.CodecContext.create("libx264", "w")
+    self._codec.width = 640
+    self._codec.height = 480
+    self._codec.pix_fmt = "yuv420p"
+    self._codec.time_base = self._time_base
+    self._codec.framerate = 20
+    self._codec.options = {"preset": "ultrafast", "tune": "zerolatency"}
+    self._codec.open()
+    self._pts = 0
+    self.timing_sei_enabled = False
+
+  async def recv(self):
+    await asyncio.sleep(self._dt)
+    frame = av.VideoFrame(self._codec.width, self._codec.height, "yuv420p")
+    frame.planes[0].update(bytes(frame.planes[0].buffer_size))
+    for plane in frame.planes[1:]:
+      plane.update(bytes([128]) * plane.buffer_size)
+    frame.pts = self._pts
+    self._pts += int(self._dt * self._clock_rate)
+    packets = self._codec.encode(frame)
+    if not packets:
+      return await self.recv()
+    packet = av.Packet(b"".join(bytes(encoded) for encoded in packets))
+    packet.pts = frame.pts
+    packet.dts = frame.pts
+    packet.time_base = self._time_base
+    packet.duration = int(self._dt * self._clock_rate)
+    return packet
+
+  def switch_camera(self, camera_type: str) -> None:
+    if camera_type not in LiveStreamVideoStreamTrack.livestream_camera_to_sock_mapping:
+      raise ValueError(f"Unknown camera {camera_type}")

iqpilot/system/webrtc/rtc/__init__.py

@@ -0,0 +1,2 @@
+from .builder import WebRTCOfferBuilder, WebRTCAnswerBuilder  # noqa
+from .stream import WebRTCBaseStream, StreamingOffer, ConnectionProvider, MessageHandler  # noqa

iqpilot/system/webrtc/rtc/builder.py

@@ -0,0 +1,89 @@
+# ruff: noqa: UP006, UP035
+
+import abc
+from typing import Dict, List, Optional
+
+from .stream import RTCSessionDescription, WebRTCBaseStream, WebRTCOfferStream, WebRTCAnswerStream, ConnectionProvider
+from .tracks import TiciVideoStreamTrack, TiciTrackWrapper
+
+
+class WebRTCStreamBuilder(abc.ABC):
+  @abc.abstractmethod
+  def stream(self) -> WebRTCBaseStream:
+    raise NotImplementedError
+
+
+class WebRTCOfferBuilder(WebRTCStreamBuilder):
+  def __init__(self, connection_provider: ConnectionProvider, bind_address: Optional[str] = None,
+               ice_servers: Optional[List[dict]] = None):
+    self.connection_provider = connection_provider
+    self.bind_address = bind_address
+    self.ice_servers = ice_servers
+    self.requested_camera_types: List[str] = []
+    self.requested_audio = False
+    self.audio_tracks: List[object] = []
+    self.messaging_enabled = False
+
+  def offer_to_receive_video_stream(self, camera_type: str):
+    assert camera_type in ["driver", "wideRoad", "road"]
+    self.requested_camera_types.append(camera_type)
+
+  def offer_to_receive_audio_stream(self):
+    self.requested_audio = True
+
+  def add_audio_stream(self, track: object):
+    assert len(self.audio_tracks) == 0
+    self.audio_tracks = [track]
+
+  def add_messaging(self):
+    self.messaging_enabled = True
+
+  def stream(self) -> WebRTCBaseStream:
+    return WebRTCOfferStream(
+      self.connection_provider,
+      consumed_camera_types=self.requested_camera_types,
+      consume_audio=self.requested_audio,
+      video_producer_tracks=[],
+      audio_producer_tracks=self.audio_tracks,
+      should_add_data_channel=self.messaging_enabled,
+      bind_address=self.bind_address,
+      ice_servers=self.ice_servers,
+    )
+
+
+class WebRTCAnswerBuilder(WebRTCStreamBuilder):
+  def __init__(self, offer_sdp: str, bind_address: Optional[str] = None,
+               ice_servers: Optional[List[dict]] = None):
+    self.offer_sdp = offer_sdp
+    self.bind_address = bind_address
+    self.ice_servers = ice_servers
+    self.video_tracks: Dict[str, TiciVideoStreamTrack] = {}
+    self.requested_audio = False
+    self.audio_tracks: List[object] = []
+
+  def offer_to_receive_audio_stream(self):
+    self.requested_audio = True
+
+  def add_video_stream(self, camera_type: str, track: object):
+    assert camera_type not in self.video_tracks
+    assert camera_type in ["driver", "wideRoad", "road"]
+    if not isinstance(track, TiciVideoStreamTrack):
+      track = TiciTrackWrapper(camera_type, track)
+    self.video_tracks[camera_type] = track
+
+  def add_audio_stream(self, track: object):
+    assert len(self.audio_tracks) == 0
+    self.audio_tracks = [track]
+
+  def stream(self) -> WebRTCBaseStream:
+    description = RTCSessionDescription(sdp=self.offer_sdp, type="offer")
+    return WebRTCAnswerStream(
+      description,
+      consumed_camera_types=[],
+      consume_audio=self.requested_audio,
+      video_producer_tracks=list(self.video_tracks.values()),
+      audio_producer_tracks=self.audio_tracks,
+      should_add_data_channel=False,
+      bind_address=self.bind_address,
+      ice_servers=self.ice_servers,
+    )

iqpilot/system/webrtc/rtc/decoder.py

@@ -0,0 +1,52 @@
+# ruff: noqa: UP006, UP035
+
+import dataclasses
+import struct
+from typing import List
+
+
+@dataclasses.dataclass(frozen=True)
+class RtcpReceiverReport:
+  ssrc: int
+  fraction_lost: int
+  packets_lost: int
+  highest_seq_no: int
+  jitter: int
+  lsr: int
+  dlsr: int
+
+
+def _decode_receiver_reports(message: bytes) -> List[RtcpReceiverReport]:
+  reports: List[RtcpReceiverReport] = []
+  offset = 0
+
+  while offset + 4 <= len(message):
+    flags, packet_type, length_words = struct.unpack_from("!BBH", message, offset)
+    packet_end = offset + (length_words + 1) * 4
+    if flags >> 6 != 2 or packet_end > len(message):
+      break
+
+    report_count = flags & 0x1F
+    if packet_type == 200:  # Sender Report
+      report_offset = offset + 28
+    elif packet_type == 201:  # Receiver Report
+      report_offset = offset + 8
+    else:
+      offset = packet_end
+      continue
+
+    if report_offset + report_count * 24 > packet_end:
+      break
+
+    for i in range(report_count):
+      block_offset = report_offset + i * 24
+      ssrc, loss, highest_seq_no, jitter, lsr, dlsr = struct.unpack_from("!IIIIII", message, block_offset)
+      fraction_lost = loss >> 24
+      packets_lost = loss & 0xFFFFFF
+      if packets_lost & 0x800000:
+        packets_lost -= 1 << 24
+      reports.append(RtcpReceiverReport(ssrc, fraction_lost, packets_lost, highest_seq_no, jitter, lsr, dlsr))
+
+    offset = packet_end
+
+  return reports

iqpilot/system/webrtc/rtc/info.py

@@ -0,0 +1,37 @@
+import dataclasses
+
+from libdatachannel import Description
+
+
+@dataclasses.dataclass
+class StreamingMediaInfo:
+  n_expected_camera_tracks: int
+  expected_audio_track: bool
+  incoming_audio_track: bool
+  incoming_datachannel: bool
+
+
+def parse_info_from_offer(sdp: str) -> StreamingMediaInfo:
+  """
+  helper function to parse info about outgoing and incoming streams from an offer sdp
+  """
+  desc = Description(sdp, Description.Type.Offer)
+  n_video = 0
+  expected_audio_track = False
+  incoming_audio_track = False
+  incoming_datachannel = desc.has_application()
+
+  for i in range(desc.media_count()):
+    media = desc.media(i)
+    if media is None:
+      continue
+    direction = media.direction()
+    if media.type() == "video" and direction in (Description.Direction.RecvOnly, Description.Direction.SendRecv):
+      n_video += 1
+    elif media.type() == "audio":
+      if direction in (Description.Direction.RecvOnly, Description.Direction.SendRecv):
+        expected_audio_track = True
+      if direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
+        incoming_audio_track = True
+
+  return StreamingMediaInfo(n_video, expected_audio_track, incoming_audio_track, incoming_datachannel)

iqpilot/system/webrtc/rtc/stream.py

@@ -0,0 +1,541 @@
+# ruff: noqa: UP006, UP035
+
+import abc
+import asyncio
+import contextlib
+import dataclasses
+import logging
+import random
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple, Union
+
+from libdatachannel import (
+  Configuration,
+  DataChannel,
+  Candidate,
+  Description,
+  FrameInfo,
+  H264RtpDepacketizer,
+  H264RtpPacketizer,
+  IceServer,
+  NalUnit,
+  OpusRtpDepacketizer,
+  OpusRtpPacketizer,
+  PeerConnection,
+  PliHandler,
+  RtcpNackResponder,
+  RtcpSrReporter,
+  RtpPacketizationConfig,
+  Track,
+)
+
+from .decoder import RtcpReceiverReport, _decode_receiver_reports
+from .tracks import TiciVideoStreamTrack, parse_video_track_id
+
+
+@dataclasses.dataclass
+class StreamingOffer:
+  sdp: str
+  video: List[str]
+
+
+@dataclasses.dataclass
+class RTCSessionDescription:
+  sdp: str
+  type: str
+
+
+ConnectionProvider = Callable[[StreamingOffer], Awaitable[RTCSessionDescription]]
+MessageHandler = Callable[[Union[bytes, str]], None]
+
+
+class WebRTCBaseStream(abc.ABC):
+  def __init__(self,
+               consumed_camera_types: List[str],
+               consume_audio: bool,
+               video_producer_tracks: List[TiciVideoStreamTrack],
+               audio_producer_tracks: List[Any],
+               should_add_data_channel: bool,
+               bind_address: Optional[str] = None,
+               ice_servers: Optional[List[dict]] = None):
+    config = Configuration()
+    config.force_media_transport = True
+    config.disable_auto_negotiation = True
+    config.ice_servers = self._make_ice_servers(ice_servers)
+    if bind_address is not None:
+      config.bind_address = bind_address
+
+    self.peer_connection = PeerConnection(config)
+    self.expected_incoming_camera_types = consumed_camera_types
+    self.expected_incoming_audio = consume_audio
+    self.expected_number_of_incoming_media: Optional[int] = None
+
+    self.incoming_camera_tracks: Dict[str, Any] = {}
+    self.incoming_audio_tracks: List[Any] = []
+    self.outgoing_video_tracks = video_producer_tracks
+    self.outgoing_audio_tracks = audio_producer_tracks
+
+    self.should_add_data_channel = should_add_data_channel
+    self.messaging_channel: Optional[DataChannel] = None
+    self.incoming_message_handlers: List[MessageHandler] = []
+    self._consumer_tracks: List[Track] = []
+    self._sender_tasks: List[asyncio.Task] = []
+    self._track_state: List[Tuple[Track, TiciVideoStreamTrack, RtpPacketizationConfig]] = []
+    self._audio_track_state: List[Tuple[Track, Any, RtpPacketizationConfig]] = []
+    self._receiver_reports: Dict[str, RtcpReceiverReport] = {}
+    self._receiver_report_tracks: Dict[str, Tuple[Track, int]] = {}
+    self._negotiated_tracks: Dict[str, Track] = {}
+
+    self.incoming_media_ready_event = asyncio.Event()
+    self.messaging_channel_ready_event = asyncio.Event()
+    self.connection_attempted_event = asyncio.Event()
+    self.connection_stopped_event = asyncio.Event()
+    self.gathering_complete_event = asyncio.Event()
+    self._loop: Optional[asyncio.AbstractEventLoop] = None
+
+    self.peer_connection.on_state_change(self._on_connectionstatechange)
+    self.peer_connection.on_gathering_state_change(self._on_gatheringstatechange)
+    self.peer_connection.on_data_channel(self._on_incoming_datachannel)
+    if self.expected_incoming_camera_types or self.expected_incoming_audio:
+      self.peer_connection.on_track(self._on_incoming_track)
+
+    self.logger = logging.getLogger("WebRTCStream")
+
+  @staticmethod
+  def _make_ice_servers(servers: Optional[List[dict]]) -> List[IceServer]:
+    parsed: List[IceServer] = []
+    for server in servers or []:
+      urls = server.get("urls", []) if isinstance(server, dict) else []
+      if isinstance(urls, str):
+        urls = [urls]
+      username = str(server.get("username") or "")
+      credential = str(server.get("credential") or "")
+      for url in urls:
+        if not isinstance(url, str) or not url:
+          continue
+        try:
+          ice_server = IceServer(url)
+          ice_server.username = username
+          ice_server.password = credential
+          parsed.append(ice_server)
+        except Exception:
+          logging.getLogger("WebRTCStream").warning("Ignoring invalid ICE server %r", url, exc_info=True)
+    # A supplied list is authoritative, including LAN-only sessions where relay is deliberately filtered.
+    if servers is not None:
+      return parsed
+    return [IceServer("stun:stun.l.google.com:19302")]
+
+  def _log_debug(self, msg: Any, *args):
+    self.logger.debug(f"{type(self)}() {msg}", *args)
+
+  def _call_soon_threadsafe(self, fn: Callable, *args) -> None:
+    if self._loop is not None and self._loop.is_running():
+      self._loop.call_soon_threadsafe(fn, *args)
+    else:
+      fn(*args)
+
+  def _set_event(self, event: asyncio.Event) -> None:
+    self._call_soon_threadsafe(event.set)
+
+  @property
+  def _number_of_incoming_media(self) -> int:
+    media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks)
+    media += int(self.messaging_channel is not None) if not self.should_add_data_channel else 0
+    return media
+
+  def _add_consumer_transceivers(self):
+    for camera_type in self.expected_incoming_camera_types:
+      media = Description.Video(camera_type, Description.Direction.RecvOnly)
+      media.add_h264_codec(96)
+      track = self.peer_connection.add_track(media)
+      track.set_media_handler(H264RtpDepacketizer())
+      self._consumer_tracks.append(track)
+      self.incoming_camera_tracks[camera_type] = track
+    if self.expected_incoming_audio:
+      media = Description.Audio("audio", Description.Direction.RecvOnly)
+      media.add_opus_codec(111)
+      track = self.peer_connection.add_track(media)
+      track.set_media_handler(OpusRtpDepacketizer())
+      self._consumer_tracks.append(track)
+      self.incoming_audio_tracks.append(track)
+
+  def _find_offer_video(self, remote_sdp: str) -> Tuple[str, int]:
+    desc = Description(remote_sdp, Description.Type.Offer)
+    for i in range(desc.media_count()):
+      media = desc.media(i)
+      if media is None or media.type() != "video":
+        continue
+      for payload_type in media.payload_types():
+        with contextlib.suppress(ValueError):
+          rtp_map = media.rtp_map(payload_type)
+          if rtp_map is not None and rtp_map.format.upper() == "H264":
+            return media.mid(), payload_type
+    raise ValueError("Remote SDP does not offer H264 video")
+
+  def _make_video_media(self, track: TiciVideoStreamTrack, remote_sdp: str) -> Tuple[Description.Video, int, int, str]:
+    mid, payload_type = self._find_offer_video(remote_sdp)
+    ssrc = random.randint(1, 0xFFFFFFFF)
+    cname = f"iqpilot-video-{random.getrandbits(32):08x}"
+    stream_id = f"stream-{random.getrandbits(32):08x}"
+    media = Description.Video(mid, Description.Direction.SendOnly)
+    media.add_h264_codec(payload_type)
+    media.add_ssrc(ssrc, cname, stream_id, track.id)
+    return media, ssrc, payload_type, cname
+
+  def _find_offer_audio(self, remote_sdp: str) -> Tuple[str, int]:
+    desc = Description(remote_sdp, Description.Type.Offer)
+    for i in range(desc.media_count()):
+      media = desc.media(i)
+      if media is None or media.type() != "audio":
+        continue
+      for payload_type in media.payload_types():
+        with contextlib.suppress(ValueError):
+          rtp_map = media.rtp_map(payload_type)
+          if rtp_map is not None and rtp_map.format.upper() == "OPUS":
+            return media.mid(), payload_type
+    raise ValueError("Remote SDP does not offer Opus audio")
+
+  def _make_audio_media(self, remote_sdp: str) -> Tuple[Description.Audio, int, int, str]:
+    mid, payload_type = self._find_offer_audio(remote_sdp)
+    ssrc = random.randint(1, 0xFFFFFFFF)
+    cname = f"iqpilot-audio-{random.getrandbits(32):08x}"
+    direction = Description.Direction.SendRecv if self.expected_incoming_audio else Description.Direction.SendOnly
+    media = Description.Audio(mid, direction)
+    media.add_opus_codec(payload_type)
+    media.add_ssrc(ssrc, cname, "audio", "audio")
+    return media, ssrc, payload_type, cname
+
+  def _add_producer_tracks(self, remote_sdp: Optional[str] = None):
+    for track in self.outgoing_video_tracks:
+      media, ssrc, payload_type, cname = self._make_video_media(track, remote_sdp or "")
+      rtc_track = self._negotiated_tracks.get(media.mid())
+      if rtc_track is not None:
+        rtc_track.set_description(media)
+      else:
+        rtc_track = self.peer_connection.add_track(media)
+
+      rtp_config = RtpPacketizationConfig(ssrc, cname, payload_type, H264RtpPacketizer.CLOCK_RATE)
+      rtp_config.start_timestamp = random.randint(0, 0xFFFFFFFF)
+      rtp_config.timestamp = rtp_config.start_timestamp
+      rtp_config.sequence_number = random.randint(0, 0xFFFF)
+
+      packetizer = H264RtpPacketizer(NalUnit.Separator.LongStartSequence, rtp_config, 1200)
+      packetizer.add_to_chain(RtcpSrReporter(rtp_config))
+      packetizer.add_to_chain(PliHandler(track.request_keyframe))
+      packetizer.add_to_chain(RtcpNackResponder())
+      rtc_track.set_media_handler(packetizer)
+
+      camera_type, _ = parse_video_track_id(track.id)
+      rtc_track.reset_callbacks()
+      self._receiver_report_tracks[camera_type] = (rtc_track, ssrc)
+      self._track_state.append((rtc_track, track, rtp_config))
+
+    for producer in self.outgoing_audio_tracks:
+      media, ssrc, payload_type, cname = self._make_audio_media(remote_sdp or "")
+      if self.expected_incoming_audio and self.incoming_audio_tracks:
+        # set_remote_description() creates the browser's sendrecv audio track. Reuse that
+        # negotiated m-line for our outbound Opus instead of adding a second track with the
+        # same MID, which leaves libdatachannel stuck in signalling/ICE negotiation.
+        rtc_track = self.incoming_audio_tracks[0]
+        negotiated_media = rtc_track.description()
+        negotiated_media.clear_ssrcs()
+        negotiated_media.remove_attribute("msid")
+        negotiated_media.add_ssrc(ssrc, cname, "audio", "audio")
+        rtc_track.set_description(negotiated_media)
+      else:
+        rtc_track = self.peer_connection.add_track(media)
+      rtp_config = RtpPacketizationConfig(ssrc, cname, payload_type, OpusRtpPacketizer.DEFAULT_CLOCK_RATE)
+      rtp_config.start_timestamp = random.randint(0, 0xFFFFFFFF)
+      rtp_config.timestamp = rtp_config.start_timestamp
+      rtp_config.sequence_number = random.randint(0, 0xFFFF)
+      packetizer = OpusRtpPacketizer(rtp_config)
+      packetizer.add_to_chain(RtcpSrReporter(rtp_config))
+      packetizer.add_to_chain(RtcpNackResponder())
+      rtc_track.set_media_handler(packetizer)
+      self._audio_track_state.append((rtc_track, producer, rtp_config))
+
+  def _add_messaging_channel(self, channel: Optional[DataChannel] = None):
+    if channel is None:
+      channel = self.peer_connection.create_data_channel("data")
+    self.messaging_channel = channel
+
+    def on_message(message: Union[bytes, str]):
+      for handler in list(self.incoming_message_handlers):
+        self._call_soon_threadsafe(handler, message)
+
+    channel.on_message(on_message)
+    channel.on_open(lambda: self._set_event(self.messaging_channel_ready_event))
+    channel.on_closed(lambda: self._set_event(self.connection_stopped_event))
+    if channel.is_open():
+      self._set_event(self.messaging_channel_ready_event)
+    self._on_after_media()
+
+  def _on_connectionstatechange(self, state: PeerConnection.State):
+    self._log_debug("connection state is %s", state)
+    if state in (PeerConnection.State.Connected, PeerConnection.State.Failed):
+      self._set_event(self.connection_attempted_event)
+    if state in (PeerConnection.State.Disconnected, PeerConnection.State.Closed, PeerConnection.State.Failed):
+      self._set_event(self.connection_stopped_event)
+
+  def _on_gatheringstatechange(self, state: PeerConnection.GatheringState):
+    self._log_debug("gathering state is %s", state)
+    if state == PeerConnection.GatheringState.Complete:
+      self._set_event(self.gathering_complete_event)
+
+  def _on_incoming_track(self, track: Track):
+    self._log_debug("got track: %s", track.mid())
+    self._negotiated_tracks[track.mid()] = track
+    media_type = track.description().type()
+    if media_type == "audio":
+      if self.expected_incoming_audio:
+        self.incoming_audio_tracks.append(track)
+      self._on_after_media()
+      return
+    if media_type != "video":
+      self._on_after_media()
+      return
+    try:
+      camera_type, _ = parse_video_track_id(track.mid())
+    except ValueError:
+      camera_type = track.mid()
+    if camera_type in self.expected_incoming_camera_types:
+      self.incoming_camera_tracks[camera_type] = track
+    self._on_after_media()
+
+  def _on_incoming_datachannel(self, channel: DataChannel):
+    self._log_debug("got data channel: %s", channel.label())
+    if channel.label() == "data" and self.messaging_channel is None:
+      self._add_messaging_channel(channel)
+
+  def _update_receiver_report(self, camera_type: str, ssrc: int, message: bytes) -> None:
+    for report in _decode_receiver_reports(message):
+      if report.ssrc == ssrc:
+        self._receiver_reports[camera_type] = report
+
+  def _on_after_media(self):
+    if self.expected_number_of_incoming_media is not None and self._number_of_incoming_media >= self.expected_number_of_incoming_media:
+      self._set_event(self.incoming_media_ready_event)
+
+  def _parse_incoming_streams(self, remote_sdp: str):
+    desc = Description(remote_sdp, Description.Type.Offer)
+    media_count = 0
+    for i in range(desc.media_count()):
+      media = desc.media(i)
+      if media is None:
+        continue
+      direction = media.direction()
+      if media.type() == "video" and direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
+        media_count += 1
+      elif media.type() == "audio" and self.expected_incoming_audio and direction in (Description.Direction.SendOnly, Description.Direction.SendRecv):
+        media_count += 1
+    data_media_count = int(desc.has_application()) if not self.should_add_data_channel else 0
+    self.expected_number_of_incoming_media = media_count + data_media_count
+    if self.expected_number_of_incoming_media == 0:
+      self._set_event(self.incoming_media_ready_event)
+
+  def has_incoming_video_track(self, camera_type: str) -> bool:
+    return camera_type in self.incoming_camera_tracks
+
+  def has_incoming_audio_track(self) -> bool:
+    return len(self.incoming_audio_tracks) > 0
+
+  def has_messaging_channel(self) -> bool:
+    return self.messaging_channel is not None
+
+  def get_incoming_video_track(self, camera_type: str) -> Track:
+    assert camera_type in self.incoming_camera_tracks, "Video tracks are not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+    return self.incoming_camera_tracks[camera_type]
+
+  def get_incoming_audio_track(self) -> Track:
+    assert len(self.incoming_audio_tracks) > 0, "Audio tracks are not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+    return self.incoming_audio_tracks[0]
+
+  def get_messaging_channel(self) -> DataChannel:
+    assert self.messaging_channel is not None, "Messaging channel is not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+    return self.messaging_channel
+
+  def get_receiver_report_stats(self) -> Dict[str, RtcpReceiverReport]:
+    return dict(self._receiver_reports)
+
+  def set_message_handler(self, message_handler: MessageHandler):
+    self.incoming_message_handlers.append(message_handler)
+
+  def add_ice_candidate(self, candidate: dict) -> None:
+    candidate_sdp = str(candidate.get("candidate") or "") if isinstance(candidate, dict) else ""
+    if not candidate_sdp:
+      return
+    try:
+      self.peer_connection.add_remote_candidate(Candidate(candidate_sdp, str(candidate.get("sdpMid") or "")))
+    except Exception:
+      self.logger.warning("Ignoring invalid trickle ICE candidate", exc_info=True)
+
+  @property
+  def is_started(self) -> bool:
+    return self.peer_connection is not None and \
+           self.peer_connection.local_description() is not None and \
+           self.peer_connection.remote_description() is not None and \
+           self.peer_connection.state() != PeerConnection.State.Closed
+
+  @property
+  def is_connected_and_ready(self) -> bool:
+    return self.peer_connection is not None and \
+           self.peer_connection.state() == PeerConnection.State.Connected and \
+           (self.expected_number_of_incoming_media == 0 or self.incoming_media_ready_event.is_set())
+
+  async def _wait_for_gathering_complete(self):
+    if self.peer_connection.gathering_state() != PeerConnection.GatheringState.Complete:
+      await self.gathering_complete_event.wait()
+
+  async def _send_track_loop(self, rtc_track: Track, producer_track: TiciVideoStreamTrack, rtp_config: RtpPacketizationConfig):
+    while True:
+      if not rtc_track.is_open():
+        await asyncio.sleep(0.01)
+        continue
+
+      try:
+        packet = await producer_track.recv()
+        data = bytes(packet)
+        if not data:
+          continue
+
+        pts = int(packet.pts or 0)
+        timestamp = (rtp_config.start_timestamp + pts) & 0xFFFFFFFF
+        rtc_track.send_frame(data, FrameInfo(timestamp))
+      except asyncio.CancelledError:
+        raise
+      except Exception:
+        self.logger.exception("Error in send track loop for track %s", producer_track.id)
+        self._set_event(self.connection_stopped_event)
+        break
+
+  async def _receiver_report_loop(self):
+    while True:
+      for camera_type, (rtc_track, ssrc) in self._receiver_report_tracks.items():
+        for _ in range(32):
+          try:
+            message = rtc_track.receive()
+            if message is None: # go until queue empty (bounded to 32)
+              break
+            if isinstance(message, bytes):
+              self._update_receiver_report(camera_type, ssrc, message)
+          except asyncio.CancelledError:
+            raise
+          except Exception:
+            self.logger.exception("Error receiving report for %s", camera_type)
+            break
+      await asyncio.sleep(0.05)
+
+  async def _send_audio_track_loop(self, rtc_track: Track, producer: Any, rtp_config: RtpPacketizationConfig):
+    while True:
+      if not rtc_track.is_open():
+        await asyncio.sleep(0.01)
+        continue
+      try:
+        packet = await producer.recv()
+        if packet is None:
+          continue
+        data, pts = packet
+        if data:
+          rtc_track.send_frame(data, FrameInfo((rtp_config.start_timestamp + int(pts)) & 0xFFFFFFFF))
+      except asyncio.CancelledError:
+        raise
+      except Exception:
+        self.logger.exception("Error in audio send loop")
+        self._set_event(self.connection_stopped_event)
+        break
+
+  def _start_sender_tasks(self):
+    for rtc_track, producer_track, rtp_config in self._track_state:
+      self._sender_tasks.append(asyncio.create_task(self._send_track_loop(rtc_track, producer_track, rtp_config)))
+    for rtc_track, producer, rtp_config in self._audio_track_state:
+      self._sender_tasks.append(asyncio.create_task(self._send_audio_track_loop(rtc_track, producer, rtp_config)))
+    if self._track_state:
+      self._sender_tasks.append(asyncio.create_task(self._receiver_report_loop()))
+
+  async def wait_for_connection(self):
+    assert self.is_started
+    await self.connection_attempted_event.wait()
+    if self.peer_connection.state() != PeerConnection.State.Connected:
+      raise ValueError("Connection failed.")
+    if self.expected_number_of_incoming_media:
+      await self.incoming_media_ready_event.wait()
+    if self.messaging_channel is not None:
+      await self.messaging_channel_ready_event.wait()
+    self._start_sender_tasks()
+
+  async def wait_for_disconnection(self):
+    assert self.is_connected_and_ready, "Stream is not connected/ready yet (make sure wait_for_connection was awaited)"
+    await self.connection_stopped_event.wait()
+
+  async def stop(self):
+    for task in self._sender_tasks:
+      task.cancel()
+    for task in self._sender_tasks:
+      with contextlib.suppress(asyncio.CancelledError):
+        await task
+    self._sender_tasks.clear()
+    self.peer_connection.close()
+    self.peer_connection.reset_callbacks()
+    self.messaging_channel = None
+    self.incoming_camera_tracks.clear()
+    self.incoming_audio_tracks.clear()
+    self._consumer_tracks.clear()
+    self._track_state.clear()
+    self._audio_track_state.clear()
+    self._receiver_reports.clear()
+    self._receiver_report_tracks.clear()
+    self._negotiated_tracks.clear()
+
+  @abc.abstractmethod
+  async def start(self) -> RTCSessionDescription:
+    raise NotImplementedError
+
+
+class WebRTCOfferStream(WebRTCBaseStream):
+  def __init__(self, session_provider: ConnectionProvider, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.session_provider = session_provider
+
+  async def start(self) -> RTCSessionDescription:
+    self._loop = asyncio.get_running_loop()
+    self._add_consumer_transceivers()
+    if self.should_add_data_channel:
+      self._add_messaging_channel()
+
+    self.peer_connection.set_local_description(Description.Type.Offer)
+    await self._wait_for_gathering_complete()
+    actual_offer = self.peer_connection.local_description()
+
+    streaming_offer = StreamingOffer(
+      sdp=str(actual_offer),
+      video=list(self.expected_incoming_camera_types),
+    )
+    remote_answer = await self.session_provider(streaming_offer)
+    self._parse_incoming_streams(remote_sdp=remote_answer.sdp)
+    self.peer_connection.set_remote_description(Description(remote_answer.sdp, Description.Type.Answer))
+    self._on_after_media()
+    actual_answer = self.peer_connection.remote_description()
+
+    return RTCSessionDescription(str(actual_answer), actual_answer.type_string())
+
+
+class WebRTCAnswerStream(WebRTCBaseStream):
+  def __init__(self, session: RTCSessionDescription, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.session = session
+
+  async def start(self) -> RTCSessionDescription:
+    self._loop = asyncio.get_running_loop()
+    assert self.peer_connection.remote_description() is None, "Connection already established"
+
+    self._parse_incoming_streams(remote_sdp=self.session.sdp)
+    self.peer_connection.set_remote_description(Description(self.session.sdp, Description.Type.Offer))
+    self._add_producer_tracks(self.session.sdp)
+
+    self.peer_connection.set_local_description(Description.Type.Answer)
+    await self._wait_for_gathering_complete()
+    actual_answer = self.peer_connection.local_description()
+
+    return RTCSessionDescription(str(actual_answer), actual_answer.type_string())

iqpilot/system/webrtc/rtc/tracks.py

@@ -0,0 +1,78 @@
+# ruff: noqa: UP006, UP035
+
+import fractions
+import logging
+import uuid
+from typing import Any, Tuple
+
+
+VIDEO_CLOCK_RATE = 90000
+VIDEO_TIME_BASE = fractions.Fraction(1, VIDEO_CLOCK_RATE)
+
+
+def video_track_id(camera_type: str, track_id: str) -> str:
+  return f"{camera_type}:{track_id}"
+
+
+def parse_video_track_id(track_id: str) -> Tuple[str, str]:
+  parts = track_id.split(":")
+  if len(parts) != 2:
+    raise ValueError(f"Invalid video track id: {track_id}")
+
+  camera_type, track_id = parts
+  return camera_type, track_id
+
+
+class TiciVideoStreamTrack:
+  """
+  Abstract video track which associates video track with camera_type.
+  """
+  kind = "video"
+
+  def __init__(self, camera_type: str, dt: float, time_base: fractions.Fraction = VIDEO_TIME_BASE, clock_rate: int = VIDEO_CLOCK_RATE):
+    assert camera_type in ["driver", "wideRoad", "road"]
+    self._id: str = video_track_id(camera_type, str(uuid.uuid4()))
+    self._dt = dt
+    self._time_base: fractions.Fraction = time_base
+    self._clock_rate: int = clock_rate
+    self._logger = logging.getLogger("WebRTCStream")
+    self.readyState = "live"
+
+  @property
+  def id(self) -> str:
+    return self._id
+
+  def stop(self) -> None:
+    self.readyState = "ended"
+
+  def log_debug(self, msg: Any, *args):
+    self._logger.debug(f"{type(self)}() {msg}", *args)
+
+  async def recv(self):
+    raise NotImplementedError()
+
+  def request_keyframe(self) -> None:
+    pass
+
+
+class TiciTrackWrapper(TiciVideoStreamTrack):
+  """
+  Associates a generic video track with camera_type.
+  """
+  def __init__(self, camera_type: str, track: Any):
+    assert track.kind == "video"
+    super().__init__(camera_type, getattr(track, "_dt", 0.05))
+    try:
+      parsed_camera, _ = parse_video_track_id(track.id)
+      self._id = track.id if parsed_camera == camera_type else video_track_id(camera_type, track.id)
+    except ValueError:
+      self._id = video_track_id(camera_type, track.id)
+    self._track = track
+
+  async def recv(self):
+    return await self._track.recv()
+
+  def stop(self) -> None:
+    super().stop()
+    if hasattr(self._track, "stop"):
+      self._track.stop()

iqpilot/system/webrtc/schema.py

@@ -0,0 +1,43 @@
+import capnp
+from typing import Any
+
+
+def generate_type(type_walker, schema_walker) -> str | list[Any] | dict[str, Any]:
+  data_type = next(type_walker)
+  if data_type.which() == 'struct':
+    return generate_struct(next(schema_walker))
+  elif data_type.which() == 'list':
+    _ = next(schema_walker)
+    return [generate_type(type_walker, schema_walker)]
+  elif data_type.which() == 'enum':
+    return "text"
+  else:
+    return str(data_type.which())
+
+
+def generate_struct(schema: capnp.lib.capnp._StructSchema) -> dict[str, Any]:
+  return {field: generate_field(schema.fields[field]) for field in schema.fields if not field.endswith("DEPRECATED")}
+
+
+def generate_field(field: capnp.lib.capnp._StructSchemaField) -> str | list[Any] | dict[str, Any]:
+  def schema_walker(field):
+    yield field.schema
+
+    s = field.schema
+    while hasattr(s, 'elementType'):
+      s = s.elementType
+      yield s
+
+  def type_walker(field):
+    yield field.proto.slot.type
+
+    t = field.proto.slot.type
+    while hasattr(getattr(t, t.which()), 'elementType'):
+      t = getattr(t, t.which()).elementType
+      yield t
+
+  if field.proto.which() == "slot":
+    schema_gen, type_gen = schema_walker(field), type_walker(field)
+    return generate_type(type_gen, schema_gen)
+  else:
+    return generate_struct(field.schema)

iqpilot/system/webrtc/session.py

@@ -0,0 +1,241 @@
+import asyncio
+import contextlib
+import json
+import logging
+import os
+import socket
+import uuid
+from typing import Any
+
+from iqpilot.common.params import Params
+from iqpilot.cereal import messaging
+
+from iqpilot.system.webrtc.webrtcd import CerealIncomingMessageProxy, CerealOutgoingMessageProxy, CerealProxyRunner, DynamicPubMaster
+
+
+def _default_route_ip() -> str | None:
+  sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+  try:
+    sock.connect(("8.8.8.8", 53))
+    return sock.getsockname()[0]
+  except OSError:
+    return None
+  finally:
+    sock.close()
+
+
+class LivestreamBitrateController:
+  bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
+  label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[2]}
+  sample_interval = 0.2
+  high_loss = 0.10
+  sustained_loss = 0.05
+  down_samples = 5
+
+  def __init__(self, get_stats, enabled: bool = True):
+    self._get_stats = get_stats
+    self._enabled = enabled
+    self._auto = True
+    self._level = 1
+    self._counter = 0
+    self._up_samples = 5
+    self._previous: tuple[Any, ...] | None = None
+    self._task: asyncio.Task | None = None
+    self.current_bitrate = self.bitrates[self._level]
+    self._publish(self.current_bitrate)
+
+  def start(self) -> None:
+    if self._task is None:
+      self._task = asyncio.create_task(self.run())
+
+  def stop(self) -> None:
+    if self._task is not None:
+      self._task.cancel()
+    self._task = None
+
+  def enable(self, enabled: bool) -> None:
+    self._enabled = enabled
+
+  def set_quality(self, quality: str) -> None:
+    if quality in self.label_to_bitrate:
+      self._auto = False
+      self._publish(self.label_to_bitrate[quality])
+    elif quality == "auto":
+      self._auto = True
+
+  def _publish(self, bitrate: int) -> None:
+    self.current_bitrate = int(bitrate)
+    Params().put("LivestreamEncoderBitrate", self.current_bitrate)
+
+  def _sample_loss(self) -> float | None:
+    report = next(iter(self._get_stats().values()), None)
+    if report is None:
+      return None
+    current = (report.ssrc, report.fraction_lost, report.packets_lost, report.highest_seq_no, report.jitter, report.lsr, report.dlsr)
+    if current == self._previous:
+      return None
+    self._previous = current
+    return report.fraction_lost / 256.0
+
+  async def run(self) -> None:
+    while True:
+      await asyncio.sleep(self.sample_interval)
+      if not self._enabled or not self._auto:
+        continue
+      loss = self._sample_loss()
+      if loss is None:
+        continue
+      if loss >= self.sustained_loss and self._level > 0:
+        self._counter += 1
+        if loss >= self.high_loss or self._counter >= self.down_samples:
+          self._level -= 1
+          self._up_samples *= 2
+          self._counter = 0
+          self._publish(self.bitrates[self._level])
+      elif loss <= 0 and self._level < len(self.bitrates) - 1:
+        self._counter -= 1
+        if -self._counter >= self._up_samples:
+          self._level += 1
+          self._counter = 0
+          self._publish(self.bitrates[self._level])
+
+
+class StreamSession:
+  shared_pub_master: DynamicPubMaster | None = None
+
+  def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
+               ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
+    from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
+    from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
+    from iqpilot.system.webrtc.rtc.builder import WebRTCAnswerBuilder
+    from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
+
+    config = parse_info_from_offer(sdp)
+    if len(cameras) != config.n_expected_camera_tracks:
+      raise ValueError("Incoming stream has misconfigured number of video tracks")
+    builder = WebRTCAnswerBuilder(sdp, bind_address=_default_route_ip(), ice_servers=ice_servers or [])
+    video_track_type = DebugVideoStreamTrack if debug_mode else LiveStreamVideoStreamTrack
+    self.video_tracks = [video_track_type(camera) for camera in cameras]
+    for camera, track in zip(cameras, self.video_tracks, strict=True):
+      builder.add_video_stream(camera, track)
+
+    audio_track_type = DebugAudioOpusProducer if debug_mode else AudioInputOpusProducer
+    self.audio_output = audio_track_type() if config.expected_audio_track else None
+    if self.audio_output is not None:
+      builder.add_audio_stream(self.audio_output)
+    if config.incoming_audio_track:
+      builder.offer_to_receive_audio_stream()
+    self.stream = builder.stream()
+
+    self.identifier = str(uuid.uuid4())
+    self.incoming_bridge_services = incoming_services
+    if incoming_services and self.shared_pub_master is None:
+      StreamSession.shared_pub_master = DynamicPubMaster([])
+    self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if self.shared_pub_master is not None and incoming_services else None
+    self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services)) if outgoing_services else None
+    self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge) if self.outgoing_bridge is not None else None
+    self.ui_stream_requested = ui_stream
+    self.ui_stream_runner: CerealProxyRunner | None = None
+    self.audio_input_proxy = None
+    self.audio_recv_requested = config.incoming_audio_track
+    self.bitrate_controller = LivestreamBitrateController(self.stream.get_receiver_report_stats)
+    self.run_task: asyncio.Task | None = None
+    self._cleanup_lock = asyncio.Lock()
+    self._cleanup_done = False
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self) -> None:
+    self.run_task = asyncio.create_task(self.run())
+
+  async def get_answer(self):
+    return await self.stream.start()
+
+  async def stop_async(self) -> None:
+    if self.run_task is not None and not self.run_task.done() and self.run_task is not asyncio.current_task():
+      self.run_task.cancel()
+      with contextlib.suppress(asyncio.CancelledError):
+        await self.run_task
+    self.run_task = None
+    await self.post_run_cleanup()
+
+  def add_ice_candidate(self, candidate: Any) -> None:
+    self.stream.add_ice_candidate(candidate)
+
+  def message_handler(self, message: bytes | str) -> None:
+    try:
+      payload = json.loads(message)
+    except (TypeError, ValueError):
+      return
+    if not isinstance(payload, dict):
+      return
+    message_type = payload.get("type")
+    if message_type == "timingSei":
+      for track in self.video_tracks:
+        track.timing_sei_enabled = bool(payload.get("enabled", False))
+    elif message_type == "setQuality":
+      self.bitrate_controller.set_quality(str(payload.get("quality", "auto")))
+    elif message_type == "setAudioEnabled" and self.audio_output is not None:
+      self.audio_output.enable(bool(payload.get("enabled", True)))
+    elif message_type == "switchCamera":
+      for track in self.video_tracks:
+        track.switch_camera(str(payload.get("camera", "")))
+    elif message_type == "setUiStream":
+      self.set_ui_stream(bool(payload.get("enabled", False)))
+    elif self.incoming_bridge is not None:
+      try:
+        self.incoming_bridge.send(message)
+      except Exception:
+        self.logger.exception("Cereal incoming proxy failure")
+
+  def set_ui_stream(self, enabled: bool) -> None:
+    if enabled:
+      if self.ui_stream_runner is not None or not self.stream.has_messaging_channel():
+        return
+      from iqpilot.system.webrtc.ui_stream import UIStreamMessageProxy
+      proxy = UIStreamMessageProxy(bitrate_getter=lambda: self.bitrate_controller.current_bitrate)
+      proxy.add_channel(self.stream.get_messaging_channel())
+      self.ui_stream_runner = CerealProxyRunner(proxy)
+      self.ui_stream_runner.start()
+    elif self.ui_stream_runner is not None:
+      self.ui_stream_runner.stop()
+      self.ui_stream_runner = None
+
+  async def run(self) -> None:
+    try:
+      await self.stream.wait_for_connection()
+      if self.stream.has_messaging_channel():
+        self.stream.set_message_handler(self.message_handler)
+        if self.incoming_bridge is not None:
+          await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
+        if self.outgoing_bridge_runner is not None:
+          self.outgoing_bridge_runner.proxy.add_channel(self.stream.get_messaging_channel())
+          self.outgoing_bridge_runner.start()
+        if self.ui_stream_requested:
+          self.set_ui_stream(True)
+      if self.audio_recv_requested and self.stream.has_incoming_audio_track():
+        from iqpilot.system.webrtc.device.native_audio import IncomingOpusCerealProxy
+        self.audio_input_proxy = IncomingOpusCerealProxy(self.stream.get_incoming_audio_track())
+        self.audio_input_proxy.start()
+      self.bitrate_controller.start()
+      await self.stream.wait_for_disconnection()
+    except Exception:
+      self.logger.exception("libdatachannel stream session failure")
+    finally:
+      await self.post_run_cleanup()
+
+  async def post_run_cleanup(self) -> None:
+    async with self._cleanup_lock:
+      if self._cleanup_done:
+        return
+      self._cleanup_done = True
+      self.bitrate_controller.stop()
+      if self.ui_stream_runner is not None:
+        self.ui_stream_runner.stop()
+        self.ui_stream_runner = None
+      if self.outgoing_bridge_runner is not None:
+        self.outgoing_bridge_runner.stop()
+      if self.audio_input_proxy is not None:
+        await self.audio_input_proxy.stop()
+      for track in self.video_tracks:
+        track.stop()
+      await self.stream.stop()

iqpilot/system/webrtc/tests/test_native_session.py

@@ -0,0 +1,96 @@
+from types import SimpleNamespace
+
+import pytest
+
+from iqpilot.system.webrtc import session as session_module
+
+
+class FakeStream:
+  def get_receiver_report_stats(self):
+    return {}
+
+
+class FakeBuilder:
+  instance = None
+
+  def __init__(self, sdp, bind_address=None, ice_servers=None):
+    self.sdp = sdp
+    self.bind_address = bind_address
+    self.ice_servers = ice_servers
+    self.video = []
+    self.audio = []
+    self.receive_audio = False
+    self.result = FakeStream()
+    FakeBuilder.instance = self
+
+  def add_video_stream(self, camera, track):
+    self.video.append((camera, track))
+
+  def add_audio_stream(self, track):
+    self.audio.append(track)
+
+  def offer_to_receive_audio_stream(self):
+    self.receive_audio = True
+
+  def stream(self):
+    return self.result
+
+
+class FakeVideoTrack:
+  def __init__(self, camera):
+    self.camera = camera
+    self.timing_sei_enabled = False
+    self.switched = None
+
+  def switch_camera(self, camera):
+    self.switched = camera
+
+
+class FakeAudioProducer:
+  def __init__(self):
+    self.enabled = True
+
+  def enable(self, enabled):
+    self.enabled = enabled
+
+
+@pytest.fixture
+def native_session(mocker):
+  config = SimpleNamespace(
+    n_expected_camera_tracks=1,
+    expected_audio_track=True,
+    incoming_audio_track=True,
+    incoming_datachannel=True,
+  )
+  mocker.patch("iqpilot.system.webrtc.rtc.info.parse_info_from_offer", return_value=config)
+  mocker.patch("iqpilot.system.webrtc.rtc.builder.WebRTCAnswerBuilder", FakeBuilder)
+  mocker.patch("iqpilot.system.webrtc.device.native_video.LiveStreamVideoStreamTrack", FakeVideoTrack)
+  mocker.patch("iqpilot.system.webrtc.device.native_audio.AudioInputOpusProducer", FakeAudioProducer)
+  mocker.patch.object(session_module, "_default_route_ip", return_value="192.0.2.1")
+  mocker.patch.object(session_module, "Params", return_value=mocker.Mock())
+  return session_module.StreamSession(
+    "offer", ["road"], [], [], [{"urls": "turn:example.com"}], ui_stream=False,
+  )
+
+
+def test_native_session_builds_duplex_audio(native_session):
+  builder = FakeBuilder.instance
+  assert builder is not None
+  assert builder.bind_address == "192.0.2.1"
+  assert builder.ice_servers == [{"urls": "turn:example.com"}]
+  assert [camera for camera, _ in builder.video] == ["road"]
+  assert builder.audio == [native_session.audio_output]
+  assert builder.receive_audio
+  assert native_session.audio_recv_requested
+
+
+def test_native_session_controls(native_session, mocker):
+  native_session.bitrate_controller = mocker.Mock()
+  native_session.message_handler('{"type":"timingSei","enabled":true}')
+  assert native_session.video_tracks[0].timing_sei_enabled
+  native_session.message_handler('{"type":"switchCamera","camera":"driver"}')
+  assert native_session.video_tracks[0].switched == "driver"
+  native_session.message_handler('{"type":"setAudioEnabled","enabled":false}')
+  assert not native_session.audio_output.enabled
+  native_session.message_handler('{"type":"setQuality","quality":"low"}')
+  native_session.bitrate_controller.set_quality.assert_called_once_with("low")

iqpilot/system/webrtc/tests/test_rtc.py

@@ -0,0 +1,116 @@
+import pytest
+
+from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
+from iqpilot.system.webrtc.rtc.stream import WebRTCBaseStream
+from iqpilot.system.webrtc.rtc.tracks import TiciTrackWrapper, TiciVideoStreamTrack, parse_video_track_id, video_track_id
+
+
+def sdp_with_media(media):
+  mids = " ".join(str(i) for i in range(len(media)))
+  sections = []
+  for index, (kind, direction) in enumerate(media):
+    if kind == "video":
+      payload, mapping = "96", "H264/90000"
+      protocol = "UDP/TLS/RTP/SAVPF"
+    elif kind == "audio":
+      payload, mapping = "111", "opus/48000/2"
+      protocol = "UDP/TLS/RTP/SAVPF"
+    else:
+      sections.extend([
+        "m=application 9 UDP/DTLS/SCTP webrtc-datachannel",
+        "c=IN IP4 0.0.0.0",
+        f"a=mid:{index}",
+        "a=sctp-port:5000",
+      ])
+      continue
+    sections.extend([
+      f"m={kind} 9 {protocol} {payload}",
+      "c=IN IP4 0.0.0.0",
+      f"a=mid:{index}",
+      f"a={direction}",
+      f"a=rtpmap:{payload} {mapping}",
+      "a=rtcp-mux",
+    ])
+  lines = [
+    "v=0",
+    "o=- 1 1 IN IP4 0.0.0.0",
+    "s=-",
+    "t=0 0",
+    f"a=group:BUNDLE {mids}",
+    *sections,
+  ]
+  return "\r\n".join(lines) + "\r\n"
+
+
+@pytest.mark.parametrize("direction,expected_outgoing,expected_incoming", [
+  ("recvonly", True, False),
+  ("sendonly", False, True),
+  ("sendrecv", True, True),
+  ("inactive", False, False),
+])
+def test_audio_directions(direction, expected_outgoing, expected_incoming):
+  info = parse_info_from_offer(sdp_with_media([("audio", direction)]))
+  assert info.expected_audio_track == expected_outgoing
+  assert info.incoming_audio_track == expected_incoming
+
+
+def test_video_and_data_channel_metadata():
+  info = parse_info_from_offer(sdp_with_media([
+    ("video", "recvonly"),
+    ("video", "sendrecv"),
+    ("application", "sendrecv"),
+  ]))
+  assert info.n_expected_camera_tracks == 2
+  assert info.incoming_datachannel
+
+
+def test_explicit_empty_ice_servers_disable_defaults():
+  assert WebRTCBaseStream._make_ice_servers([]) == []
+
+
+def test_default_ice_server():
+  servers = WebRTCBaseStream._make_ice_servers(None)
+  assert len(servers) == 1
+  assert servers[0].hostname == "stun.l.google.com"
+  assert servers[0].port == 19302
+
+
+def test_authenticated_ice_servers():
+  servers = WebRTCBaseStream._make_ice_servers([{
+    "urls": ["turn:relay.example.com:3478", "stun:stun.example.com:3478"],
+    "username": "user",
+    "credential": "secret",
+  }])
+  assert [(server.hostname, server.port) for server in servers] == [
+    ("relay.example.com", 3478),
+    ("stun.example.com", 3478),
+  ]
+  assert all(server.username == "user" and server.password == "secret" for server in servers)
+
+
+def test_track_id_roundtrip():
+  assert parse_video_track_id(video_track_id("driver", "track")) == ("driver", "track")
+
+
+def test_invalid_track_id():
+  with pytest.raises(ValueError):
+    parse_video_track_id("driver")
+
+
+def test_track_wrapper_preserves_camera():
+  class Track:
+    kind = "video"
+    id = "source"
+
+    async def recv(self):
+      return b"frame"
+
+  wrapper = TiciTrackWrapper("road", Track())
+  assert parse_video_track_id(wrapper.id)[0] == "road"
+  wrapper.stop()
+  assert wrapper.readyState == "ended"
+
+
+def test_track_stores_frame_period():
+  track = TiciVideoStreamTrack("wideRoad", 0.05)
+  assert track._dt == 0.05

iqpilot/system/webrtc/tests/test_rtc_integration.py

@@ -0,0 +1,95 @@
+import asyncio
+import os
+
+import pytest
+
+from iqpilot.system.webrtc.rtc import WebRTCOfferBuilder
+from iqpilot.system.webrtc.rtc.stream import RTCSessionDescription
+from iqpilot.system.webrtc.session import StreamSession
+
+
+@pytest.mark.asyncio
+async def test_native_video_audio_and_data_channel():
+  if not os.environ.get("CI"):
+    return
+
+  answer_session = None
+  video_received = asyncio.Event()
+  audio_received = asyncio.Event()
+
+  async def connect(offer):
+    nonlocal answer_session
+    browser_offer = offer.sdp.replace("profile-level-id=42e01f", "profile-level-id=640c1f", 1)
+    audio_offset = browser_offer.index("m=audio")
+    browser_offer = browser_offer[:audio_offset] + browser_offer[audio_offset:].replace(
+      "a=recvonly",
+      "a=sendrecv\r\na=msid:ios-microphone ios-audio-track\r\na=ssrc:123456 cname:ios-audio\r\na=ssrc:123456 msid:ios-microphone ios-audio-track",
+      1,
+    )
+    answer_session = StreamSession(browser_offer, offer.video, [], [], [], debug_mode=True)
+    answer = await answer_session.get_answer()
+    assert not any(line.startswith("m=video 0 ") for line in answer.sdp.splitlines())
+    assert answer_session.stream._track_state[0][0] is answer_session.stream._negotiated_tracks["road"]
+    answer_session.start()
+    return RTCSessionDescription(answer.sdp, answer.type)
+
+  builder = WebRTCOfferBuilder(connect, ice_servers=[])
+  builder.offer_to_receive_video_stream("road")
+  builder.offer_to_receive_audio_stream()
+  builder.add_messaging()
+  stream = builder.stream()
+
+  try:
+    await asyncio.wait_for(stream.start(), 10)
+    stream.get_incoming_video_track("road").on_frame(lambda *_: video_received.set())
+    stream.get_incoming_audio_track().on_frame(lambda *_: audio_received.set())
+    await asyncio.wait_for(stream.wait_for_connection(), 10)
+    await asyncio.wait_for(video_received.wait(), 10)
+    await asyncio.wait_for(audio_received.wait(), 10)
+    stream.get_messaging_channel().send('{"type":"timingSei","enabled":true}')
+    await asyncio.sleep(0.1)
+    assert answer_session is not None
+    assert answer_session.video_tracks[0].timing_sei_enabled
+  finally:
+    await stream.stop()
+    if answer_session is not None:
+      await answer_session.stop_async()
+
+
+@pytest.mark.asyncio
+async def test_native_duplex_audio_negotiation():
+  if not os.environ.get("CI"):
+    return
+
+  answer_session = None
+
+  async def connect(offer):
+    nonlocal answer_session
+    sendrecv_offer = offer.sdp.replace(
+      "a=recvonly",
+      "a=sendrecv\r\na=msid:ios-microphone ios-audio-track\r\na=ssrc:123456 cname:ios-audio\r\na=ssrc:123456 msid:ios-microphone ios-audio-track",
+      1,
+    )
+    answer_session = StreamSession(sendrecv_offer, offer.video, [], [], [], debug_mode=True)
+    answer = await answer_session.get_answer()
+    assert "ios-microphone" not in answer.sdp
+    assert "ios-audio-track" not in answer.sdp
+    assert answer.sdp.count("a=msid:audio audio") == 1
+    answer_session.start()
+    return RTCSessionDescription(answer.sdp, answer.type)
+
+  builder = WebRTCOfferBuilder(connect, ice_servers=[])
+  builder.offer_to_receive_audio_stream()
+  stream = builder.stream()
+
+  try:
+    await asyncio.wait_for(stream.start(), 10)
+    await asyncio.wait_for(stream.wait_for_connection(), 10)
+    assert stream.has_incoming_audio_track()
+    assert answer_session is not None
+    assert answer_session.audio_recv_requested
+    assert answer_session.audio_output is not None
+  finally:
+    await stream.stop()
+    if answer_session is not None:
+      await answer_session.stop_async()

iqpilot/system/webrtc/tests/test_stream_session.py

@@ -0,0 +1,110 @@
+import asyncio
+import gc
+import json
+
+import capnp
+from iqpilot.cereal import messaging, log
+
+from iqpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
+from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
+from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
+from iqpilot.system.webrtc.rtc.tracks import VIDEO_TIME_BASE
+
+
+class TestStreamSession:
+  def setup_method(self):
+    self.loop = asyncio.new_event_loop()
+
+  def teardown_method(self):
+    self.loop.stop()
+    self.loop.close()
+    gc.collect()
+
+  def test_outgoing_proxy(self, mocker):
+    test_msg = log.Event.new_message()
+    test_msg.logMonoTime = 123
+    test_msg.valid = True
+    test_msg.customReservedRawData0 = b"test"
+    expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
+    expected_json = json.dumps(expected_dict).encode()
+
+    channel = mocker.Mock()
+    mocked_submaster = mocker.MagicMock()
+    mocked_submaster.updated = {"customReservedRawData0": True}
+    mocked_submaster.logMonoTime = {"customReservedRawData0": 123}
+    mocked_submaster.valid = {"customReservedRawData0": True}
+    mocked_submaster.__getitem__.return_value = test_msg.customReservedRawData0
+    proxy = CerealOutgoingMessageProxy(mocked_submaster)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    channel.send.assert_called_once_with(expected_json)
+
+  def test_incoming_proxy(self, mocker):
+    tested_msgs = [
+      {"type": "customReservedRawData0", "data": "test"},
+      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]},
+      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}},
+    ]
+
+    mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
+
+    proxy = CerealIncomingMessageProxy(mocked_pubmaster)
+
+    for msg in tested_msgs:
+      proxy.send(json.dumps(msg).encode())
+
+      mocked_pubmaster.send.assert_called_once()
+      mt, md = mocked_pubmaster.send.call_args.args
+      assert mt == msg["type"]
+      assert isinstance(md, capnp._DynamicStructBuilder)
+      assert hasattr(md, msg["type"])
+
+      mocked_pubmaster.reset_mock()
+
+  def test_livestream_track(self, mocker):
+    fake_msg = messaging.new_message("livestreamDriverEncodeData")
+    fake_msg.livestreamDriverEncodeData.header = b"header"
+    fake_msg.livestreamDriverEncodeData.data = b"\x00\x00\x00\x01\x65"
+
+    mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.sub_sock", return_value=mocker.Mock())
+    mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.recv_one_or_none", return_value=fake_msg)
+    track = LiveStreamVideoStreamTrack("driver")
+
+    assert track.id.startswith("driver")
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet.time_base == VIDEO_TIME_BASE
+    assert packet.pts is not None
+    assert packet.size == len(b"header\x00\x00\x00\x01\x65")
+
+  def test_input_audio_track(self, mocker):
+    packet_time, rate = 0.02, 16000
+    sample_count = int(packet_time * rate)
+    fake_msg = messaging.new_message("rawAudioData")
+    fake_msg.rawAudioData.data = b"\x00" * 2 * sample_count
+    fake_msg.rawAudioData.sampleRate = rate
+    mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.sub_sock", return_value=mocker.Mock())
+    track = AudioInputOpusProducer()
+    track._source_rate = rate
+    mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.recv_one_or_none", return_value=fake_msg)
+
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet is not None
+    payload, pts = packet
+    assert payload
+    assert pts >= 0
+
+  def test_debug_video_track(self):
+    track = DebugVideoStreamTrack("road")
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet.size > 0
+    assert packet.pts == 0
+
+  def test_debug_audio_track(self):
+    track = DebugAudioOpusProducer()
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet is not None
+    payload, pts = packet
+    assert payload
+    assert pts == 0

iqpilot/system/webrtc/tests/test_ui_stream.py

@@ -0,0 +1,303 @@
+import asyncio
+import json
+import math
+
+import pytest
+
+from iqpilot.cereal import log, messaging
+from iqpilot.system.webrtc.ui_stream import (
+  UI_STREAM_SERVICES,
+  UIStreamMessageProxy,
+  compute_ui_status,
+  frame_to_str,
+  MAX_BUFFERED_BYTES,
+)
+
+OpenpilotState = log.SelfdriveState.OpenpilotState
+
+
+def make_readers(**overrides):
+  readers = {}
+  for service in UI_STREAM_SERVICES:
+    if service == "onroadEvents":
+      msg = messaging.new_message(service, 0)
+    else:
+      msg = messaging.new_message(service)
+    readers[service] = msg
+  readers.update(overrides)
+  return {s: getattr(m, s) for s, m in readers.items()}
+
+
+class FakeSubMaster:
+  def __init__(self, readers, updated=None, valid=None):
+    self.readers = readers
+    self.updated = updated or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.valid = valid or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.logMonoTime = dict.fromkeys(UI_STREAM_SERVICES, 42)
+    self.update_calls = 0
+
+  def __getitem__(self, service):
+    return self.readers[service]
+
+  def update(self, timeout):
+    self.update_calls += 1
+
+
+class FakeChannel:
+  def __init__(self, buffered_amount=0):
+    self.bufferedAmount = buffered_amount
+    self.sent = []
+
+  def send(self, data):
+    self.sent.append(data)
+
+
+def make_proxy(sm, **kwargs):
+  return UIStreamMessageProxy(sm=sm, **kwargs)
+
+
+class TestComputeUiStatus:
+  def _msgs(self):
+    ss = messaging.new_message("selfdriveState")
+    iq = messaging.new_message("iqState")
+    ev = messaging.new_message("onroadEvents", 0)
+    return ss.selfdriveState, iq.iqState, ev.onroadEvents
+
+  def test_disengaged(self):
+    ss, iq, ev = self._msgs()
+    assert compute_ui_status(ss, iq, ev) == "disengaged"
+
+  def test_engaged_no_guidance(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+  def test_pre_enabled_is_override(self):
+    ss, iq, ev = self._msgs()
+    ss.state = OpenpilotState.preEnabled
+    assert compute_ui_status(ss, iq, ev) == "override"
+
+  def test_lat_only(self):
+    ss, iq, ev = self._msgs()
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "lat_only"
+
+  def test_long_only(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    assert compute_ui_status(ss, iq, ev) == "long_only"
+
+  def test_both_engaged(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+
+class TestUIStreamFrame:
+  def test_frame_shape_and_json(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) for i in range(33)]
+    model.position.y = [0.123456] * 33
+    model.position.z = [0.0] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [1.0, 2.0]
+      lane.y = [0.1, 0.2]
+      lane.z = [0.0, 0.0]
+    model.laneLineProbs = [0.9, 0.8, 0.7, 0.6]
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [1.0]
+      edge.y = [2.0]
+      edge.z = [0.0]
+    model.roadEdgeStds = [0.1, 0.2]
+    model.acceleration.x = [0.5] * 33
+
+    cs_msg = messaging.new_message("carState")
+    cs_msg.carState.vEgo = 12.345
+    cs_msg.carState.leftBlinker = True
+
+    readers = make_readers(modelV2=model_msg, carState=cs_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"
+    data = frame["data"]
+    assert len(data["modelV2"]["position"]["x"]) == 33
+    assert data["modelV2"]["position"]["y"][0] == 0.12
+    assert len(data["modelV2"]["laneLines"]) == 4
+    assert data["carState"]["vEgo"] == 12.35
+    assert data["carState"]["leftBlinker"] is True
+    assert data["uiStatus"] == "disengaged"
+    assert data["selfdriveState"]["alertSize"] == "none"
+    assert "hasLongitudinalControl" in data["init"]
+    assert "cameraOffset" in data["init"]
+    assert "isMetric" in data["init"]
+
+  def test_nan_scrubbed(self):
+    model_msg = messaging.new_message("modelV2")
+    model_msg.modelV2.position.x = [math.nan, math.inf, 1.0]
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    raw = channel.sent[0]
+    assert "NaN" not in raw and "Infinity" not in raw
+    frame = json.loads(raw)
+    assert frame["data"]["modelV2"]["position"]["x"] == [0.0, 0.0, 1.0]
+
+  def test_backpressure_drops_frames(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel(buffered_amount=MAX_BUFFERED_BYTES + 1)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert proxy.dropped_frames == 1
+
+  def test_no_send_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    proxy._last_emit_time = float("inf")
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert sm.update_calls == 1
+
+  def test_heartbeat_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["data"]["modelV2"] is None
+
+  def test_low_bandwidth_decimation(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 500_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 2
+
+  def test_full_rate_at_high_bitrate(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 5_000_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 4
+
+  def test_sticky_status_when_engaged_like(self):
+    ss_msg = messaging.new_message("selfdriveState")
+    ss_msg.selfdriveState.enabled = True
+    iq_msg = messaging.new_message("iqState")
+    iq_msg.iqState.aol.available = True
+    iq_msg.iqState.aol.enabled = True
+
+    readers = make_readers(selfdriveState=ss_msg, iqState=iq_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+    iq_msg.iqState.aol.available = False
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+  def test_frame_size_budget(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) * 3.03 for i in range(33)]
+    model.position.y = [1.234567] * 33
+    model.position.z = [0.456789] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [float(i) * 3.03 for i in range(33)]
+      lane.y = [1.234567] * 33
+      lane.z = [0.456789] * 33
+    model.laneLineProbs = [0.9] * 4
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [float(i) * 3.03 for i in range(33)]
+      edge.y = [1.234567] * 33
+      edge.z = [0.456789] * 33
+    model.acceleration.x = [1.23] * 33
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    frame = proxy._build_frame()
+    encoded = frame_to_str(frame)
+    assert len(encoded) < 8 * 1024
+
+
+class TestSessionWiring:
+  @pytest.mark.asyncio
+  async def test_set_ui_stream_control_message(self, mocker):
+    import logging
+    from types import SimpleNamespace
+    from iqpilot.system.webrtc.session import StreamSession
+
+    mocker.patch("iqpilot.system.webrtc.ui_stream.messaging.SubMaster", return_value=FakeSubMaster(make_readers()))
+
+    session = StreamSession.__new__(StreamSession)
+    session.logger = logging.getLogger("webrtcd")
+    session.ui_stream_runner = None
+    session.bitrate_controller = None
+    session.incoming_bridge = None
+    channel = FakeChannel()
+    session.stream = SimpleNamespace(
+      has_messaging_channel=lambda: True,
+      get_messaging_channel=lambda: channel,
+    )
+
+    session.message_handler(b'{"type":"setUiStream","enabled":true}')
+    assert session.ui_stream_runner is not None
+    await asyncio.sleep(0.05)
+    session.message_handler(b'{"type":"setUiStream","enabled":false}')
+    assert session.ui_stream_runner is None
+
+    assert len(channel.sent) >= 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"

iqpilot/system/webrtc/tests/test_webrtcd.py

@@ -0,0 +1,75 @@
+import json
+from types import SimpleNamespace
+
+from parameterized import parameterized_class
+import pytest
+
+from iqpilot.system.webrtc.webrtcd import add_ice, get_stream
+
+
+class FakeSession:
+  instances = []
+
+  def __init__(self, identifier="session"):
+    self.identifier = identifier
+    self.started = False
+    self.stopped = False
+    self.candidates = []
+    self.instances.append(self)
+
+  async def get_answer(self):
+    return SimpleNamespace(sdp="answer", type="answer")
+
+  def start(self):
+    self.started = True
+
+  async def stop_async(self):
+    self.stopped = True
+
+  async def add_ice_candidate(self, candidate):
+    self.candidates.append(candidate)
+
+
+@parameterized_class(("in_services", "out_services"), [
+  (["testJoystick"], ["carState"]),
+  ([], ["carState"]),
+  (["testJoystick"], []),
+  ([], []),
+])
+@pytest.mark.asyncio
+class TestWebrtcdProc:
+  async def test_webrtcd(self, mocker):
+    session = FakeSession()
+    mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
+    request = mocker.MagicMock()
+    request.app = {"streams": {}, "debug": False}
+    request.json = mocker.AsyncMock(return_value={
+      "sdp": "offer",
+      "cameras": ["road"],
+      "bridge_services_in": self.in_services,
+      "bridge_services_out": self.out_services,
+    })
+
+    response = await get_stream(request)
+
+    assert response.status == 200
+    assert json.loads(response.text) == {"sdp": "answer", "type": "answer"}
+    assert request.app["streams"] == {session.identifier: session}
+    assert session.started
+
+  async def test_replaces_session_and_routes_ice(self, mocker):
+    previous = FakeSession("previous")
+    session = FakeSession("current")
+    mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
+    request = mocker.MagicMock()
+    request.app = {"streams": {previous.identifier: previous}, "debug": False}
+    request.json = mocker.AsyncMock(return_value={"sdp": "offer", "cameras": ["road"]})
+
+    response = await get_stream(request)
+
+    assert response.status == 200
+    assert previous.stopped
+    request.json = mocker.AsyncMock(return_value={"candidate": {"candidate": "candidate:1"}})
+    ice_response = await add_ice(request)
+    assert ice_response.status == 200
+    assert session.candidates == [{"candidate": "candidate:1"}]

iqpilot/system/webrtc/ui_stream.py

@@ -0,0 +1,269 @@
+import json
+import math
+import time
+
+import numpy as np
+
+from iqpilot.cereal import car, log, custom, messaging
+from iqpilot.common.params import Params
+
+OpenpilotState = log.SelfdriveState.OpenpilotState
+GuidanceState = custom.AlwaysOnLateral.AlwaysOnLateralState
+
+UI_STREAM_SERVICES = [
+  "modelV2", "carState", "selfdriveState", "controlsState", "extrinsicsCalibration",
+  "radarState", "longitudinalPlan", "deviceState", "roadCameraState",
+  "iqState", "onroadEvents",
+]
+
+# Above this the viewer is not draining the channel; drop frames instead of queueing,
+# telemetry is newest-wins and unbounded SCTP buffering is how webrtcd leaked before.
+MAX_BUFFERED_BYTES = 256 * 1024
+
+# Bitrate at/below which modelV2 frames are decimated to half rate to leave
+# headroom for video on a struggling uplink.
+LOW_BANDWIDTH_BITRATE = 500_000
+
+HEARTBEAT_INTERVAL = 1.0
+
+
+def _round_list(vals, decimals: int) -> list[float]:
+  arr = np.asarray(vals, dtype=np.float64)
+  if arr.size == 0:
+    return []
+  arr = np.round(np.where(np.isfinite(arr), arr, 0.0), decimals)
+  return arr.tolist()
+
+
+def _round_float(val, decimals: int = 3, default: float = 0.0) -> float:
+  try:
+    v = float(val)
+  except (TypeError, ValueError):
+    return default
+  return round(v, decimals) if math.isfinite(v) else default
+
+
+def _xyz(line, decimals: int = 2) -> dict[str, list[float]]:
+  return {
+    "x": _round_list(line.x, decimals),
+    "y": _round_list(line.y, decimals),
+    "z": _round_list(line.z, decimals),
+  }
+
+
+def _lead(lead) -> dict:
+  return {
+    "status": bool(lead.status),
+    "dRel": _round_float(lead.dRel, 2),
+    "yRel": _round_float(lead.yRel, 2),
+    "vRel": _round_float(lead.vRel, 2),
+  }
+
+
+def compute_ui_status(ss, iq_state, onroad_events) -> str:
+  # Mirrors IQUIState.update_status; that module pulls in the raylib UI stack,
+  # which must not be imported into webrtcd.
+  guidance = iq_state.aol
+  guidance_state = guidance.state
+
+  if ss.state == OpenpilotState.preEnabled:
+    return "override"
+
+  if ss.state == OpenpilotState.overriding:
+    if not guidance.available:
+      return "override"
+    if any(e.overrideLongitudinal for e in onroad_events):
+      return "override"
+
+  if guidance_state in (GuidanceState.paused, GuidanceState.overriding):
+    return "override"
+
+  if not guidance.available:
+    return "engaged" if ss.enabled else "disengaged"
+
+  if not guidance.enabled and not ss.enabled:
+    return "disengaged"
+
+  if guidance.enabled and ss.enabled:
+    return "engaged"
+
+  if guidance.enabled:
+    return "lat_only"
+
+  if ss.enabled:
+    return "long_only"
+
+  return "disengaged"
+
+
+def build_init_payload(params: Params | None = None) -> dict:
+  params = params or Params()
+
+  has_longitudinal_control = False
+  cp_bytes = params.get("CarParamsPersistent")
+  if cp_bytes is not None:
+    try:
+      cp = messaging.log_from_bytes(cp_bytes, car.CarParams)
+      if cp.alphaLongitudinalAvailable:
+        has_longitudinal_control = params.get_bool("AlphaLongitudinalEnabled")
+      else:
+        has_longitudinal_control = bool(cp.openpilotLongitudinalControl)
+    except Exception:
+      pass
+
+  camera_offset = 0.0
+  if params.get("ModelManager_ActiveBundle"):
+    try:
+      camera_offset = float(params.get("CameraOffset", return_default=True) or 0.0)
+    except (TypeError, ValueError):
+      camera_offset = 0.0
+
+  return {
+    "hasLongitudinalControl": has_longitudinal_control,
+    "cameraOffset": _round_float(camera_offset, 3),
+    "isMetric": params.get_bool("IsMetric"),
+  }
+
+
+class UIStreamMessageProxy:
+  """Sends a trimmed, HUD-only JSON projection of UI state over the session data
+  channel, clocked by modelV2 (~20Hz). Payload stays a few KB per frame; anything
+  the client renderers don't read is not serialized."""
+
+  def __init__(self, sm: messaging.SubMaster | None = None, bitrate_getter=None):
+    self.sm = sm if sm is not None else messaging.SubMaster(UI_STREAM_SERVICES)
+    self.channels = []
+    self.bitrate_getter = bitrate_getter
+    self.dropped_frames = 0
+    self._last_non_disengaged = "disengaged"
+    self._last_emit_time = 0.0
+    self._decimate_flip = False
+    self._init_payload = build_init_payload()
+
+  def add_channel(self, channel):
+    self.channels.append(channel)
+
+  def update(self):
+    self.sm.update(0)
+
+    model_updated = self.sm.updated["modelV2"]
+    now = time.monotonic()
+    if not model_updated:
+      if now - self._last_emit_time < HEARTBEAT_INTERVAL:
+        return
+    elif self._low_bandwidth():
+      self._decimate_flip = not self._decimate_flip
+      if self._decimate_flip:
+        return
+
+    # Send as a text frame: react-native-webrtc surfaces binary frames as
+    # ArrayBuffers that Hermes cannot reliably decode without TextDecoder.
+    frame = self._build_frame(include_model=model_updated)
+    encoded = frame_to_str(frame)
+    self._last_emit_time = now
+    for channel in self.channels:
+      if channel.bufferedAmount > MAX_BUFFERED_BYTES:
+        self.dropped_frames += 1
+        continue
+      channel.send(encoded)
+
+  def _low_bandwidth(self) -> bool:
+    if self.bitrate_getter is None:
+      return False
+    try:
+      bitrate = self.bitrate_getter()
+    except Exception:
+      return False
+    return bitrate is not None and bitrate <= LOW_BANDWIDTH_BITRATE
+
+  def _ui_status(self) -> str:
+    sm = self.sm
+    ss = sm["selfdriveState"]
+    iq_state = sm["iqState"]
+    status = compute_ui_status(ss, iq_state, sm["onroadEvents"])
+
+    # Same stickiness as UIState._update_status: while still engaged-like, a
+    # transient disengaged classification keeps the last non-disengaged status.
+    if status != "disengaged":
+      self._last_non_disengaged = status
+      return status
+
+    if ss.enabled or iq_state.aol.enabled:
+      if self._last_non_disengaged != "disengaged":
+        return self._last_non_disengaged
+      return "engaged" if ss.enabled else "disengaged"
+
+    self._last_non_disengaged = "disengaged"
+    return "disengaged"
+
+  def _build_frame(self, include_model: bool = True) -> dict:
+    sm = self.sm
+    cs = sm["carState"]
+    ss = sm["selfdriveState"]
+    calib = sm["extrinsicsCalibration"]
+    radar = sm["radarState"]
+    device_state = sm["deviceState"]
+
+    model_data = None
+    if include_model:
+      model = sm["modelV2"]
+      model_data = {
+        "position": _xyz(model.position),
+        "laneLines": [_xyz(line) for line in model.laneLines],
+        "laneLineProbs": _round_list(model.laneLineProbs, 3),
+        "roadEdges": [_xyz(edge) for edge in model.roadEdges],
+        "roadEdgeStds": _round_list(model.roadEdgeStds, 3),
+        "acceleration": {"x": _round_list(model.acceleration.x, 2)},
+      }
+
+    data = {
+      "modelV2": model_data,
+      "carState": {
+        "vEgo": _round_float(cs.vEgo, 2),
+        "vEgoCluster": _round_float(cs.vEgoCluster, 2),
+        "vCruiseCluster": _round_float(cs.vCruiseCluster, 2),
+        "leftBlinker": bool(cs.leftBlinker),
+        "rightBlinker": bool(cs.rightBlinker),
+      },
+      "selfdriveState": {
+        "enabled": bool(ss.enabled),
+        "experimentalMode": bool(ss.experimentalMode),
+        "state": str(ss.state),
+        "alertText1": str(ss.alertText1),
+        "alertText2": str(ss.alertText2),
+        "alertSize": str(ss.alertSize),
+        "alertStatus": str(ss.alertStatus),
+      },
+      "controlsState": {
+        "vCruiseDEPRECATED": _round_float(sm["controlsState"].vCruiseDEPRECATED, 2),
+      },
+      "extrinsicsCalibration": {
+        "calStatus": str(calib.calStatus),
+        "rpyCalib": _round_list(calib.rpyCalib, 5),
+        "wideFromDeviceEuler": _round_list(calib.wideFromDeviceEuler, 5),
+        "height": _round_list(calib.height, 3),
+      },
+      "radarState": {
+        "valid": bool(sm.valid["radarState"]),
+        "leadOne": _lead(radar.leadOne),
+        "leadTwo": _lead(radar.leadTwo),
+      },
+      "longitudinalPlan": {
+        "allowThrottle": bool(sm["longitudinalPlan"].allowThrottle),
+      },
+      "deviceState": {
+        "deviceType": str(device_state.deviceType),
+        "started": bool(device_state.started),
+      },
+      "roadCameraState": {
+        "sensor": str(sm["roadCameraState"].sensor),
+      },
+      "uiStatus": self._ui_status(),
+      "init": self._init_payload,
+    }
+
+    return {"type": "uiStream", "logMonoTime": sm.logMonoTime["modelV2"], "data": data}
+
+
+def frame_to_str(frame: dict) -> str:
+  return json.dumps(frame, separators=(",", ":"))

iqpilot/system/webrtc/webrtcd.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+import json
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning)
+
+import capnp
+from aiohttp import web
+
+from iqpilot.system.webrtc.schema import generate_field
+from iqpilot.cereal import messaging, log
+
+
+class CerealOutgoingMessageProxy:
+  def __init__(self, sm: messaging.SubMaster):
+    self.sm = sm
+    self.channels: list[Any] = []
+
+  def add_channel(self, channel: Any):
+    self.channels.append(channel)
+
+  def to_json(self, msg_content: Any):
+    if isinstance(msg_content, capnp._DynamicStructReader):
+      msg_dict = msg_content.to_dict()
+    elif isinstance(msg_content, capnp._DynamicListReader):
+      msg_dict = [self.to_json(msg) for msg in msg_content]
+    elif isinstance(msg_content, bytes):
+      msg_dict = msg_content.decode()
+    else:
+      msg_dict = msg_content
+
+    return msg_dict
+
+  def update(self):
+    # this is blocking in async context...
+    self.sm.update(0)
+    for service, updated in self.sm.updated.items():
+      if not updated:
+        continue
+      msg_dict = self.to_json(self.sm[service])
+      mono_time, valid = self.sm.logMonoTime[service], self.sm.valid[service]
+      outgoing_msg = {"type": service, "logMonoTime": mono_time, "valid": valid, "data": msg_dict}
+      encoded_msg = json.dumps(outgoing_msg).encode()
+      for channel in self.channels:
+        channel.send(encoded_msg)
+
+
+class CerealIncomingMessageProxy:
+  def __init__(self, pm: messaging.PubMaster):
+    self.pm = pm
+
+  def send(self, message: bytes):
+    msg_json = json.loads(message)
+    msg_type, msg_data = msg_json["type"], msg_json["data"]
+    size = None
+    if not isinstance(msg_data, dict):
+      size = len(msg_data)
+
+    msg = messaging.new_message(msg_type, size=size)
+    setattr(msg, msg_type, msg_data)
+    self.pm.send(msg_type, msg)
+
+
+class AsyncTaskRunner:
+  def __init__(self):
+    self.task: asyncio.Task | None = None
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self):
+    if self.task is None:
+      self.task = asyncio.create_task(self.run())
+
+  async def stop(self):
+    if self.task is None:
+      return
+    if not self.task.done():
+      self.task.cancel()
+      try:
+        await self.task
+      except asyncio.CancelledError:
+        pass
+    self.task = None
+
+
+class CerealProxyRunner:
+  def __init__(self, proxy: CerealOutgoingMessageProxy):
+    self.proxy = proxy
+    self.is_running = False
+    self.task = None
+    self.logger = logging.getLogger("webrtcd")
+
+  def start(self):
+    assert self.task is None
+    self.task = asyncio.create_task(self.run())
+
+  def stop(self):
+    if self.task is None or self.task.done():
+      return
+    self.task.cancel()
+    self.task = None
+
+  async def run(self):
+    while True:
+      try:
+        self.proxy.update()
+      except Exception:
+        self.logger.exception("Cereal outgoing proxy failure")
+      await asyncio.sleep(0.01)
+
+
+class DynamicPubMaster(messaging.PubMaster):
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.lock = asyncio.Lock()
+
+  async def add_services_if_needed(self, services):
+    async with self.lock:
+      for service in services:
+        if service not in self.sock:
+          self.sock[service] = messaging.pub_sock(service)
+
+
+def _is_retryable_stream_error(e: Exception) -> bool:
+  return isinstance(e, (ValueError, OSError))
+
+
+async def _cleanup_failed_session(session: Any | None, logger: logging.Logger) -> None:
+  if session is None:
+    return
+  try:
+    await session.stop_async()
+  except Exception:
+    logger.exception("Failed to clean up failed stream session")
+
+
+def _strip_mdns_host_candidates(sdp: str) -> tuple[str, int]:
+  lines = sdp.split("\r\n")
+  kept = [line for line in lines if not (line.startswith("a=candidate:") and ".local" in line)]
+  return "\r\n".join(kept), len(lines) - len(kept)
+
+
+@dataclass
+class StreamRequestBody:
+  sdp: str
+  cameras: list[str]
+  bridge_services_in: list[str] = field(default_factory=list)
+  bridge_services_out: list[str] = field(default_factory=list)
+  iceServers: list[dict[str, Any]] = field(default_factory=list)
+  ui_stream: bool = False
+
+
+def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: bool):
+  from iqpilot.system.webrtc.session import StreamSession
+  return StreamSession(
+    offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
+    ui_stream=body.ui_stream,
+  )
+
+
+async def get_stream(request: 'web.Request'):
+  stream_dict, debug_mode = request.app['streams'], request.app['debug']
+  logger = logging.getLogger("webrtcd")
+  session: Any | None = None
+  try:
+    raw_body = await request.json()
+    body = StreamRequestBody(**raw_body)
+    offer_sdp = body.sdp
+
+    # Single active session on the device: tear down any prior session before starting a new
+    # one. webrtcd is long-lived (manager-owned), so without this, repeated offers would leak
+    # sessions and contend for the same livestream topics.
+    for prev in list(stream_dict.values()):
+      try:
+        await prev.stop_async()
+      except Exception:
+        logger.exception("Failed to stop previous stream session")
+    stream_dict.clear()
+
+    session = _new_stream_session(offer_sdp, body, debug_mode)
+    # Creating an answer can occasionally stall (ICE gathering, codec negotiation, etc).
+    # Bound it so the HTTP request doesn't hang forever and athena can surface a useful error.
+    try:
+      answer = await asyncio.wait_for(session.get_answer(), timeout=15.0)
+    except Exception as e:
+      if not _is_retryable_stream_error(e):
+        raise
+
+      logger.warning("Transient stream creation error (%s); retrying once with a fresh session", e)
+      await _cleanup_failed_session(session, logger)
+      retry_offer_sdp, removed_mdns = _strip_mdns_host_candidates(offer_sdp)
+      if removed_mdns > 0:
+        logger.info("Retrying with SDP sanitized; removed %d mDNS host ICE candidate(s)", removed_mdns)
+      else:
+        logger.info("Retrying with fresh session and original SDP (no mDNS host candidates removed)")
+
+      session = _new_stream_session(retry_offer_sdp, body, debug_mode)
+      answer = await asyncio.wait_for(session.get_answer(), timeout=15.0)
+
+    session.start()
+
+    stream_dict[session.identifier] = session
+
+    return web.json_response({"sdp": answer.sdp, "type": answer.type})
+  except TimeoutError:
+    await _cleanup_failed_session(session, logger)
+    logger.exception("Timed out generating WebRTC answer")
+    return web.json_response({"error": "answer_timeout", "message": "Timed out generating WebRTC answer"}, status=504)
+  except Exception as e:
+    await _cleanup_failed_session(session, logger)
+    logger.exception("Failed to create WebRTC stream session")
+    return web.json_response({"error": "stream_create_failed", "message": str(e)}, status=500)
+
+
+async def add_ice(request: 'web.Request'):
+  stream_dict = request.app['streams']
+  try:
+    body = await request.json()
+  except Exception:
+    return web.json_response({"error": "bad_request"}, status=400)
+  cand = body.get("candidate")
+  # Single active session on the device; apply to whatever is live.
+  for session in list(stream_dict.values()):
+    await session.add_ice_candidate(cand)
+  return web.json_response({"ok": True})
+
+
+async def get_schema(request: 'web.Request'):
+  services = request.query["services"].split(",")
+  services = [s for s in services if s]
+  assert all(s in log.Event.schema.fields and not s.endswith("DEPRECATED") for s in services), "Invalid service name"
+  schema_dict = {s: generate_field(log.Event.schema.fields[s]) for s in services}
+  return web.json_response(schema_dict)
+
+
+async def on_shutdown(app: 'web.Application'):
+  for session in app['streams'].values():
+    await session.stop_async()
+  del app['streams']
+
+
+def webrtcd_thread(host: str, port: int, debug: bool):
+  logging.basicConfig(level=logging.CRITICAL, handlers=[logging.StreamHandler()])
+  logging_level = logging.DEBUG if debug else logging.INFO
+  logging.getLogger("WebRTCStream").setLevel(logging_level)
+  logging.getLogger("webrtcd").setLevel(logging_level)
+  logging.getLogger("LiveStreamVideoStreamTrack").setLevel(logging_level)
+
+  app = web.Application()
+
+  app['streams'] = dict()
+  app['debug'] = debug
+  app.on_shutdown.append(on_shutdown)
+  app.router.add_post("/stream", get_stream)
+  app.router.add_post("/ice", add_ice)
+  app.router.add_get("/schema", get_schema)
+
+  web.run_app(app, host=host, port=port)
+
+
+def main():
+  parser = argparse.ArgumentParser(description="WebRTC daemon")
+  parser.add_argument("--host", type=str, default="0.0.0.0", help="Host to listen on")
+  parser.add_argument("--port", type=int, default=5001, help="Port to listen on")
+  parser.add_argument("--debug", action="store_true", help="Enable debug mode")
+  args = parser.parse_args()
+
+  webrtcd_thread(args.host, args.port, args.debug)
+
+
+if __name__=="__main__":
+  main()