IQ.Pilot Release Commit @ 3fe374f

This commit is contained in:
IQ.Lvbs CI [bot]
2026-08-24 09:07:31 -05:00
parent 7954d43e00
commit 2f0ec679ec
99 changed files with 805 additions and 3278 deletions

View File

@@ -1,105 +0,0 @@
import asyncio
import fractions
import aiortc
import av
import numpy as np
from iqpilot.cereal import messaging
WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
WEBRTC_AUDIO_PTIME = 0.020
class AudioInputStreamTrack(aiortc.mediastreams.AudioStreamTrack):
"""Device microphone -> WebRTC, sourced from micd's `rawAudioData` cereal stream.
micd owns the ALSA capture device, so opening it again via PyAudio fails with a host error
('audio in use', PortAudio errno -9999). Instead we consume micd's already-published int16 mono
PCM and repacketize it into WebRTC audio frames — no device contention, and it works whenever micd
is running. Reading one message per recv() paces playout to micd's real-time publish rate.
"""
def __init__(self, rate: int = 16000, channels: int = 1):
super().__init__()
self.rate = rate
self.channels = channels
# conflate=False: keep audio continuous (don't drop buffered samples) for clean playback.
self._sock = messaging.sub_sock("rawAudioData", conflate=False)
self._start: float | None = None
self.pts = 0
self.enabled = True
self._audio_buffer = bytearray()
def enable(self, enabled: bool) -> None:
self.enabled = enabled
async def _fill_audio_buffer(self, target_bytes: int) -> None:
deadline = asyncio.get_running_loop().time() + WEBRTC_AUDIO_PTIME
while len(self._audio_buffer) < target_bytes:
msg = messaging.recv_one_or_none(self._sock)
if msg is not None:
audio = msg.rawAudioData
rate = int(audio.sampleRate) or self.rate
if rate != self.rate:
self.rate = rate
self._audio_buffer.clear()
self._start = None
self.pts = 0
self._audio_buffer.extend(bytes(audio.data))
continue
if asyncio.get_running_loop().time() >= deadline:
break
await asyncio.sleep(0.005)
async def _next_audio_data(self) -> tuple[bytes, int]:
samples = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
target_bytes = samples * 2
await self._fill_audio_buffer(target_bytes)
if len(self._audio_buffer) >= target_bytes:
data = bytes(self._audio_buffer[:target_bytes])
del self._audio_buffer[:target_bytes]
else:
data = bytes(self._audio_buffer)
self._audio_buffer.clear()
data += bytes(target_bytes - len(data))
return data, self.rate
async def _pace(self, pts: int, sample_rate: int) -> None:
if self._start is None:
self._start = asyncio.get_running_loop().time()
return
wait = self._start + (pts / sample_rate) - asyncio.get_running_loop().time()
if wait > 0:
await asyncio.sleep(wait)
async def recv(self):
while True:
if not self.enabled:
break
data, sample_rate = await self._next_audio_data()
if data:
samples = len(data) // 2
pts = self.pts
self.pts += samples
await self._pace(pts, sample_rate)
frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
frame.planes[0].update(data)
frame.pts = pts
frame.sample_rate = sample_rate
frame.time_base = fractions.Fraction(1, sample_rate)
return frame
samples_per_frame = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
samples = np.zeros((1, samples_per_frame), dtype=np.int16)
frame = av.AudioFrame.from_ndarray(samples, format='s16', layout='mono')
frame.sample_rate = self.rate
frame.time_base = fractions.Fraction(1, self.rate)
frame.pts = self.pts
self.pts += frame.samples
await self._pace(frame.pts, self.rate)
return frame

View File

@@ -7,17 +7,19 @@ import av
from iqpilot.cereal import messaging
from iqpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
from iqpilot.system.webrtc.device.audio import WEBRTC_AUDIO_PTIME, WEBRTC_AUDIO_SERVICE
WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
WEBRTC_AUDIO_PTIME = 0.020
class AudioInputOpusProducer:
"""Micd PCM -> 48 kHz Opus payloads for libdatachannel's RTP packetizer."""
def __init__(self) -> None:
self._sock = messaging.sub_sock("rawAudioData", conflate=False)
self._pcm = bytearray()
self._source_rate = 16_000
self._next_pts = 0
self._packet_pts = 0
self._pending: deque[tuple[bytes, int]] = deque()
self._enabled = True
self._resampler = av.AudioResampler(format="fltp", layout="mono", rate=48_000)
@@ -65,13 +67,22 @@ class AudioInputOpusProducer:
frame.time_base = Fraction(1, 48_000)
self._next_pts += frame.samples
for packet in self._encoder.encode(frame):
self._pending.append((bytes(packet), int(packet.pts or 0)))
self._pending.append((bytes(packet), self._packet_pts))
self._packet_pts += int(packet.duration or frame.samples)
return self._pending.popleft()
class IncomingOpusCerealProxy:
"""libdatachannel Opus payloads -> soundd-compatible PCM cereal messages."""
class DebugAudioOpusProducer(AudioInputOpusProducer):
async def _read_pcm_frame(self) -> av.AudioFrame:
samples = int(WEBRTC_AUDIO_PTIME * self._source_rate)
await asyncio.sleep(WEBRTC_AUDIO_PTIME)
frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
frame.planes[0].update(bytes(samples * 2))
frame.sample_rate = self._source_rate
return frame
class IncomingOpusCerealProxy:
def __init__(self, track) -> None:
self._loop = asyncio.get_running_loop()
self._queue: asyncio.Queue[bytes] = asyncio.Queue(maxsize=32)

View File

@@ -4,7 +4,7 @@ import struct
import time
import av
from iqpilot.system.webrtc.teleoprtc_ldc.tracks import TiciVideoStreamTrack
from iqpilot.system.webrtc.rtc.tracks import TiciVideoStreamTrack
from iqpilot.cereal import messaging
from iqpilot.common.params import Params
@@ -272,3 +272,40 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
def codec_preference(self) -> str | None:
return "H264"
class DebugVideoStreamTrack(TiciVideoStreamTrack):
def __init__(self, camera_type: str):
super().__init__(camera_type, 0.05)
self._codec = av.CodecContext.create("libx264", "w")
self._codec.width = 640
self._codec.height = 480
self._codec.pix_fmt = "yuv420p"
self._codec.time_base = self._time_base
self._codec.framerate = 20
self._codec.options = {"preset": "ultrafast", "tune": "zerolatency"}
self._codec.open()
self._pts = 0
self.timing_sei_enabled = False
async def recv(self):
await asyncio.sleep(self._dt)
frame = av.VideoFrame(self._codec.width, self._codec.height, "yuv420p")
frame.planes[0].update(bytes(frame.planes[0].buffer_size))
for plane in frame.planes[1:]:
plane.update(bytes([128]) * plane.buffer_size)
frame.pts = self._pts
self._pts += int(self._dt * self._clock_rate)
packets = self._codec.encode(frame)
if not packets:
return await self.recv()
packet = av.Packet(b"".join(bytes(encoded) for encoded in packets))
packet.pts = frame.pts
packet.dts = frame.pts
packet.time_base = self._time_base
packet.duration = int(self._dt * self._clock_rate)
return packet
def switch_camera(self, camera_type: str) -> None:
if camera_type not in LiveStreamVideoStreamTrack.livestream_camera_to_sock_mapping:
raise ValueError(f"Unknown camera {camera_type}")

View File

@@ -1,273 +0,0 @@
import asyncio
import logging
import struct
import time
import av
from teleoprtc.tracks import TiciVideoStreamTrack
from iqpilot.cereal import messaging
from iqpilot.common.params import Params
from iqpilot.common.realtime import DT_MDL, DT_DMON
# Arbitrary 16-byte UUID identifying konn3kt frame-timing SEI messages. When timing
# telemetry is enabled, each frame carries a user_data_unregistered SEI NAL with four
# big-endian doubles (ms): encode duration, IPC/queue delay, host transit, and the
# device wall clock. The client decodes these to compute true glass-to-glass latency.
TIMING_SEI_UUID = bytes([
0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e,
0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc,
])
# Annex-B start code + SEI NAL (type 6) + user_data_unregistered (type 5) + payload size
# (0x30 = 48 bytes = 16 UUID + 32 data). Trailing 0x80 is the RBSP stop bit.
_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID
class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
livestream_camera_to_sock_mapping = {
"driver": "livestreamDriverEncodeData",
"wideRoad": "livestreamWideRoadEncodeData",
"road": "livestreamRoadEncodeData",
}
main_camera_to_sock_mapping = {
"driver": "driverEncodeData",
"wideRoad": "wideRoadEncodeData",
"road": "roadEncodeData",
}
# Number of live tracks still waiting for their first keyframe. The on-demand
# keyframe request (LivestreamRequestKeyframe) is a single global param honored by
# every encoder, so with multiple concurrent tracks (dual-camera PiP) we must not
# clear it until *all* tracks have received an IDR — otherwise the first track to
# get its keyframe clears the request and starves the others (black feed).
_kf_pending_count = 0
def __init__(self, camera_type: str):
dt = DT_DMON if camera_type == "driver" else DT_MDL
super().__init__(camera_type, dt)
self._params = Params()
self._camera_type = camera_type
self._candidate_topics = [
self.main_camera_to_sock_mapping[camera_type],
self.livestream_camera_to_sock_mapping[camera_type],
]
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
self._active_topic = self._preferred_topics()[0]
self._pts = 0
self._t0_ns = time.monotonic_ns()
self._cached_header: bytes = b""
self._sent_keyframe = False
self._kf_requested = False # whether this track counts toward _kf_pending_count
self._frame_count = 0
self._last_frame_time = 0.0
self._last_preference_refresh = 0.0
# Tracks how long the H264 livestream feed has been silent, to gate the last-resort main-feed
# fallback (see recv) without flapping between sources frame-by-frame.
self._live_silent_since: float | None = None
# Opt-in glass-to-glass latency telemetry (toggled by the client over the data channel).
self.timing_sei_enabled = False
self._logger = logging.getLogger("LiveStreamVideoStreamTrack")
# Ask the encoder for an immediate IDR so the stream starts fast instead of waiting up to a full
# GOP for the next periodic keyframe (encoderd honors LivestreamRequestKeyframe per-frame).
self._mark_keyframe_needed()
def _request_keyframe(self, enabled: bool) -> None:
try:
self._params.put_bool("LivestreamRequestKeyframe", enabled)
except Exception:
self._logger.exception("failed to set LivestreamRequestKeyframe")
def _mark_keyframe_needed(self) -> None:
"""This track needs (another) keyframe: keep the global request asserted."""
if not self._kf_requested:
LiveStreamVideoStreamTrack._kf_pending_count += 1
self._kf_requested = True
self._request_keyframe(True)
def request_keyframe(self) -> None:
self._mark_keyframe_needed()
def _mark_keyframe_received(self) -> None:
"""This track got its keyframe; only clear the global request once no track needs one."""
if self._kf_requested:
self._kf_requested = False
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
if LiveStreamVideoStreamTrack._kf_pending_count == 0:
self._request_keyframe(False)
def stop(self):
# Release our pending-keyframe hold so a torn-down track that never received an
# IDR doesn't pin LivestreamRequestKeyframe True forever (continuous keyframes).
if getattr(self, "_kf_requested", False):
self._kf_requested = False
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
try:
super().stop()
except Exception:
pass
def switch_camera(self, camera_type: str) -> None:
"""Repoint this track at a different camera without renegotiating the peer connection.
Lets a single video track back the whole Live View — the client flips cameras over the
data channel and we swap the source here, instead of uplinking every camera at once."""
if camera_type not in self.livestream_camera_to_sock_mapping:
self._logger.warning("[%s] ignoring switch to unknown camera %s", self._id, camera_type)
return
if camera_type == self._camera_type:
return
self._logger.info("[%s] switching camera %s -> %s", self._id, self._camera_type, camera_type)
self._camera_type = camera_type
self._candidate_topics = [
self.main_camera_to_sock_mapping[camera_type],
self.livestream_camera_to_sock_mapping[camera_type],
]
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
self._active_topic = self._preferred_topics()[0]
# Force a fresh keyframe/header before emitting frames from the new source, and ask the encoder
# for an immediate IDR so the camera switch isn't stalled waiting for the next periodic keyframe.
self._cached_header = b""
self._sent_keyframe = False
self._last_preference_refresh = 0.0
self._live_silent_since = None
self._mark_keyframe_needed()
def _preferred_topics(self) -> list[str]:
# WebRTC currently forces H.264. The dedicated livestream topics are the H.264 feeds,
# while the main encode topics are the full-resolution HEVC recordings. Prefer the
# livestream feeds both onroad and offroad, and keep the main topics only as fallback.
return [
self.livestream_camera_to_sock_mapping[self._camera_type],
self.main_camera_to_sock_mapping[self._camera_type],
]
def _reset_decoder_state(self, topic: str) -> None:
if topic == self._active_topic:
return
self._logger.info("[%s] switching video source from %s to %s", self._id, self._active_topic, topic)
self._active_topic = topic
self._cached_header = b""
self._sent_keyframe = False
def _timing_sei(self, evta, log_mono_time: int) -> bytes:
"""Build a timing SEI NAL from encode metadata, or empty bytes when disabled."""
if not self.timing_sei_enabled:
return b""
idx = evta.idx
return _SEI_PREFIX + struct.pack(
'>4d',
(idx.timestampEof - idx.timestampSof) / 1e6, # encode duration (ms)
(log_mono_time - idx.timestampEof) / 1e6, # IPC/queue delay (ms)
(time.monotonic_ns() - log_mono_time) / 1e6, # host transit so far (ms)
time.time() * 1000, # device wall clock (ms) # noqa: TID251
) + b'\x80'
def _is_keyframe(self, data: bytes) -> bool:
"""Check if H.264 NAL unit contains an IDR keyframe (NAL type 5)."""
i = 0
while i < len(data) - 4:
# Look for Annex B start codes: 0x000001 or 0x00000001
if data[i:i+3] == b'\x00\x00\x01':
nal_type = data[i+3] & 0x1f
if nal_type == 5: # IDR slice
return True
i += 3
elif data[i:i+4] == b'\x00\x00\x00\x01':
nal_type = data[i+4] & 0x1f
if nal_type == 5: # IDR slice
return True
i += 4
else:
i += 1
return False
async def recv(self):
while True:
now = time.monotonic()
# Resolve topics each iteration: a camera switch (different async task) can rebuild self._socks
# across the await below, so a value cached before the loop would index a stale key (KeyError).
live_topic = self.livestream_camera_to_sock_mapping[self._camera_type]
main_topic = self.main_camera_to_sock_mapping[self._camera_type]
# Lock onto the dedicated H264 livestream feed. Onroad the HEVC main feed also publishes at
# 20fps; eagerly preferring whichever socket had a frame ready raced frame-by-frame, reset the
# decoder every frame, and (the track is negotiated H264) shoved HEVC garbage into the stream —
# the onroad choppiness. Only fall back to the main feed as a last resort after a long
# livestream silence (e.g. stream_encoderd still spinning up), and snap back when it returns.
msg = messaging.recv_one_or_none(self._socks[live_topic])
if msg is not None:
self._reset_decoder_state(live_topic)
self._last_frame_time = now
self._live_silent_since = None
break
if self._live_silent_since is None:
self._live_silent_since = now
elif now - self._live_silent_since > 3.0:
maybe_msg = messaging.recv_one_or_none(self._socks[main_topic])
if maybe_msg is not None:
self._reset_decoder_state(main_topic)
self._last_frame_time = now
msg = maybe_msg
break
await asyncio.sleep(0.005)
evta = getattr(msg, msg.which())
header = bytes(evta.header)
data = bytes(evta.data)
self._frame_count += 1
# Cache SPS/PPS header when it arrives
if header:
self._cached_header = header
self._logger.debug(f"[{self._id}] cached SPS/PPS header ({len(header)} bytes)")
# CRITICAL: Cannot decode without SPS/PPS. Wait for it.
if not self._cached_header:
self._logger.debug(f"[{self._id}] frame {self._frame_count}: no SPS/PPS yet, skipping")
return await self.recv()
is_keyframe = self._is_keyframe(data)
# Wait for first keyframe before sending any frames
# Browser decoder needs IDR to initialize properly
if not self._sent_keyframe:
if not is_keyframe:
self._logger.debug(f"[{self._id}] frame {self._frame_count}: waiting for keyframe")
return await self.recv()
self._sent_keyframe = True
# Got the IDR we asked for — stop nagging the encoder, but only once every
# concurrent track has its keyframe (multi-track PiP shares the global param).
self._mark_keyframe_received()
self._logger.info(f"[{self._id}] first keyframe received, starting stream")
# Optional timing SEI NAL, inserted before the slice data (and after SPS/PPS on keyframes).
sei_nal = self._timing_sei(evta, msg.logMonoTime)
# Prepend SPS/PPS header to keyframes (required by some decoders)
# For non-keyframes, header is optional but safe to include
if is_keyframe:
payload = self._cached_header + sei_nal + data
else:
payload = sei_nal + data
self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
packet = av.Packet(payload)
packet.time_base = self._time_base
packet.pts = int(self._pts)
packet.dts = int(self._pts)
packet.duration = int(self._dt * self._clock_rate)
if is_keyframe:
packet.is_keyframe = True
self.log_debug("track sending frame %s (keyframe=%s, size=%d)", self._pts, is_keyframe, len(payload))
return packet
def codec_preference(self) -> str | None:
return "H264"

View File

@@ -1,4 +1,4 @@
# ruff: noqa: TID251, UP006, UP035
# ruff: noqa: UP006, UP035
import abc
from typing import Dict, List, Optional

View File

@@ -1,4 +1,4 @@
# ruff: noqa: TID251, UP006, UP035
# ruff: noqa: UP006, UP035
import abc
import asyncio
@@ -14,6 +14,7 @@ from libdatachannel import (
Candidate,
Description,
FrameInfo,
H264RtpDepacketizer,
H264RtpPacketizer,
IceServer,
NalUnit,
@@ -100,7 +101,6 @@ class WebRTCBaseStream(abc.ABC):
@staticmethod
def _make_ice_servers(servers: Optional[List[dict]]) -> List[IceServer]:
"""Preserve Konn3kt's authenticated STUN/TURN configuration in libdatachannel."""
parsed: List[IceServer] = []
for server in servers or []:
urls = server.get("urls", []) if isinstance(server, dict) else []
@@ -119,7 +119,9 @@ class WebRTCBaseStream(abc.ABC):
except Exception:
logging.getLogger("WebRTCStream").warning("Ignoring invalid ICE server %r", url, exc_info=True)
# A supplied list is authoritative, including LAN-only sessions where relay is deliberately filtered.
return parsed or [IceServer("stun:stun.l.google.com:19302")]
if servers is not None:
return parsed
return [IceServer("stun:stun.l.google.com:19302")]
def _log_debug(self, msg: Any, *args):
self.logger.debug(f"{type(self)}() {msg}", *args)
@@ -144,13 +146,14 @@ class WebRTCBaseStream(abc.ABC):
media = Description.Video(camera_type, Description.Direction.RecvOnly)
media.add_h264_codec(96)
track = self.peer_connection.add_track(media)
track.set_media_handler(OpusRtpDepacketizer())
track.set_media_handler(H264RtpDepacketizer())
self._consumer_tracks.append(track)
self.incoming_camera_tracks[camera_type] = track
if self.expected_incoming_audio:
media = Description.Audio("audio", Description.Direction.RecvOnly)
media.add_opus_codec(111)
track = self.peer_connection.add_track(media)
track.set_media_handler(OpusRtpDepacketizer())
self._consumer_tracks.append(track)
self.incoming_audio_tracks.append(track)
@@ -170,7 +173,7 @@ class WebRTCBaseStream(abc.ABC):
def _make_video_media(self, track: TiciVideoStreamTrack, remote_sdp: str) -> Tuple[Description.Video, int, int, str]:
mid, payload_type = self._find_offer_video(remote_sdp)
ssrc = random.randint(1, 0xFFFFFFFF)
cname = f"teleoprtc-{random.getrandbits(32):08x}"
cname = f"iqpilot-video-{random.getrandbits(32):08x}"
stream_id = f"stream-{random.getrandbits(32):08x}"
media = Description.Video(mid, Description.Direction.SendOnly)
media.add_h264_codec(payload_type)
@@ -193,7 +196,7 @@ class WebRTCBaseStream(abc.ABC):
def _make_audio_media(self, remote_sdp: str) -> Tuple[Description.Audio, int, int, str]:
mid, payload_type = self._find_offer_audio(remote_sdp)
ssrc = random.randint(1, 0xFFFFFFFF)
cname = f"teleoprtc-audio-{random.getrandbits(32):08x}"
cname = f"iqpilot-audio-{random.getrandbits(32):08x}"
direction = Description.Direction.SendRecv if self.expected_incoming_audio else Description.Direction.SendOnly
media = Description.Audio(mid, direction)
media.add_opus_codec(payload_type)
@@ -352,7 +355,6 @@ class WebRTCBaseStream(abc.ABC):
self.incoming_message_handlers.append(message_handler)
def add_ice_candidate(self, candidate: dict) -> None:
"""Accept post-offer browser candidates for Konn3kt's existing trickle endpoint."""
candidate_sdp = str(candidate.get("candidate") or "") if isinstance(candidate, dict) else ""
if not candidate_sdp:
return

View File

@@ -32,6 +32,7 @@ class TiciVideoStreamTrack:
def __init__(self, camera_type: str, dt: float, time_base: fractions.Fraction = VIDEO_TIME_BASE, clock_rate: int = VIDEO_CLOCK_RATE):
assert camera_type in ["driver", "wideRoad", "road"]
self._id: str = video_track_id(camera_type, str(uuid.uuid4()))
self._dt = dt
self._time_base: fractions.Fraction = time_base
self._clock_rate: int = clock_rate
self._logger = logging.getLogger("WebRTCStream")

View File

@@ -14,7 +14,6 @@ from iqpilot.system.webrtc.webrtcd import CerealIncomingMessageProxy, CerealOutg
def _default_route_ip() -> str | None:
"""Use the interface the kernel will actually use for Internet/relay media."""
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
sock.connect(("8.8.8.8", 53))
@@ -25,9 +24,7 @@ def _default_route_ip() -> str | None:
sock.close()
class LibdatachannelBitrateController:
"""Loss-driven bitrate control using native RTCP receiver reports."""
class LivestreamBitrateController:
bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[2]}
sample_interval = 0.2
@@ -103,45 +100,45 @@ class LibdatachannelBitrateController:
self._publish(self.bitrates[self._level])
class StreamSessionLibdatachannel:
"""IQ.Pilot's libdatachannel stream path. It preserves Konn3kt signalling and controls."""
shared_pub_master = DynamicPubMaster([])
class StreamSession:
shared_pub_master: DynamicPubMaster | None = None
def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
from iqpilot.system.webrtc.device.video_ldc import LiveStreamVideoStreamTrack
from iqpilot.system.webrtc.teleoprtc_ldc.builder import WebRTCAnswerBuilder
from iqpilot.system.webrtc.teleoprtc_ldc.info import parse_info_from_offer
from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
from iqpilot.system.webrtc.rtc.builder import WebRTCAnswerBuilder
from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
config = parse_info_from_offer(sdp)
if len(cameras) != config.n_expected_camera_tracks:
raise ValueError("Incoming stream has misconfigured number of video tracks")
if debug_mode:
raise ValueError("libdatachannel debug tracks are not supported")
builder = WebRTCAnswerBuilder(sdp, bind_address=_default_route_ip(), ice_servers=ice_servers or [])
self.video_tracks = [LiveStreamVideoStreamTrack(camera) for camera in cameras]
video_track_type = DebugVideoStreamTrack if debug_mode else LiveStreamVideoStreamTrack
self.video_tracks = [video_track_type(camera) for camera in cameras]
for camera, track in zip(cameras, self.video_tracks, strict=True):
builder.add_video_stream(camera, track)
# The browser uses a single sendrecv audio m-line. libdatachannel's Python binding
# currently cannot negotiate that bidirectional track reliably, so this experimental
# transport deliberately remains video/control-only. The default aiortc path keeps
# both audio directions until the native duplex path passes the same integration test.
self.audio_output = None
audio_track_type = DebugAudioOpusProducer if debug_mode else AudioInputOpusProducer
self.audio_output = audio_track_type() if config.expected_audio_track else None
if self.audio_output is not None:
builder.add_audio_stream(self.audio_output)
if config.incoming_audio_track:
builder.offer_to_receive_audio_stream()
self.stream = builder.stream()
self.identifier = str(uuid.uuid4())
self.incoming_bridge_services = incoming_services
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if incoming_services else None
if incoming_services and self.shared_pub_master is None:
StreamSession.shared_pub_master = DynamicPubMaster([])
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if self.shared_pub_master is not None and incoming_services else None
self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services)) if outgoing_services else None
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge) if self.outgoing_bridge is not None else None
self.ui_stream_requested = ui_stream
self.ui_stream_runner: CerealProxyRunner | None = None
self.audio_input_proxy = None
self.audio_recv_requested = False
self.bitrate_controller = LibdatachannelBitrateController(self.stream.get_receiver_report_stats)
self.audio_recv_requested = config.incoming_audio_track
self.bitrate_controller = LivestreamBitrateController(self.stream.get_receiver_report_stats)
self.run_task: asyncio.Task | None = None
self._cleanup_lock = asyncio.Lock()
self._cleanup_done = False
@@ -216,7 +213,7 @@ class StreamSessionLibdatachannel:
if self.ui_stream_requested:
self.set_ui_stream(True)
if self.audio_recv_requested and self.stream.has_incoming_audio_track():
from iqpilot.system.webrtc.device.audio_ldc import IncomingOpusCerealProxy
from iqpilot.system.webrtc.device.native_audio import IncomingOpusCerealProxy
self.audio_input_proxy = IncomingOpusCerealProxy(self.stream.get_incoming_audio_track())
self.audio_input_proxy.start()
self.bitrate_controller.start()

View File

@@ -0,0 +1,96 @@
from types import SimpleNamespace
import pytest
from iqpilot.system.webrtc import session as session_module
class FakeStream:
def get_receiver_report_stats(self):
return {}
class FakeBuilder:
instance = None
def __init__(self, sdp, bind_address=None, ice_servers=None):
self.sdp = sdp
self.bind_address = bind_address
self.ice_servers = ice_servers
self.video = []
self.audio = []
self.receive_audio = False
self.result = FakeStream()
FakeBuilder.instance = self
def add_video_stream(self, camera, track):
self.video.append((camera, track))
def add_audio_stream(self, track):
self.audio.append(track)
def offer_to_receive_audio_stream(self):
self.receive_audio = True
def stream(self):
return self.result
class FakeVideoTrack:
def __init__(self, camera):
self.camera = camera
self.timing_sei_enabled = False
self.switched = None
def switch_camera(self, camera):
self.switched = camera
class FakeAudioProducer:
def __init__(self):
self.enabled = True
def enable(self, enabled):
self.enabled = enabled
@pytest.fixture
def native_session(mocker):
config = SimpleNamespace(
n_expected_camera_tracks=1,
expected_audio_track=True,
incoming_audio_track=True,
incoming_datachannel=True,
)
mocker.patch("iqpilot.system.webrtc.rtc.info.parse_info_from_offer", return_value=config)
mocker.patch("iqpilot.system.webrtc.rtc.builder.WebRTCAnswerBuilder", FakeBuilder)
mocker.patch("iqpilot.system.webrtc.device.native_video.LiveStreamVideoStreamTrack", FakeVideoTrack)
mocker.patch("iqpilot.system.webrtc.device.native_audio.AudioInputOpusProducer", FakeAudioProducer)
mocker.patch.object(session_module, "_default_route_ip", return_value="192.0.2.1")
mocker.patch.object(session_module, "Params", return_value=mocker.Mock())
return session_module.StreamSession(
"offer", ["road"], [], [], [{"urls": "turn:example.com"}], ui_stream=False,
)
def test_native_session_builds_duplex_audio(native_session):
builder = FakeBuilder.instance
assert builder is not None
assert builder.bind_address == "192.0.2.1"
assert builder.ice_servers == [{"urls": "turn:example.com"}]
assert [camera for camera, _ in builder.video] == ["road"]
assert builder.audio == [native_session.audio_output]
assert builder.receive_audio
assert native_session.audio_recv_requested
def test_native_session_controls(native_session, mocker):
native_session.bitrate_controller = mocker.Mock()
native_session.message_handler('{"type":"timingSei","enabled":true}')
assert native_session.video_tracks[0].timing_sei_enabled
native_session.message_handler('{"type":"switchCamera","camera":"driver"}')
assert native_session.video_tracks[0].switched == "driver"
native_session.message_handler('{"type":"setAudioEnabled","enabled":false}')
assert not native_session.audio_output.enabled
native_session.message_handler('{"type":"setQuality","quality":"low"}')
native_session.bitrate_controller.set_quality.assert_called_once_with("low")

View File

@@ -0,0 +1,116 @@
import pytest
from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
from iqpilot.system.webrtc.rtc.stream import WebRTCBaseStream
from iqpilot.system.webrtc.rtc.tracks import TiciTrackWrapper, TiciVideoStreamTrack, parse_video_track_id, video_track_id
def sdp_with_media(media):
mids = " ".join(str(i) for i in range(len(media)))
sections = []
for index, (kind, direction) in enumerate(media):
if kind == "video":
payload, mapping = "96", "H264/90000"
protocol = "UDP/TLS/RTP/SAVPF"
elif kind == "audio":
payload, mapping = "111", "opus/48000/2"
protocol = "UDP/TLS/RTP/SAVPF"
else:
sections.extend([
"m=application 9 UDP/DTLS/SCTP webrtc-datachannel",
"c=IN IP4 0.0.0.0",
f"a=mid:{index}",
"a=sctp-port:5000",
])
continue
sections.extend([
f"m={kind} 9 {protocol} {payload}",
"c=IN IP4 0.0.0.0",
f"a=mid:{index}",
f"a={direction}",
f"a=rtpmap:{payload} {mapping}",
"a=rtcp-mux",
])
lines = [
"v=0",
"o=- 1 1 IN IP4 0.0.0.0",
"s=-",
"t=0 0",
f"a=group:BUNDLE {mids}",
*sections,
]
return "\r\n".join(lines) + "\r\n"
@pytest.mark.parametrize("direction,expected_outgoing,expected_incoming", [
("recvonly", True, False),
("sendonly", False, True),
("sendrecv", True, True),
("inactive", False, False),
])
def test_audio_directions(direction, expected_outgoing, expected_incoming):
info = parse_info_from_offer(sdp_with_media([("audio", direction)]))
assert info.expected_audio_track == expected_outgoing
assert info.incoming_audio_track == expected_incoming
def test_video_and_data_channel_metadata():
info = parse_info_from_offer(sdp_with_media([
("video", "recvonly"),
("video", "sendrecv"),
("application", "sendrecv"),
]))
assert info.n_expected_camera_tracks == 2
assert info.incoming_datachannel
def test_explicit_empty_ice_servers_disable_defaults():
assert WebRTCBaseStream._make_ice_servers([]) == []
def test_default_ice_server():
servers = WebRTCBaseStream._make_ice_servers(None)
assert len(servers) == 1
assert servers[0].hostname == "stun.l.google.com"
assert servers[0].port == 19302
def test_authenticated_ice_servers():
servers = WebRTCBaseStream._make_ice_servers([{
"urls": ["turn:relay.example.com:3478", "stun:stun.example.com:3478"],
"username": "user",
"credential": "secret",
}])
assert [(server.hostname, server.port) for server in servers] == [
("relay.example.com", 3478),
("stun.example.com", 3478),
]
assert all(server.username == "user" and server.password == "secret" for server in servers)
def test_track_id_roundtrip():
assert parse_video_track_id(video_track_id("driver", "track")) == ("driver", "track")
def test_invalid_track_id():
with pytest.raises(ValueError):
parse_video_track_id("driver")
def test_track_wrapper_preserves_camera():
class Track:
kind = "video"
id = "source"
async def recv(self):
return b"frame"
wrapper = TiciTrackWrapper("road", Track())
assert parse_video_track_id(wrapper.id)[0] == "road"
wrapper.stop()
assert wrapper.readyState == "ended"
def test_track_stores_frame_period():
track = TiciVideoStreamTrack("wideRoad", 0.05)
assert track._dt == 0.05

View File

@@ -0,0 +1,79 @@
import asyncio
import os
import pytest
from iqpilot.system.webrtc.rtc import WebRTCOfferBuilder
from iqpilot.system.webrtc.rtc.stream import RTCSessionDescription
from iqpilot.system.webrtc.session import StreamSession
@pytest.mark.asyncio
async def test_native_video_audio_and_data_channel():
if not os.environ.get("CI"):
return
answer_session = None
video_received = asyncio.Event()
audio_received = asyncio.Event()
async def connect(offer):
nonlocal answer_session
answer_session = StreamSession(offer.sdp, offer.video, [], [], [], debug_mode=True)
answer = await answer_session.get_answer()
answer_session.start()
return RTCSessionDescription(answer.sdp, answer.type)
builder = WebRTCOfferBuilder(connect, ice_servers=[])
builder.offer_to_receive_video_stream("road")
builder.offer_to_receive_audio_stream()
builder.add_messaging()
stream = builder.stream()
try:
await asyncio.wait_for(stream.start(), 10)
stream.get_incoming_video_track("road").on_frame(lambda *_: video_received.set())
stream.get_incoming_audio_track().on_frame(lambda *_: audio_received.set())
await asyncio.wait_for(stream.wait_for_connection(), 10)
await asyncio.wait_for(video_received.wait(), 10)
await asyncio.wait_for(audio_received.wait(), 10)
stream.get_messaging_channel().send('{"type":"timingSei","enabled":true}')
await asyncio.sleep(0.1)
assert answer_session is not None
assert answer_session.video_tracks[0].timing_sei_enabled
finally:
await stream.stop()
if answer_session is not None:
await answer_session.stop_async()
@pytest.mark.asyncio
async def test_native_duplex_audio_negotiation():
if not os.environ.get("CI"):
return
answer_session = None
async def connect(offer):
nonlocal answer_session
sendrecv_offer = offer.sdp.replace("a=recvonly", "a=sendrecv", 1)
answer_session = StreamSession(sendrecv_offer, offer.video, [], [], [], debug_mode=True)
answer = await answer_session.get_answer()
answer_session.start()
return RTCSessionDescription(answer.sdp, answer.type)
builder = WebRTCOfferBuilder(connect, ice_servers=[])
builder.offer_to_receive_audio_stream()
stream = builder.stream()
try:
await asyncio.wait_for(stream.start(), 10)
await asyncio.wait_for(stream.wait_for_connection(), 10)
assert stream.has_incoming_audio_track()
assert answer_session is not None
assert answer_session.audio_recv_requested
assert answer_session.audio_output is not None
finally:
await stream.stop()
if answer_session is not None:
await answer_session.stop_async()

View File

@@ -1,19 +1,14 @@
import asyncio
import gc
import json
import time
# for aiortc and its dependencies
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
from aiortc import RTCDataChannel
from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
import capnp
from iqpilot.cereal import messaging, log
from iqpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
from iqpilot.system.webrtc.rtc.tracks import VIDEO_TIME_BASE
class TestStreamSession:
@@ -23,6 +18,7 @@ class TestStreamSession:
def teardown_method(self):
self.loop.stop()
self.loop.close()
gc.collect()
def test_outgoing_proxy(self, mocker):
test_msg = log.Event.new_message()
@@ -32,12 +28,12 @@ class TestStreamSession:
expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
expected_json = json.dumps(expected_dict).encode()
channel = mocker.Mock(spec=RTCDataChannel)
mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
def mocked_update(t):
mocked_submaster.update_msgs(0, [test_msg])
mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
channel = mocker.Mock()
mocked_submaster = mocker.MagicMock()
mocked_submaster.updated = {"customReservedRawData0": True}
mocked_submaster.logMonoTime = {"customReservedRawData0": 123}
mocked_submaster.valid = {"customReservedRawData0": True}
mocked_submaster.__getitem__.return_value = test_msg.customReservedRawData0
proxy = CerealOutgoingMessageProxy(mocked_submaster)
proxy.add_channel(channel)
@@ -47,9 +43,9 @@ class TestStreamSession:
def test_incoming_proxy(self, mocker):
tested_msgs = [
{"type": "customReservedRawData0", "data": "test"}, # primitive
{"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
{"type": "customReservedRawData0", "data": "test"},
{"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]},
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}},
]
mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
@@ -72,20 +68,15 @@ class TestStreamSession:
fake_msg.livestreamDriverEncodeData.header = b"header"
fake_msg.livestreamDriverEncodeData.data = b"\x00\x00\x00\x01\x65"
mocker.patch("iqpilot.system.webrtc.device.video.messaging.recv_one_or_none", return_value=fake_msg)
mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.sub_sock", return_value=mocker.Mock())
mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.recv_one_or_none", return_value=fake_msg)
track = LiveStreamVideoStreamTrack("driver")
assert track.id.startswith("driver")
assert track.codec_preference() == "H264"
for i in range(5):
packet = self.loop.run_until_complete(track.recv())
assert packet.time_base == VIDEO_TIME_BASE
if i == 0:
start_ns = time.monotonic_ns()
start_pts = packet.pts
assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
assert packet.size == len(b"header\x00\x00\x00\x01\x65")
packet = self.loop.run_until_complete(track.recv())
assert packet.time_base == VIDEO_TIME_BASE
assert packet.pts is not None
assert packet.size == len(b"header\x00\x00\x00\x01\x65")
def test_input_audio_track(self, mocker):
packet_time, rate = 0.02, 16000
@@ -93,11 +84,27 @@ class TestStreamSession:
fake_msg = messaging.new_message("rawAudioData")
fake_msg.rawAudioData.data = b"\x00" * 2 * sample_count
fake_msg.rawAudioData.sampleRate = rate
mocker.patch("iqpilot.system.webrtc.device.audio.messaging.recv_one_or_none", return_value=fake_msg)
track = AudioInputStreamTrack(rate=rate)
mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.sub_sock", return_value=mocker.Mock())
track = AudioInputOpusProducer()
track._source_rate = rate
mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.recv_one_or_none", return_value=fake_msg)
for i in range(5):
frame = self.loop.run_until_complete(track.recv())
assert frame.rate == rate
assert frame.samples == sample_count
assert frame.pts == i * sample_count
packet = self.loop.run_until_complete(track.recv())
assert packet is not None
payload, pts = packet
assert payload
assert pts >= 0
def test_debug_video_track(self):
track = DebugVideoStreamTrack("road")
packet = self.loop.run_until_complete(track.recv())
assert packet.size > 0
assert packet.pts == 0
def test_debug_audio_track(self):
track = DebugAudioOpusProducer()
packet = self.loop.run_until_complete(track.recv())
assert packet is not None
payload, pts = packet
assert payload
assert pts == 0

View File

@@ -1,6 +1,9 @@
import asyncio
import json
import math
import pytest
from iqpilot.cereal import log, messaging
from iqpilot.system.webrtc.ui_stream import (
UI_STREAM_SERVICES,
@@ -270,11 +273,13 @@ class TestUIStreamFrame:
class TestSessionWiring:
def test_set_ui_stream_control_message(self):
import asyncio
@pytest.mark.asyncio
async def test_set_ui_stream_control_message(self, mocker):
import logging
from types import SimpleNamespace
from iqpilot.system.webrtc.webrtcd import StreamSession
from iqpilot.system.webrtc.session import StreamSession
mocker.patch("iqpilot.system.webrtc.ui_stream.messaging.SubMaster", return_value=FakeSubMaster(make_readers()))
session = StreamSession.__new__(StreamSession)
session.logger = logging.getLogger("webrtcd")
@@ -287,14 +292,11 @@ class TestSessionWiring:
get_messaging_channel=lambda: channel,
)
async def go():
await session.message_handler(b'{"type":"setUiStream","enabled":true}')
assert session.ui_stream_runner is not None
await asyncio.sleep(0.05)
await session.message_handler(b'{"type":"setUiStream","enabled":false}')
assert session.ui_stream_runner is None
asyncio.run(go())
session.message_handler(b'{"type":"setUiStream","enabled":true}')
assert session.ui_stream_runner is not None
await asyncio.sleep(0.05)
session.message_handler(b'{"type":"setUiStream","enabled":false}')
assert session.ui_stream_runner is None
assert len(channel.sent) >= 1
frame = json.loads(channel.sent[0])

View File

@@ -1,16 +1,33 @@
import pytest
import asyncio
import json
# for aiortc and its dependencies
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
from types import SimpleNamespace
from iqpilot.system.webrtc.webrtcd import get_stream
import aiortc
from teleoprtc import WebRTCOfferBuilder
from parameterized import parameterized_class
import pytest
from iqpilot.system.webrtc.webrtcd import add_ice, get_stream
class FakeSession:
instances = []
def __init__(self, identifier="session"):
self.identifier = identifier
self.started = False
self.stopped = False
self.candidates = []
self.instances.append(self)
async def get_answer(self):
return SimpleNamespace(sdp="answer", type="answer")
def start(self):
self.started = True
async def stop_async(self):
self.stopped = True
async def add_ice_candidate(self, candidate):
self.candidates.append(candidate)
@parameterized_class(("in_services", "out_services"), [
@@ -21,44 +38,38 @@ from parameterized import parameterized_class
])
@pytest.mark.asyncio
class TestWebrtcdProc:
async def assertCompletesWithTimeout(self, awaitable, timeout=10):
try:
async with asyncio.timeout(timeout):
await awaitable
except TimeoutError:
pytest.fail("Timeout while waiting for awaitable to complete")
async def test_webrtcd(self, mocker):
mock_request = mocker.MagicMock()
async def connect(offer):
body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
response = await get_stream(mock_request)
response_json = json.loads(response.text)
return aiortc.RTCSessionDescription(**response_json)
session = FakeSession()
mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
request = mocker.MagicMock()
request.app = {"streams": {}, "debug": False}
request.json = mocker.AsyncMock(return_value={
"sdp": "offer",
"cameras": ["road"],
"bridge_services_in": self.in_services,
"bridge_services_out": self.out_services,
})
builder = WebRTCOfferBuilder(connect, ice_servers=[])
builder.offer_to_receive_video_stream("road")
builder.offer_to_receive_audio_stream()
if len(self.in_services) > 0 or len(self.out_services) > 0:
builder.add_messaging()
response = await get_stream(request)
stream = builder.stream()
assert response.status == 200
assert json.loads(response.text) == {"sdp": "answer", "type": "answer"}
assert request.app["streams"] == {session.identifier: session}
assert session.started
await self.assertCompletesWithTimeout(stream.start())
await self.assertCompletesWithTimeout(stream.wait_for_connection())
async def test_replaces_session_and_routes_ice(self, mocker):
previous = FakeSession("previous")
session = FakeSession("current")
mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
request = mocker.MagicMock()
request.app = {"streams": {previous.identifier: previous}, "debug": False}
request.json = mocker.AsyncMock(return_value={"sdp": "offer", "cameras": ["road"]})
assert stream.has_incoming_video_track("road")
assert stream.has_incoming_audio_track()
assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
response = await get_stream(request)
video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
await self.assertCompletesWithTimeout(video_track.recv())
await self.assertCompletesWithTimeout(audio_track.recv())
await self.assertCompletesWithTimeout(stream.stop())
# cleanup, very implementation specific, test may break if it changes
assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
_, session = mock_request.app["streams"].__setitem__.call_args.args
await self.assertCompletesWithTimeout(session.post_run_cleanup())
assert response.status == 200
assert previous.stopped
request.json = mocker.AsyncMock(return_value={"candidate": {"candidate": "candidate:1"}})
ice_response = await add_ice(request)
assert ice_response.status == 200
assert session.candidates == [{"candidate": "candidate:1"}]

View File

@@ -3,52 +3,27 @@
import argparse
import asyncio
import json
import os
import uuid
import logging
from dataclasses import dataclass, field
from typing import Any, TYPE_CHECKING
from iqpilot.common.params import Params
from typing import Any
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)
import capnp
import aiortc.rtcrtpsender
from aiohttp import web
from aiortc.rtp import RTCP_PSFB_APP, RtcpPsfbPacket, unpack_remb_fci
if TYPE_CHECKING:
from aiortc.rtcdatachannel import RTCDataChannel
from iqpilot.system.webrtc.schema import generate_field
from iqpilot.cereal import messaging, log
_handle_rtcp_packet = aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet
async def _handle_rtcp_packet_with_remb(self, packet):
if isinstance(packet, RtcpPsfbPacket) and packet.fmt == RTCP_PSFB_APP:
try:
bitrate, ssrcs = unpack_remb_fci(packet.fci)
if getattr(self, "_ssrc", None) in ssrcs:
self._remb_bitrate = bitrate
except ValueError:
pass
return await _handle_rtcp_packet(self, packet)
aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet = _handle_rtcp_packet_with_remb
class CerealOutgoingMessageProxy:
def __init__(self, sm: messaging.SubMaster):
self.sm = sm
self.channels: list[RTCDataChannel] = []
self.channels: list[Any] = []
def add_channel(self, channel: 'RTCDataChannel'):
def add_channel(self, channel: Any):
self.channels.append(channel)
def to_json(self, msg_content: Any):
@@ -114,43 +89,6 @@ class AsyncTaskRunner:
self.task = None
class IncomingAudioCerealProxy(AsyncTaskRunner):
def __init__(self, track: Any):
super().__init__()
from av.audio.resampler import AudioResampler
from iqpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
from iqpilot.system.webrtc.device.audio import WEBRTC_AUDIO_SERVICE
self.track = track
self.service = WEBRTC_AUDIO_SERVICE
self.pm = messaging.PubMaster([self.service])
self.resampler = AudioResampler(format="s16", layout="mono", rate=SOUND_SAMPLE_RATE)
def _publish(self, frame: Any) -> None:
data = frame.to_ndarray().tobytes()
if not data:
return
msg = messaging.new_message(self.service, valid=True)
msg.webrtcAudioData.data = data
msg.webrtcAudioData.sampleRate = frame.sample_rate
self.pm.send(self.service, msg)
async def run(self):
from aiortc.mediastreams import MediaStreamError
while True:
try:
frame = await self.track.recv()
for resampled_frame in self.resampler.resample(frame):
self._publish(resampled_frame)
except MediaStreamError:
break
except Exception:
self.logger.exception("Incoming audio cereal proxy failure")
await asyncio.sleep(0.1)
class CerealProxyRunner:
def __init__(self, proxy: CerealOutgoingMessageProxy):
self.proxy = proxy
@@ -169,110 +107,14 @@ class CerealProxyRunner:
self.task = None
async def run(self):
from aiortc.exceptions import InvalidStateError
while True:
try:
self.proxy.update()
except InvalidStateError:
self.logger.warning("Cereal outgoing proxy invalid state (connection closed)")
break
except Exception:
self.logger.exception("Cereal outgoing proxy failure")
await asyncio.sleep(0.01)
class LivestreamBitrateController:
"""Adaptive bitrate for the livestream encoder using browser REMB feedback."""
# Match comma's rung choices more closely. A steadier capped stream tends to look better than
# an occasionally-higher bitrate stream that induces queueing, jitter, and frame pacing swings.
bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[-1]}
sample_interval = 1.0
lower_factor = 0.9
probe_after = 10
settle_samples = 3
def __init__(self, peer_connection: Any):
self.pc = peer_connection
self.params = Params()
self.task: asyncio.Task | None = None
# Start conservative and probe UP only when REMB proves headroom. Previously this started at
# the top rung (5 Mbps); with no REMB feedback (e.g. transport-cc-only receivers, or a flaky
# uplink that never delivers RTCP), _bandwidth_estimate() returns None and run() hits
# `if estimate is None: continue` — so the level never moves and the encoder stays pinned at
# 5 Mbps, flooding a marginal cellular uplink until webrtcd's send buffer balloons and trips the
# device's lowMemory soft-disable. The med rung is carriable on typical cellular; healthy links
# with working REMB still probe up to high within ~probe_after seconds.
self.level = min(1, len(self.bitrates) - 1)
self.stable = 0
self.settle = 0
self._auto = True
self.current_bitrate = self.bitrates[self.level]
self._publish(self.bitrates[self.level])
def start(self):
if self.task is None:
self.task = asyncio.create_task(self.run())
def stop(self):
if self.task is not None and not self.task.done():
self.task.cancel()
self.task = None
async def run(self):
while True:
await asyncio.sleep(self.sample_interval)
if not self._auto:
continue
estimate = self._bandwidth_estimate()
if estimate is None:
continue
if self.settle > 0:
self.settle -= 1
continue
if estimate < self.bitrates[self.level] * self.lower_factor:
while self.level > 0 and estimate < self.bitrates[self.level] * self.lower_factor:
self.level -= 1
self.stable = 0
self._publish(self.bitrates[self.level])
elif self.level < len(self.bitrates) - 1:
self.stable += 1
if self.stable >= self.probe_after:
self.level += 1
self.stable = 0
self.settle = self.settle_samples
self._publish(self.bitrates[self.level])
else:
self.stable = 0
def _bandwidth_estimate(self) -> int | None:
estimate = None
for sender in self.pc.getSenders():
bitrate = getattr(sender, "_remb_bitrate", None)
if bitrate is not None:
estimate = bitrate if estimate is None else min(estimate, bitrate)
return estimate
def set_quality(self, quality: str):
if quality in self.label_to_bitrate:
self._auto = False
self._publish(self.label_to_bitrate[quality])
elif quality == "auto":
self._auto = True
def _publish(self, bitrate: int):
# Param is registered as INT — must pass a Python int, not str. Passing str throws
# TypeError in Params.put (type mismatch) and crashes StreamSession.__init__ → HTTP 500.
self.current_bitrate = int(bitrate)
self.params.put("LivestreamEncoderBitrate", int(bitrate))
class DynamicPubMaster(messaging.PubMaster):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -285,239 +127,11 @@ class DynamicPubMaster(messaging.PubMaster):
self.sock[service] = messaging.pub_sock(service)
class StreamSession:
shared_pub_master = DynamicPubMaster([])
def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
from aiortc.mediastreams import VideoStreamTrack, AudioStreamTrack
from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
from teleoprtc import WebRTCAnswerBuilder
from teleoprtc.info import parse_info_from_offer
config = parse_info_from_offer(sdp)
builder = WebRTCAnswerBuilder(sdp, ice_servers=ice_servers or [])
assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
self.video_tracks: list[LiveStreamVideoStreamTrack] = []
for cam in cameras:
track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()
if isinstance(track, LiveStreamVideoStreamTrack):
self.video_tracks.append(track)
builder.add_video_stream(cam, track)
# Audio init may fail if openpilot is using the audio subsystem - skip gracefully
if config.expected_audio_track:
try:
self.audio_input_track = AudioInputStreamTrack() if not debug_mode else AudioStreamTrack()
builder.add_audio_stream(self.audio_input_track)
self.audio_send_enabled = True
except Exception as e:
logging.warning(f"Could not init audio input (audio in use?): {e}")
if config.incoming_audio_track:
builder.offer_to_receive_audio_stream()
self.stream = builder.stream()
self.identifier = str(uuid.uuid4())
self.incoming_bridge: CerealIncomingMessageProxy | None = None
self.incoming_bridge_services = incoming_services
self.outgoing_bridge: CerealOutgoingMessageProxy | None = None
self.outgoing_bridge_runner: CerealProxyRunner | None = None
if len(incoming_services) > 0:
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master)
if len(outgoing_services) > 0:
self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services))
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)
self.ui_stream_requested = ui_stream
self.ui_stream_runner: CerealProxyRunner | None = None
self.incoming_audio_proxy: IncomingAudioCerealProxy | None = None
self.audio_input_track: AudioInputStreamTrack | AudioStreamTrack | None = None
self.audio_send_enabled = False
self.audio_recv_requested = bool(config.incoming_audio_track)
self.audio_send_requested = bool(config.expected_audio_track)
self.run_task: asyncio.Task | None = None
# Adaptive bitrate controller for the livestream encoder (no-op in debug mode).
self.bitrate_controller: LivestreamBitrateController | None = None
if not debug_mode and len(self.video_tracks) > 0:
self.bitrate_controller = LivestreamBitrateController(self.stream.peer_connection)
self.logger = logging.getLogger("webrtcd")
self.logger.info("New stream session (%s), cameras %s, audio in %s out %s, incoming services %s, outgoing services %s",
self.identifier, cameras, config.incoming_audio_track, config.expected_audio_track, incoming_services, outgoing_services)
def start(self):
self.run_task = asyncio.create_task(self.run())
async def stop_async(self):
if self.run_task is not None and not self.run_task.done():
self.run_task.cancel()
try:
await self.run_task
except asyncio.CancelledError:
pass
except Exception:
self.logger.exception("Stream session stop task failure")
self.run_task = None
await self.post_run_cleanup()
def stop(self):
# Backwards-compatible sync wrapper. Prefer `await stop_async()` from async contexts.
try:
loop = asyncio.get_running_loop()
# If we're already in an event loop, schedule async shutdown and return.
loop.create_task(self.stop_async())
return
except RuntimeError:
pass
asyncio.run(self.stop_async())
async def get_answer(self):
return await self.stream.start()
async def message_handler(self, message: bytes):
# Control messages are handled in-process and don't require an incoming cereal bridge.
try:
payload = json.loads(message) if isinstance(message, (bytes, str)) else None
except (ValueError, TypeError):
payload = None
if isinstance(payload, dict) and payload.get("type") == "timingSei":
enabled = bool(payload.get("enabled", False))
for track in self.video_tracks:
track.timing_sei_enabled = enabled
self.logger.info("timing SEI %s", "enabled" if enabled else "disabled")
return
if isinstance(payload, dict) and payload.get("type") == "setQuality":
if self.bitrate_controller is not None:
quality = str(payload.get("quality", "auto"))
self.bitrate_controller.set_quality(quality)
self.logger.info("livestream quality set to %s", quality)
return
if isinstance(payload, dict) and payload.get("type") == "setAudioEnabled":
enabled = bool(payload.get("enabled", True))
if hasattr(self.audio_input_track, "enable"):
self.audio_input_track.enable(enabled)
self.audio_send_enabled = enabled
self.logger.info("livestream audio send %s", "enabled" if enabled else "disabled")
return
if isinstance(payload, dict) and payload.get("type") == "setUiStream":
enabled = bool(payload.get("enabled", False))
self.set_ui_stream(enabled)
self.logger.info("ui stream %s", "enabled" if enabled else "disabled")
return
if isinstance(payload, dict) and payload.get("type") == "switchCamera":
camera = str(payload.get("camera", ""))
# Single-track model: repoint the (one) video track at the requested camera.
for track in self.video_tracks:
track.switch_camera(camera)
return
if self.incoming_bridge is None:
return
try:
self.incoming_bridge.send(message)
except Exception:
self.logger.exception("Cereal incoming proxy failure")
def set_ui_stream(self, enabled: bool):
if enabled:
if self.ui_stream_runner is not None or not self.stream.has_messaging_channel():
return
from iqpilot.system.webrtc.ui_stream import UIStreamMessageProxy
bitrate_getter = None
if self.bitrate_controller is not None:
controller = self.bitrate_controller
def bitrate_getter():
return controller.current_bitrate
proxy = UIStreamMessageProxy(bitrate_getter=bitrate_getter)
proxy.add_channel(self.stream.get_messaging_channel())
self.ui_stream_runner = CerealProxyRunner(proxy)
self.ui_stream_runner.start()
elif self.ui_stream_runner is not None:
self.ui_stream_runner.stop()
self.ui_stream_runner = None
async def add_ice_candidate(self, cand: Any):
"""Add a trickled ICE candidate from the client to the live peer connection."""
if not isinstance(cand, dict):
return
cand_str = cand.get("candidate") or ""
if not cand_str:
return # end-of-candidates marker; aiortc needs no explicit signal
try:
from aiortc.sdp import candidate_from_sdp
sdp_str = cand_str.split(":", 1)[-1] if cand_str.startswith("candidate:") else cand_str
ice = candidate_from_sdp(sdp_str)
ice.sdpMid = cand.get("sdpMid")
ice.sdpMLineIndex = cand.get("sdpMLineIndex")
await self.stream.peer_connection.addIceCandidate(ice)
except Exception:
self.logger.exception("Failed to add ICE candidate")
async def run(self):
try:
await self.stream.wait_for_connection()
if self.stream.has_messaging_channel():
# Always install the handler so control messages (e.g. timing SEI toggle) work
# even when no incoming cereal bridge service was requested.
self.stream.set_message_handler(self.message_handler)
if self.incoming_bridge is not None:
await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
if self.outgoing_bridge_runner is not None:
channel = self.stream.get_messaging_channel()
self.outgoing_bridge_runner.proxy.add_channel(channel)
self.outgoing_bridge_runner.start()
if self.ui_stream_requested:
self.set_ui_stream(True)
if self.audio_recv_requested and self.stream.has_incoming_audio_track():
track = self.stream.get_incoming_audio_track(buffered=False)
self.incoming_audio_proxy = IncomingAudioCerealProxy(track)
self.incoming_audio_proxy.start()
self.logger.info("Stream session (%s) incoming audio proxy started", self.identifier)
else:
self.logger.info("Stream session (%s) no incoming audio track from client", self.identifier)
if self.bitrate_controller is not None:
self.bitrate_controller.start()
self.logger.info(
"Stream session (%s) audio state send_requested=%s send_enabled=%s recv_requested=%s recv_active=%s",
self.identifier,
self.audio_send_requested,
self.audio_send_enabled,
self.audio_recv_requested,
self.incoming_audio_proxy is not None,
)
self.logger.info("Stream session (%s) connected", self.identifier)
await self.stream.wait_for_disconnection()
await self.post_run_cleanup()
self.logger.info("Stream session (%s) ended", self.identifier)
except Exception:
self.logger.exception("Stream session failure")
async def post_run_cleanup(self):
if self.bitrate_controller is not None:
self.bitrate_controller.stop()
await self.stream.stop()
if self.ui_stream_runner is not None:
self.ui_stream_runner.stop()
self.ui_stream_runner = None
if self.outgoing_bridge is not None:
self.outgoing_bridge_runner.stop()
if self.incoming_audio_proxy is not None:
await self.incoming_audio_proxy.stop()
def _is_retryable_stream_error(e: Exception) -> bool:
# Transient failures seen during answer generation: SDP/candidate parse issues
# (typically browser mDNS .local host candidates aiortc can't resolve) and
# socket-level hiccups while gathering. Anything else is a real error.
return isinstance(e, (ValueError, OSError))
async def _cleanup_failed_session(session: 'StreamSession | None', logger: logging.Logger) -> None:
async def _cleanup_failed_session(session: Any | None, logger: logging.Logger) -> None:
if session is None:
return
try:
@@ -543,15 +157,7 @@ class StreamRequestBody:
def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: bool):
if Params().get_bool("Konn3ktLibdatachannelWebRTC"):
try:
from iqpilot.system.webrtc.webrtcd_ldc import StreamSessionLibdatachannel
return StreamSessionLibdatachannel(
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream,
)
except Exception:
logging.getLogger("webrtcd").exception("libdatachannel unavailable; falling back to aiortc")
from iqpilot.system.webrtc.session import StreamSession
return StreamSession(
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
ui_stream=body.ui_stream,
@@ -561,7 +167,7 @@ def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: boo
async def get_stream(request: 'web.Request'):
stream_dict, debug_mode = request.app['streams'], request.app['debug']
logger = logging.getLogger("webrtcd")
session: StreamSession | None = None
session: Any | None = None
try:
raw_body = await request.json()
body = StreamRequestBody(**raw_body)