IQ.Pilot Release Commit @ 3fe374f
This commit is contained in:
@@ -1,105 +0,0 @@
|
||||
import asyncio
|
||||
import fractions
|
||||
|
||||
import aiortc
|
||||
import av
|
||||
import numpy as np
|
||||
|
||||
from iqpilot.cereal import messaging
|
||||
|
||||
|
||||
WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
|
||||
WEBRTC_AUDIO_PTIME = 0.020
|
||||
|
||||
|
||||
class AudioInputStreamTrack(aiortc.mediastreams.AudioStreamTrack):
|
||||
"""Device microphone -> WebRTC, sourced from micd's `rawAudioData` cereal stream.
|
||||
|
||||
micd owns the ALSA capture device, so opening it again via PyAudio fails with a host error
|
||||
('audio in use', PortAudio errno -9999). Instead we consume micd's already-published int16 mono
|
||||
PCM and repacketize it into WebRTC audio frames — no device contention, and it works whenever micd
|
||||
is running. Reading one message per recv() paces playout to micd's real-time publish rate.
|
||||
"""
|
||||
def __init__(self, rate: int = 16000, channels: int = 1):
|
||||
super().__init__()
|
||||
self.rate = rate
|
||||
self.channels = channels
|
||||
# conflate=False: keep audio continuous (don't drop buffered samples) for clean playback.
|
||||
self._sock = messaging.sub_sock("rawAudioData", conflate=False)
|
||||
self._start: float | None = None
|
||||
self.pts = 0
|
||||
self.enabled = True
|
||||
self._audio_buffer = bytearray()
|
||||
|
||||
def enable(self, enabled: bool) -> None:
|
||||
self.enabled = enabled
|
||||
|
||||
async def _fill_audio_buffer(self, target_bytes: int) -> None:
|
||||
deadline = asyncio.get_running_loop().time() + WEBRTC_AUDIO_PTIME
|
||||
while len(self._audio_buffer) < target_bytes:
|
||||
msg = messaging.recv_one_or_none(self._sock)
|
||||
if msg is not None:
|
||||
audio = msg.rawAudioData
|
||||
rate = int(audio.sampleRate) or self.rate
|
||||
if rate != self.rate:
|
||||
self.rate = rate
|
||||
self._audio_buffer.clear()
|
||||
self._start = None
|
||||
self.pts = 0
|
||||
self._audio_buffer.extend(bytes(audio.data))
|
||||
continue
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
break
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
async def _next_audio_data(self) -> tuple[bytes, int]:
|
||||
samples = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
|
||||
target_bytes = samples * 2
|
||||
await self._fill_audio_buffer(target_bytes)
|
||||
|
||||
if len(self._audio_buffer) >= target_bytes:
|
||||
data = bytes(self._audio_buffer[:target_bytes])
|
||||
del self._audio_buffer[:target_bytes]
|
||||
else:
|
||||
data = bytes(self._audio_buffer)
|
||||
self._audio_buffer.clear()
|
||||
data += bytes(target_bytes - len(data))
|
||||
|
||||
return data, self.rate
|
||||
|
||||
async def _pace(self, pts: int, sample_rate: int) -> None:
|
||||
if self._start is None:
|
||||
self._start = asyncio.get_running_loop().time()
|
||||
return
|
||||
|
||||
wait = self._start + (pts / sample_rate) - asyncio.get_running_loop().time()
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
async def recv(self):
|
||||
while True:
|
||||
if not self.enabled:
|
||||
break
|
||||
data, sample_rate = await self._next_audio_data()
|
||||
if data:
|
||||
samples = len(data) // 2
|
||||
pts = self.pts
|
||||
self.pts += samples
|
||||
await self._pace(pts, sample_rate)
|
||||
|
||||
frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
|
||||
frame.planes[0].update(data)
|
||||
frame.pts = pts
|
||||
frame.sample_rate = sample_rate
|
||||
frame.time_base = fractions.Fraction(1, sample_rate)
|
||||
return frame
|
||||
|
||||
samples_per_frame = max(1, int(WEBRTC_AUDIO_PTIME * self.rate))
|
||||
samples = np.zeros((1, samples_per_frame), dtype=np.int16)
|
||||
frame = av.AudioFrame.from_ndarray(samples, format='s16', layout='mono')
|
||||
frame.sample_rate = self.rate
|
||||
frame.time_base = fractions.Fraction(1, self.rate)
|
||||
frame.pts = self.pts
|
||||
self.pts += frame.samples
|
||||
await self._pace(frame.pts, self.rate)
|
||||
return frame
|
||||
@@ -7,17 +7,19 @@ import av
|
||||
|
||||
from iqpilot.cereal import messaging
|
||||
from iqpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
|
||||
from iqpilot.system.webrtc.device.audio import WEBRTC_AUDIO_PTIME, WEBRTC_AUDIO_SERVICE
|
||||
|
||||
|
||||
WEBRTC_AUDIO_SERVICE = "webrtcAudioData"
|
||||
WEBRTC_AUDIO_PTIME = 0.020
|
||||
|
||||
|
||||
class AudioInputOpusProducer:
|
||||
"""Micd PCM -> 48 kHz Opus payloads for libdatachannel's RTP packetizer."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._sock = messaging.sub_sock("rawAudioData", conflate=False)
|
||||
self._pcm = bytearray()
|
||||
self._source_rate = 16_000
|
||||
self._next_pts = 0
|
||||
self._packet_pts = 0
|
||||
self._pending: deque[tuple[bytes, int]] = deque()
|
||||
self._enabled = True
|
||||
self._resampler = av.AudioResampler(format="fltp", layout="mono", rate=48_000)
|
||||
@@ -65,13 +67,22 @@ class AudioInputOpusProducer:
|
||||
frame.time_base = Fraction(1, 48_000)
|
||||
self._next_pts += frame.samples
|
||||
for packet in self._encoder.encode(frame):
|
||||
self._pending.append((bytes(packet), int(packet.pts or 0)))
|
||||
self._pending.append((bytes(packet), self._packet_pts))
|
||||
self._packet_pts += int(packet.duration or frame.samples)
|
||||
return self._pending.popleft()
|
||||
|
||||
|
||||
class IncomingOpusCerealProxy:
|
||||
"""libdatachannel Opus payloads -> soundd-compatible PCM cereal messages."""
|
||||
class DebugAudioOpusProducer(AudioInputOpusProducer):
|
||||
async def _read_pcm_frame(self) -> av.AudioFrame:
|
||||
samples = int(WEBRTC_AUDIO_PTIME * self._source_rate)
|
||||
await asyncio.sleep(WEBRTC_AUDIO_PTIME)
|
||||
frame = av.AudioFrame(format="s16", layout="mono", samples=samples)
|
||||
frame.planes[0].update(bytes(samples * 2))
|
||||
frame.sample_rate = self._source_rate
|
||||
return frame
|
||||
|
||||
|
||||
class IncomingOpusCerealProxy:
|
||||
def __init__(self, track) -> None:
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._queue: asyncio.Queue[bytes] = asyncio.Queue(maxsize=32)
|
||||
@@ -4,7 +4,7 @@ import struct
|
||||
import time
|
||||
|
||||
import av
|
||||
from iqpilot.system.webrtc.teleoprtc_ldc.tracks import TiciVideoStreamTrack
|
||||
from iqpilot.system.webrtc.rtc.tracks import TiciVideoStreamTrack
|
||||
|
||||
from iqpilot.cereal import messaging
|
||||
from iqpilot.common.params import Params
|
||||
@@ -272,3 +272,40 @@ class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
|
||||
|
||||
def codec_preference(self) -> str | None:
|
||||
return "H264"
|
||||
|
||||
|
||||
class DebugVideoStreamTrack(TiciVideoStreamTrack):
|
||||
def __init__(self, camera_type: str):
|
||||
super().__init__(camera_type, 0.05)
|
||||
self._codec = av.CodecContext.create("libx264", "w")
|
||||
self._codec.width = 640
|
||||
self._codec.height = 480
|
||||
self._codec.pix_fmt = "yuv420p"
|
||||
self._codec.time_base = self._time_base
|
||||
self._codec.framerate = 20
|
||||
self._codec.options = {"preset": "ultrafast", "tune": "zerolatency"}
|
||||
self._codec.open()
|
||||
self._pts = 0
|
||||
self.timing_sei_enabled = False
|
||||
|
||||
async def recv(self):
|
||||
await asyncio.sleep(self._dt)
|
||||
frame = av.VideoFrame(self._codec.width, self._codec.height, "yuv420p")
|
||||
frame.planes[0].update(bytes(frame.planes[0].buffer_size))
|
||||
for plane in frame.planes[1:]:
|
||||
plane.update(bytes([128]) * plane.buffer_size)
|
||||
frame.pts = self._pts
|
||||
self._pts += int(self._dt * self._clock_rate)
|
||||
packets = self._codec.encode(frame)
|
||||
if not packets:
|
||||
return await self.recv()
|
||||
packet = av.Packet(b"".join(bytes(encoded) for encoded in packets))
|
||||
packet.pts = frame.pts
|
||||
packet.dts = frame.pts
|
||||
packet.time_base = self._time_base
|
||||
packet.duration = int(self._dt * self._clock_rate)
|
||||
return packet
|
||||
|
||||
def switch_camera(self, camera_type: str) -> None:
|
||||
if camera_type not in LiveStreamVideoStreamTrack.livestream_camera_to_sock_mapping:
|
||||
raise ValueError(f"Unknown camera {camera_type}")
|
||||
@@ -1,273 +0,0 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import struct
|
||||
import time
|
||||
|
||||
import av
|
||||
from teleoprtc.tracks import TiciVideoStreamTrack
|
||||
|
||||
from iqpilot.cereal import messaging
|
||||
from iqpilot.common.params import Params
|
||||
from iqpilot.common.realtime import DT_MDL, DT_DMON
|
||||
|
||||
# Arbitrary 16-byte UUID identifying konn3kt frame-timing SEI messages. When timing
|
||||
# telemetry is enabled, each frame carries a user_data_unregistered SEI NAL with four
|
||||
# big-endian doubles (ms): encode duration, IPC/queue delay, host transit, and the
|
||||
# device wall clock. The client decodes these to compute true glass-to-glass latency.
|
||||
TIMING_SEI_UUID = bytes([
|
||||
0xa5, 0xe0, 0xc4, 0xa4, 0x5b, 0x6e, 0x4e, 0x1e,
|
||||
0x9c, 0x7e, 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc,
|
||||
])
|
||||
# Annex-B start code + SEI NAL (type 6) + user_data_unregistered (type 5) + payload size
|
||||
# (0x30 = 48 bytes = 16 UUID + 32 data). Trailing 0x80 is the RBSP stop bit.
|
||||
_SEI_PREFIX = b'\x00\x00\x00\x01\x06\x05\x30' + TIMING_SEI_UUID
|
||||
|
||||
|
||||
class LiveStreamVideoStreamTrack(TiciVideoStreamTrack):
|
||||
livestream_camera_to_sock_mapping = {
|
||||
"driver": "livestreamDriverEncodeData",
|
||||
"wideRoad": "livestreamWideRoadEncodeData",
|
||||
"road": "livestreamRoadEncodeData",
|
||||
}
|
||||
main_camera_to_sock_mapping = {
|
||||
"driver": "driverEncodeData",
|
||||
"wideRoad": "wideRoadEncodeData",
|
||||
"road": "roadEncodeData",
|
||||
}
|
||||
|
||||
# Number of live tracks still waiting for their first keyframe. The on-demand
|
||||
# keyframe request (LivestreamRequestKeyframe) is a single global param honored by
|
||||
# every encoder, so with multiple concurrent tracks (dual-camera PiP) we must not
|
||||
# clear it until *all* tracks have received an IDR — otherwise the first track to
|
||||
# get its keyframe clears the request and starves the others (black feed).
|
||||
_kf_pending_count = 0
|
||||
|
||||
def __init__(self, camera_type: str):
|
||||
dt = DT_DMON if camera_type == "driver" else DT_MDL
|
||||
super().__init__(camera_type, dt)
|
||||
|
||||
self._params = Params()
|
||||
self._camera_type = camera_type
|
||||
self._candidate_topics = [
|
||||
self.main_camera_to_sock_mapping[camera_type],
|
||||
self.livestream_camera_to_sock_mapping[camera_type],
|
||||
]
|
||||
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
|
||||
self._active_topic = self._preferred_topics()[0]
|
||||
self._pts = 0
|
||||
self._t0_ns = time.monotonic_ns()
|
||||
self._cached_header: bytes = b""
|
||||
self._sent_keyframe = False
|
||||
self._kf_requested = False # whether this track counts toward _kf_pending_count
|
||||
self._frame_count = 0
|
||||
self._last_frame_time = 0.0
|
||||
self._last_preference_refresh = 0.0
|
||||
# Tracks how long the H264 livestream feed has been silent, to gate the last-resort main-feed
|
||||
# fallback (see recv) without flapping between sources frame-by-frame.
|
||||
self._live_silent_since: float | None = None
|
||||
# Opt-in glass-to-glass latency telemetry (toggled by the client over the data channel).
|
||||
self.timing_sei_enabled = False
|
||||
self._logger = logging.getLogger("LiveStreamVideoStreamTrack")
|
||||
|
||||
# Ask the encoder for an immediate IDR so the stream starts fast instead of waiting up to a full
|
||||
# GOP for the next periodic keyframe (encoderd honors LivestreamRequestKeyframe per-frame).
|
||||
self._mark_keyframe_needed()
|
||||
|
||||
def _request_keyframe(self, enabled: bool) -> None:
|
||||
try:
|
||||
self._params.put_bool("LivestreamRequestKeyframe", enabled)
|
||||
except Exception:
|
||||
self._logger.exception("failed to set LivestreamRequestKeyframe")
|
||||
|
||||
def _mark_keyframe_needed(self) -> None:
|
||||
"""This track needs (another) keyframe: keep the global request asserted."""
|
||||
if not self._kf_requested:
|
||||
LiveStreamVideoStreamTrack._kf_pending_count += 1
|
||||
self._kf_requested = True
|
||||
self._request_keyframe(True)
|
||||
|
||||
def request_keyframe(self) -> None:
|
||||
self._mark_keyframe_needed()
|
||||
|
||||
def _mark_keyframe_received(self) -> None:
|
||||
"""This track got its keyframe; only clear the global request once no track needs one."""
|
||||
if self._kf_requested:
|
||||
self._kf_requested = False
|
||||
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
|
||||
if LiveStreamVideoStreamTrack._kf_pending_count == 0:
|
||||
self._request_keyframe(False)
|
||||
|
||||
def stop(self):
|
||||
# Release our pending-keyframe hold so a torn-down track that never received an
|
||||
# IDR doesn't pin LivestreamRequestKeyframe True forever (continuous keyframes).
|
||||
if getattr(self, "_kf_requested", False):
|
||||
self._kf_requested = False
|
||||
LiveStreamVideoStreamTrack._kf_pending_count = max(0, LiveStreamVideoStreamTrack._kf_pending_count - 1)
|
||||
try:
|
||||
super().stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def switch_camera(self, camera_type: str) -> None:
|
||||
"""Repoint this track at a different camera without renegotiating the peer connection.
|
||||
|
||||
Lets a single video track back the whole Live View — the client flips cameras over the
|
||||
data channel and we swap the source here, instead of uplinking every camera at once."""
|
||||
if camera_type not in self.livestream_camera_to_sock_mapping:
|
||||
self._logger.warning("[%s] ignoring switch to unknown camera %s", self._id, camera_type)
|
||||
return
|
||||
if camera_type == self._camera_type:
|
||||
return
|
||||
self._logger.info("[%s] switching camera %s -> %s", self._id, self._camera_type, camera_type)
|
||||
self._camera_type = camera_type
|
||||
self._candidate_topics = [
|
||||
self.main_camera_to_sock_mapping[camera_type],
|
||||
self.livestream_camera_to_sock_mapping[camera_type],
|
||||
]
|
||||
self._socks = {topic: messaging.sub_sock(topic, conflate=True) for topic in self._candidate_topics}
|
||||
self._active_topic = self._preferred_topics()[0]
|
||||
# Force a fresh keyframe/header before emitting frames from the new source, and ask the encoder
|
||||
# for an immediate IDR so the camera switch isn't stalled waiting for the next periodic keyframe.
|
||||
self._cached_header = b""
|
||||
self._sent_keyframe = False
|
||||
self._last_preference_refresh = 0.0
|
||||
self._live_silent_since = None
|
||||
self._mark_keyframe_needed()
|
||||
|
||||
def _preferred_topics(self) -> list[str]:
|
||||
# WebRTC currently forces H.264. The dedicated livestream topics are the H.264 feeds,
|
||||
# while the main encode topics are the full-resolution HEVC recordings. Prefer the
|
||||
# livestream feeds both onroad and offroad, and keep the main topics only as fallback.
|
||||
return [
|
||||
self.livestream_camera_to_sock_mapping[self._camera_type],
|
||||
self.main_camera_to_sock_mapping[self._camera_type],
|
||||
]
|
||||
|
||||
def _reset_decoder_state(self, topic: str) -> None:
|
||||
if topic == self._active_topic:
|
||||
return
|
||||
self._logger.info("[%s] switching video source from %s to %s", self._id, self._active_topic, topic)
|
||||
self._active_topic = topic
|
||||
self._cached_header = b""
|
||||
self._sent_keyframe = False
|
||||
|
||||
def _timing_sei(self, evta, log_mono_time: int) -> bytes:
|
||||
"""Build a timing SEI NAL from encode metadata, or empty bytes when disabled."""
|
||||
if not self.timing_sei_enabled:
|
||||
return b""
|
||||
idx = evta.idx
|
||||
return _SEI_PREFIX + struct.pack(
|
||||
'>4d',
|
||||
(idx.timestampEof - idx.timestampSof) / 1e6, # encode duration (ms)
|
||||
(log_mono_time - idx.timestampEof) / 1e6, # IPC/queue delay (ms)
|
||||
(time.monotonic_ns() - log_mono_time) / 1e6, # host transit so far (ms)
|
||||
time.time() * 1000, # device wall clock (ms) # noqa: TID251
|
||||
) + b'\x80'
|
||||
|
||||
def _is_keyframe(self, data: bytes) -> bool:
|
||||
"""Check if H.264 NAL unit contains an IDR keyframe (NAL type 5)."""
|
||||
i = 0
|
||||
while i < len(data) - 4:
|
||||
# Look for Annex B start codes: 0x000001 or 0x00000001
|
||||
if data[i:i+3] == b'\x00\x00\x01':
|
||||
nal_type = data[i+3] & 0x1f
|
||||
if nal_type == 5: # IDR slice
|
||||
return True
|
||||
i += 3
|
||||
elif data[i:i+4] == b'\x00\x00\x00\x01':
|
||||
nal_type = data[i+4] & 0x1f
|
||||
if nal_type == 5: # IDR slice
|
||||
return True
|
||||
i += 4
|
||||
else:
|
||||
i += 1
|
||||
return False
|
||||
|
||||
async def recv(self):
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
# Resolve topics each iteration: a camera switch (different async task) can rebuild self._socks
|
||||
# across the await below, so a value cached before the loop would index a stale key (KeyError).
|
||||
live_topic = self.livestream_camera_to_sock_mapping[self._camera_type]
|
||||
main_topic = self.main_camera_to_sock_mapping[self._camera_type]
|
||||
# Lock onto the dedicated H264 livestream feed. Onroad the HEVC main feed also publishes at
|
||||
# 20fps; eagerly preferring whichever socket had a frame ready raced frame-by-frame, reset the
|
||||
# decoder every frame, and (the track is negotiated H264) shoved HEVC garbage into the stream —
|
||||
# the onroad choppiness. Only fall back to the main feed as a last resort after a long
|
||||
# livestream silence (e.g. stream_encoderd still spinning up), and snap back when it returns.
|
||||
msg = messaging.recv_one_or_none(self._socks[live_topic])
|
||||
if msg is not None:
|
||||
self._reset_decoder_state(live_topic)
|
||||
self._last_frame_time = now
|
||||
self._live_silent_since = None
|
||||
break
|
||||
|
||||
if self._live_silent_since is None:
|
||||
self._live_silent_since = now
|
||||
elif now - self._live_silent_since > 3.0:
|
||||
maybe_msg = messaging.recv_one_or_none(self._socks[main_topic])
|
||||
if maybe_msg is not None:
|
||||
self._reset_decoder_state(main_topic)
|
||||
self._last_frame_time = now
|
||||
msg = maybe_msg
|
||||
break
|
||||
|
||||
await asyncio.sleep(0.005)
|
||||
|
||||
evta = getattr(msg, msg.which())
|
||||
|
||||
header = bytes(evta.header)
|
||||
data = bytes(evta.data)
|
||||
self._frame_count += 1
|
||||
|
||||
# Cache SPS/PPS header when it arrives
|
||||
if header:
|
||||
self._cached_header = header
|
||||
self._logger.debug(f"[{self._id}] cached SPS/PPS header ({len(header)} bytes)")
|
||||
|
||||
# CRITICAL: Cannot decode without SPS/PPS. Wait for it.
|
||||
if not self._cached_header:
|
||||
self._logger.debug(f"[{self._id}] frame {self._frame_count}: no SPS/PPS yet, skipping")
|
||||
return await self.recv()
|
||||
|
||||
is_keyframe = self._is_keyframe(data)
|
||||
|
||||
# Wait for first keyframe before sending any frames
|
||||
# Browser decoder needs IDR to initialize properly
|
||||
if not self._sent_keyframe:
|
||||
if not is_keyframe:
|
||||
self._logger.debug(f"[{self._id}] frame {self._frame_count}: waiting for keyframe")
|
||||
return await self.recv()
|
||||
self._sent_keyframe = True
|
||||
# Got the IDR we asked for — stop nagging the encoder, but only once every
|
||||
# concurrent track has its keyframe (multi-track PiP shares the global param).
|
||||
self._mark_keyframe_received()
|
||||
self._logger.info(f"[{self._id}] first keyframe received, starting stream")
|
||||
|
||||
# Optional timing SEI NAL, inserted before the slice data (and after SPS/PPS on keyframes).
|
||||
sei_nal = self._timing_sei(evta, msg.logMonoTime)
|
||||
|
||||
# Prepend SPS/PPS header to keyframes (required by some decoders)
|
||||
# For non-keyframes, header is optional but safe to include
|
||||
if is_keyframe:
|
||||
payload = self._cached_header + sei_nal + data
|
||||
else:
|
||||
payload = sei_nal + data
|
||||
|
||||
self._pts = ((time.monotonic_ns() - self._t0_ns) * self._clock_rate) // 1_000_000_000
|
||||
|
||||
packet = av.Packet(payload)
|
||||
packet.time_base = self._time_base
|
||||
packet.pts = int(self._pts)
|
||||
packet.dts = int(self._pts)
|
||||
packet.duration = int(self._dt * self._clock_rate)
|
||||
|
||||
if is_keyframe:
|
||||
packet.is_keyframe = True
|
||||
|
||||
self.log_debug("track sending frame %s (keyframe=%s, size=%d)", self._pts, is_keyframe, len(payload))
|
||||
|
||||
return packet
|
||||
|
||||
def codec_preference(self) -> str | None:
|
||||
return "H264"
|
||||
@@ -1,4 +1,4 @@
|
||||
# ruff: noqa: TID251, UP006, UP035
|
||||
# ruff: noqa: UP006, UP035
|
||||
|
||||
import abc
|
||||
from typing import Dict, List, Optional
|
||||
@@ -1,4 +1,4 @@
|
||||
# ruff: noqa: TID251, UP006, UP035
|
||||
# ruff: noqa: UP006, UP035
|
||||
|
||||
import abc
|
||||
import asyncio
|
||||
@@ -14,6 +14,7 @@ from libdatachannel import (
|
||||
Candidate,
|
||||
Description,
|
||||
FrameInfo,
|
||||
H264RtpDepacketizer,
|
||||
H264RtpPacketizer,
|
||||
IceServer,
|
||||
NalUnit,
|
||||
@@ -100,7 +101,6 @@ class WebRTCBaseStream(abc.ABC):
|
||||
|
||||
@staticmethod
|
||||
def _make_ice_servers(servers: Optional[List[dict]]) -> List[IceServer]:
|
||||
"""Preserve Konn3kt's authenticated STUN/TURN configuration in libdatachannel."""
|
||||
parsed: List[IceServer] = []
|
||||
for server in servers or []:
|
||||
urls = server.get("urls", []) if isinstance(server, dict) else []
|
||||
@@ -119,7 +119,9 @@ class WebRTCBaseStream(abc.ABC):
|
||||
except Exception:
|
||||
logging.getLogger("WebRTCStream").warning("Ignoring invalid ICE server %r", url, exc_info=True)
|
||||
# A supplied list is authoritative, including LAN-only sessions where relay is deliberately filtered.
|
||||
return parsed or [IceServer("stun:stun.l.google.com:19302")]
|
||||
if servers is not None:
|
||||
return parsed
|
||||
return [IceServer("stun:stun.l.google.com:19302")]
|
||||
|
||||
def _log_debug(self, msg: Any, *args):
|
||||
self.logger.debug(f"{type(self)}() {msg}", *args)
|
||||
@@ -144,13 +146,14 @@ class WebRTCBaseStream(abc.ABC):
|
||||
media = Description.Video(camera_type, Description.Direction.RecvOnly)
|
||||
media.add_h264_codec(96)
|
||||
track = self.peer_connection.add_track(media)
|
||||
track.set_media_handler(OpusRtpDepacketizer())
|
||||
track.set_media_handler(H264RtpDepacketizer())
|
||||
self._consumer_tracks.append(track)
|
||||
self.incoming_camera_tracks[camera_type] = track
|
||||
if self.expected_incoming_audio:
|
||||
media = Description.Audio("audio", Description.Direction.RecvOnly)
|
||||
media.add_opus_codec(111)
|
||||
track = self.peer_connection.add_track(media)
|
||||
track.set_media_handler(OpusRtpDepacketizer())
|
||||
self._consumer_tracks.append(track)
|
||||
self.incoming_audio_tracks.append(track)
|
||||
|
||||
@@ -170,7 +173,7 @@ class WebRTCBaseStream(abc.ABC):
|
||||
def _make_video_media(self, track: TiciVideoStreamTrack, remote_sdp: str) -> Tuple[Description.Video, int, int, str]:
|
||||
mid, payload_type = self._find_offer_video(remote_sdp)
|
||||
ssrc = random.randint(1, 0xFFFFFFFF)
|
||||
cname = f"teleoprtc-{random.getrandbits(32):08x}"
|
||||
cname = f"iqpilot-video-{random.getrandbits(32):08x}"
|
||||
stream_id = f"stream-{random.getrandbits(32):08x}"
|
||||
media = Description.Video(mid, Description.Direction.SendOnly)
|
||||
media.add_h264_codec(payload_type)
|
||||
@@ -193,7 +196,7 @@ class WebRTCBaseStream(abc.ABC):
|
||||
def _make_audio_media(self, remote_sdp: str) -> Tuple[Description.Audio, int, int, str]:
|
||||
mid, payload_type = self._find_offer_audio(remote_sdp)
|
||||
ssrc = random.randint(1, 0xFFFFFFFF)
|
||||
cname = f"teleoprtc-audio-{random.getrandbits(32):08x}"
|
||||
cname = f"iqpilot-audio-{random.getrandbits(32):08x}"
|
||||
direction = Description.Direction.SendRecv if self.expected_incoming_audio else Description.Direction.SendOnly
|
||||
media = Description.Audio(mid, direction)
|
||||
media.add_opus_codec(payload_type)
|
||||
@@ -352,7 +355,6 @@ class WebRTCBaseStream(abc.ABC):
|
||||
self.incoming_message_handlers.append(message_handler)
|
||||
|
||||
def add_ice_candidate(self, candidate: dict) -> None:
|
||||
"""Accept post-offer browser candidates for Konn3kt's existing trickle endpoint."""
|
||||
candidate_sdp = str(candidate.get("candidate") or "") if isinstance(candidate, dict) else ""
|
||||
if not candidate_sdp:
|
||||
return
|
||||
@@ -32,6 +32,7 @@ class TiciVideoStreamTrack:
|
||||
def __init__(self, camera_type: str, dt: float, time_base: fractions.Fraction = VIDEO_TIME_BASE, clock_rate: int = VIDEO_CLOCK_RATE):
|
||||
assert camera_type in ["driver", "wideRoad", "road"]
|
||||
self._id: str = video_track_id(camera_type, str(uuid.uuid4()))
|
||||
self._dt = dt
|
||||
self._time_base: fractions.Fraction = time_base
|
||||
self._clock_rate: int = clock_rate
|
||||
self._logger = logging.getLogger("WebRTCStream")
|
||||
@@ -14,7 +14,6 @@ from iqpilot.system.webrtc.webrtcd import CerealIncomingMessageProxy, CerealOutg
|
||||
|
||||
|
||||
def _default_route_ip() -> str | None:
|
||||
"""Use the interface the kernel will actually use for Internet/relay media."""
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
try:
|
||||
sock.connect(("8.8.8.8", 53))
|
||||
@@ -25,9 +24,7 @@ def _default_route_ip() -> str | None:
|
||||
sock.close()
|
||||
|
||||
|
||||
class LibdatachannelBitrateController:
|
||||
"""Loss-driven bitrate control using native RTCP receiver reports."""
|
||||
|
||||
class LivestreamBitrateController:
|
||||
bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
|
||||
label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[2]}
|
||||
sample_interval = 0.2
|
||||
@@ -103,45 +100,45 @@ class LibdatachannelBitrateController:
|
||||
self._publish(self.bitrates[self._level])
|
||||
|
||||
|
||||
class StreamSessionLibdatachannel:
|
||||
"""IQ.Pilot's libdatachannel stream path. It preserves Konn3kt signalling and controls."""
|
||||
|
||||
shared_pub_master = DynamicPubMaster([])
|
||||
class StreamSession:
|
||||
shared_pub_master: DynamicPubMaster | None = None
|
||||
|
||||
def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
|
||||
ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
|
||||
from iqpilot.system.webrtc.device.video_ldc import LiveStreamVideoStreamTrack
|
||||
from iqpilot.system.webrtc.teleoprtc_ldc.builder import WebRTCAnswerBuilder
|
||||
from iqpilot.system.webrtc.teleoprtc_ldc.info import parse_info_from_offer
|
||||
from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
|
||||
from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
|
||||
from iqpilot.system.webrtc.rtc.builder import WebRTCAnswerBuilder
|
||||
from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
|
||||
|
||||
config = parse_info_from_offer(sdp)
|
||||
if len(cameras) != config.n_expected_camera_tracks:
|
||||
raise ValueError("Incoming stream has misconfigured number of video tracks")
|
||||
if debug_mode:
|
||||
raise ValueError("libdatachannel debug tracks are not supported")
|
||||
|
||||
builder = WebRTCAnswerBuilder(sdp, bind_address=_default_route_ip(), ice_servers=ice_servers or [])
|
||||
self.video_tracks = [LiveStreamVideoStreamTrack(camera) for camera in cameras]
|
||||
video_track_type = DebugVideoStreamTrack if debug_mode else LiveStreamVideoStreamTrack
|
||||
self.video_tracks = [video_track_type(camera) for camera in cameras]
|
||||
for camera, track in zip(cameras, self.video_tracks, strict=True):
|
||||
builder.add_video_stream(camera, track)
|
||||
|
||||
# The browser uses a single sendrecv audio m-line. libdatachannel's Python binding
|
||||
# currently cannot negotiate that bidirectional track reliably, so this experimental
|
||||
# transport deliberately remains video/control-only. The default aiortc path keeps
|
||||
# both audio directions until the native duplex path passes the same integration test.
|
||||
self.audio_output = None
|
||||
audio_track_type = DebugAudioOpusProducer if debug_mode else AudioInputOpusProducer
|
||||
self.audio_output = audio_track_type() if config.expected_audio_track else None
|
||||
if self.audio_output is not None:
|
||||
builder.add_audio_stream(self.audio_output)
|
||||
if config.incoming_audio_track:
|
||||
builder.offer_to_receive_audio_stream()
|
||||
self.stream = builder.stream()
|
||||
|
||||
self.identifier = str(uuid.uuid4())
|
||||
self.incoming_bridge_services = incoming_services
|
||||
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if incoming_services else None
|
||||
if incoming_services and self.shared_pub_master is None:
|
||||
StreamSession.shared_pub_master = DynamicPubMaster([])
|
||||
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master) if self.shared_pub_master is not None and incoming_services else None
|
||||
self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services)) if outgoing_services else None
|
||||
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge) if self.outgoing_bridge is not None else None
|
||||
self.ui_stream_requested = ui_stream
|
||||
self.ui_stream_runner: CerealProxyRunner | None = None
|
||||
self.audio_input_proxy = None
|
||||
self.audio_recv_requested = False
|
||||
self.bitrate_controller = LibdatachannelBitrateController(self.stream.get_receiver_report_stats)
|
||||
self.audio_recv_requested = config.incoming_audio_track
|
||||
self.bitrate_controller = LivestreamBitrateController(self.stream.get_receiver_report_stats)
|
||||
self.run_task: asyncio.Task | None = None
|
||||
self._cleanup_lock = asyncio.Lock()
|
||||
self._cleanup_done = False
|
||||
@@ -216,7 +213,7 @@ class StreamSessionLibdatachannel:
|
||||
if self.ui_stream_requested:
|
||||
self.set_ui_stream(True)
|
||||
if self.audio_recv_requested and self.stream.has_incoming_audio_track():
|
||||
from iqpilot.system.webrtc.device.audio_ldc import IncomingOpusCerealProxy
|
||||
from iqpilot.system.webrtc.device.native_audio import IncomingOpusCerealProxy
|
||||
self.audio_input_proxy = IncomingOpusCerealProxy(self.stream.get_incoming_audio_track())
|
||||
self.audio_input_proxy.start()
|
||||
self.bitrate_controller.start()
|
||||
96
iqpilot/system/webrtc/tests/test_native_session.py
Normal file
96
iqpilot/system/webrtc/tests/test_native_session.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from iqpilot.system.webrtc import session as session_module
|
||||
|
||||
|
||||
class FakeStream:
|
||||
def get_receiver_report_stats(self):
|
||||
return {}
|
||||
|
||||
|
||||
class FakeBuilder:
|
||||
instance = None
|
||||
|
||||
def __init__(self, sdp, bind_address=None, ice_servers=None):
|
||||
self.sdp = sdp
|
||||
self.bind_address = bind_address
|
||||
self.ice_servers = ice_servers
|
||||
self.video = []
|
||||
self.audio = []
|
||||
self.receive_audio = False
|
||||
self.result = FakeStream()
|
||||
FakeBuilder.instance = self
|
||||
|
||||
def add_video_stream(self, camera, track):
|
||||
self.video.append((camera, track))
|
||||
|
||||
def add_audio_stream(self, track):
|
||||
self.audio.append(track)
|
||||
|
||||
def offer_to_receive_audio_stream(self):
|
||||
self.receive_audio = True
|
||||
|
||||
def stream(self):
|
||||
return self.result
|
||||
|
||||
|
||||
class FakeVideoTrack:
|
||||
def __init__(self, camera):
|
||||
self.camera = camera
|
||||
self.timing_sei_enabled = False
|
||||
self.switched = None
|
||||
|
||||
def switch_camera(self, camera):
|
||||
self.switched = camera
|
||||
|
||||
|
||||
class FakeAudioProducer:
|
||||
def __init__(self):
|
||||
self.enabled = True
|
||||
|
||||
def enable(self, enabled):
|
||||
self.enabled = enabled
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_session(mocker):
|
||||
config = SimpleNamespace(
|
||||
n_expected_camera_tracks=1,
|
||||
expected_audio_track=True,
|
||||
incoming_audio_track=True,
|
||||
incoming_datachannel=True,
|
||||
)
|
||||
mocker.patch("iqpilot.system.webrtc.rtc.info.parse_info_from_offer", return_value=config)
|
||||
mocker.patch("iqpilot.system.webrtc.rtc.builder.WebRTCAnswerBuilder", FakeBuilder)
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_video.LiveStreamVideoStreamTrack", FakeVideoTrack)
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_audio.AudioInputOpusProducer", FakeAudioProducer)
|
||||
mocker.patch.object(session_module, "_default_route_ip", return_value="192.0.2.1")
|
||||
mocker.patch.object(session_module, "Params", return_value=mocker.Mock())
|
||||
return session_module.StreamSession(
|
||||
"offer", ["road"], [], [], [{"urls": "turn:example.com"}], ui_stream=False,
|
||||
)
|
||||
|
||||
|
||||
def test_native_session_builds_duplex_audio(native_session):
|
||||
builder = FakeBuilder.instance
|
||||
assert builder is not None
|
||||
assert builder.bind_address == "192.0.2.1"
|
||||
assert builder.ice_servers == [{"urls": "turn:example.com"}]
|
||||
assert [camera for camera, _ in builder.video] == ["road"]
|
||||
assert builder.audio == [native_session.audio_output]
|
||||
assert builder.receive_audio
|
||||
assert native_session.audio_recv_requested
|
||||
|
||||
|
||||
def test_native_session_controls(native_session, mocker):
|
||||
native_session.bitrate_controller = mocker.Mock()
|
||||
native_session.message_handler('{"type":"timingSei","enabled":true}')
|
||||
assert native_session.video_tracks[0].timing_sei_enabled
|
||||
native_session.message_handler('{"type":"switchCamera","camera":"driver"}')
|
||||
assert native_session.video_tracks[0].switched == "driver"
|
||||
native_session.message_handler('{"type":"setAudioEnabled","enabled":false}')
|
||||
assert not native_session.audio_output.enabled
|
||||
native_session.message_handler('{"type":"setQuality","quality":"low"}')
|
||||
native_session.bitrate_controller.set_quality.assert_called_once_with("low")
|
||||
116
iqpilot/system/webrtc/tests/test_rtc.py
Normal file
116
iqpilot/system/webrtc/tests/test_rtc.py
Normal file
@@ -0,0 +1,116 @@
|
||||
import pytest
|
||||
|
||||
from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
|
||||
from iqpilot.system.webrtc.rtc.stream import WebRTCBaseStream
|
||||
from iqpilot.system.webrtc.rtc.tracks import TiciTrackWrapper, TiciVideoStreamTrack, parse_video_track_id, video_track_id
|
||||
|
||||
|
||||
def sdp_with_media(media):
|
||||
mids = " ".join(str(i) for i in range(len(media)))
|
||||
sections = []
|
||||
for index, (kind, direction) in enumerate(media):
|
||||
if kind == "video":
|
||||
payload, mapping = "96", "H264/90000"
|
||||
protocol = "UDP/TLS/RTP/SAVPF"
|
||||
elif kind == "audio":
|
||||
payload, mapping = "111", "opus/48000/2"
|
||||
protocol = "UDP/TLS/RTP/SAVPF"
|
||||
else:
|
||||
sections.extend([
|
||||
"m=application 9 UDP/DTLS/SCTP webrtc-datachannel",
|
||||
"c=IN IP4 0.0.0.0",
|
||||
f"a=mid:{index}",
|
||||
"a=sctp-port:5000",
|
||||
])
|
||||
continue
|
||||
sections.extend([
|
||||
f"m={kind} 9 {protocol} {payload}",
|
||||
"c=IN IP4 0.0.0.0",
|
||||
f"a=mid:{index}",
|
||||
f"a={direction}",
|
||||
f"a=rtpmap:{payload} {mapping}",
|
||||
"a=rtcp-mux",
|
||||
])
|
||||
lines = [
|
||||
"v=0",
|
||||
"o=- 1 1 IN IP4 0.0.0.0",
|
||||
"s=-",
|
||||
"t=0 0",
|
||||
f"a=group:BUNDLE {mids}",
|
||||
*sections,
|
||||
]
|
||||
return "\r\n".join(lines) + "\r\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("direction,expected_outgoing,expected_incoming", [
|
||||
("recvonly", True, False),
|
||||
("sendonly", False, True),
|
||||
("sendrecv", True, True),
|
||||
("inactive", False, False),
|
||||
])
|
||||
def test_audio_directions(direction, expected_outgoing, expected_incoming):
|
||||
info = parse_info_from_offer(sdp_with_media([("audio", direction)]))
|
||||
assert info.expected_audio_track == expected_outgoing
|
||||
assert info.incoming_audio_track == expected_incoming
|
||||
|
||||
|
||||
def test_video_and_data_channel_metadata():
|
||||
info = parse_info_from_offer(sdp_with_media([
|
||||
("video", "recvonly"),
|
||||
("video", "sendrecv"),
|
||||
("application", "sendrecv"),
|
||||
]))
|
||||
assert info.n_expected_camera_tracks == 2
|
||||
assert info.incoming_datachannel
|
||||
|
||||
|
||||
def test_explicit_empty_ice_servers_disable_defaults():
|
||||
assert WebRTCBaseStream._make_ice_servers([]) == []
|
||||
|
||||
|
||||
def test_default_ice_server():
|
||||
servers = WebRTCBaseStream._make_ice_servers(None)
|
||||
assert len(servers) == 1
|
||||
assert servers[0].hostname == "stun.l.google.com"
|
||||
assert servers[0].port == 19302
|
||||
|
||||
|
||||
def test_authenticated_ice_servers():
|
||||
servers = WebRTCBaseStream._make_ice_servers([{
|
||||
"urls": ["turn:relay.example.com:3478", "stun:stun.example.com:3478"],
|
||||
"username": "user",
|
||||
"credential": "secret",
|
||||
}])
|
||||
assert [(server.hostname, server.port) for server in servers] == [
|
||||
("relay.example.com", 3478),
|
||||
("stun.example.com", 3478),
|
||||
]
|
||||
assert all(server.username == "user" and server.password == "secret" for server in servers)
|
||||
|
||||
|
||||
def test_track_id_roundtrip():
|
||||
assert parse_video_track_id(video_track_id("driver", "track")) == ("driver", "track")
|
||||
|
||||
|
||||
def test_invalid_track_id():
|
||||
with pytest.raises(ValueError):
|
||||
parse_video_track_id("driver")
|
||||
|
||||
|
||||
def test_track_wrapper_preserves_camera():
|
||||
class Track:
|
||||
kind = "video"
|
||||
id = "source"
|
||||
|
||||
async def recv(self):
|
||||
return b"frame"
|
||||
|
||||
wrapper = TiciTrackWrapper("road", Track())
|
||||
assert parse_video_track_id(wrapper.id)[0] == "road"
|
||||
wrapper.stop()
|
||||
assert wrapper.readyState == "ended"
|
||||
|
||||
|
||||
def test_track_stores_frame_period():
|
||||
track = TiciVideoStreamTrack("wideRoad", 0.05)
|
||||
assert track._dt == 0.05
|
||||
79
iqpilot/system/webrtc/tests/test_rtc_integration.py
Normal file
79
iqpilot/system/webrtc/tests/test_rtc_integration.py
Normal file
@@ -0,0 +1,79 @@
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from iqpilot.system.webrtc.rtc import WebRTCOfferBuilder
|
||||
from iqpilot.system.webrtc.rtc.stream import RTCSessionDescription
|
||||
from iqpilot.system.webrtc.session import StreamSession
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_video_audio_and_data_channel():
|
||||
if not os.environ.get("CI"):
|
||||
return
|
||||
|
||||
answer_session = None
|
||||
video_received = asyncio.Event()
|
||||
audio_received = asyncio.Event()
|
||||
|
||||
async def connect(offer):
|
||||
nonlocal answer_session
|
||||
answer_session = StreamSession(offer.sdp, offer.video, [], [], [], debug_mode=True)
|
||||
answer = await answer_session.get_answer()
|
||||
answer_session.start()
|
||||
return RTCSessionDescription(answer.sdp, answer.type)
|
||||
|
||||
builder = WebRTCOfferBuilder(connect, ice_servers=[])
|
||||
builder.offer_to_receive_video_stream("road")
|
||||
builder.offer_to_receive_audio_stream()
|
||||
builder.add_messaging()
|
||||
stream = builder.stream()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(stream.start(), 10)
|
||||
stream.get_incoming_video_track("road").on_frame(lambda *_: video_received.set())
|
||||
stream.get_incoming_audio_track().on_frame(lambda *_: audio_received.set())
|
||||
await asyncio.wait_for(stream.wait_for_connection(), 10)
|
||||
await asyncio.wait_for(video_received.wait(), 10)
|
||||
await asyncio.wait_for(audio_received.wait(), 10)
|
||||
stream.get_messaging_channel().send('{"type":"timingSei","enabled":true}')
|
||||
await asyncio.sleep(0.1)
|
||||
assert answer_session is not None
|
||||
assert answer_session.video_tracks[0].timing_sei_enabled
|
||||
finally:
|
||||
await stream.stop()
|
||||
if answer_session is not None:
|
||||
await answer_session.stop_async()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_duplex_audio_negotiation():
|
||||
if not os.environ.get("CI"):
|
||||
return
|
||||
|
||||
answer_session = None
|
||||
|
||||
async def connect(offer):
|
||||
nonlocal answer_session
|
||||
sendrecv_offer = offer.sdp.replace("a=recvonly", "a=sendrecv", 1)
|
||||
answer_session = StreamSession(sendrecv_offer, offer.video, [], [], [], debug_mode=True)
|
||||
answer = await answer_session.get_answer()
|
||||
answer_session.start()
|
||||
return RTCSessionDescription(answer.sdp, answer.type)
|
||||
|
||||
builder = WebRTCOfferBuilder(connect, ice_servers=[])
|
||||
builder.offer_to_receive_audio_stream()
|
||||
stream = builder.stream()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(stream.start(), 10)
|
||||
await asyncio.wait_for(stream.wait_for_connection(), 10)
|
||||
assert stream.has_incoming_audio_track()
|
||||
assert answer_session is not None
|
||||
assert answer_session.audio_recv_requested
|
||||
assert answer_session.audio_output is not None
|
||||
finally:
|
||||
await stream.stop()
|
||||
if answer_session is not None:
|
||||
await answer_session.stop_async()
|
||||
@@ -1,19 +1,14 @@
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import time
|
||||
# for aiortc and its dependencies
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
|
||||
|
||||
from aiortc import RTCDataChannel
|
||||
from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
|
||||
import capnp
|
||||
from iqpilot.cereal import messaging, log
|
||||
|
||||
from iqpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
|
||||
from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
|
||||
from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
|
||||
from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
|
||||
from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
|
||||
from iqpilot.system.webrtc.rtc.tracks import VIDEO_TIME_BASE
|
||||
|
||||
|
||||
class TestStreamSession:
|
||||
@@ -23,6 +18,7 @@ class TestStreamSession:
|
||||
def teardown_method(self):
|
||||
self.loop.stop()
|
||||
self.loop.close()
|
||||
gc.collect()
|
||||
|
||||
def test_outgoing_proxy(self, mocker):
|
||||
test_msg = log.Event.new_message()
|
||||
@@ -32,12 +28,12 @@ class TestStreamSession:
|
||||
expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
|
||||
expected_json = json.dumps(expected_dict).encode()
|
||||
|
||||
channel = mocker.Mock(spec=RTCDataChannel)
|
||||
mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
|
||||
def mocked_update(t):
|
||||
mocked_submaster.update_msgs(0, [test_msg])
|
||||
|
||||
mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
|
||||
channel = mocker.Mock()
|
||||
mocked_submaster = mocker.MagicMock()
|
||||
mocked_submaster.updated = {"customReservedRawData0": True}
|
||||
mocked_submaster.logMonoTime = {"customReservedRawData0": 123}
|
||||
mocked_submaster.valid = {"customReservedRawData0": True}
|
||||
mocked_submaster.__getitem__.return_value = test_msg.customReservedRawData0
|
||||
proxy = CerealOutgoingMessageProxy(mocked_submaster)
|
||||
proxy.add_channel(channel)
|
||||
|
||||
@@ -47,9 +43,9 @@ class TestStreamSession:
|
||||
|
||||
def test_incoming_proxy(self, mocker):
|
||||
tested_msgs = [
|
||||
{"type": "customReservedRawData0", "data": "test"}, # primitive
|
||||
{"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
|
||||
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
|
||||
{"type": "customReservedRawData0", "data": "test"},
|
||||
{"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]},
|
||||
{"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}},
|
||||
]
|
||||
|
||||
mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
|
||||
@@ -72,20 +68,15 @@ class TestStreamSession:
|
||||
fake_msg.livestreamDriverEncodeData.header = b"header"
|
||||
fake_msg.livestreamDriverEncodeData.data = b"\x00\x00\x00\x01\x65"
|
||||
|
||||
mocker.patch("iqpilot.system.webrtc.device.video.messaging.recv_one_or_none", return_value=fake_msg)
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.sub_sock", return_value=mocker.Mock())
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.recv_one_or_none", return_value=fake_msg)
|
||||
track = LiveStreamVideoStreamTrack("driver")
|
||||
|
||||
assert track.id.startswith("driver")
|
||||
assert track.codec_preference() == "H264"
|
||||
|
||||
for i in range(5):
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet.time_base == VIDEO_TIME_BASE
|
||||
if i == 0:
|
||||
start_ns = time.monotonic_ns()
|
||||
start_pts = packet.pts
|
||||
assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
|
||||
assert packet.size == len(b"header\x00\x00\x00\x01\x65")
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet.time_base == VIDEO_TIME_BASE
|
||||
assert packet.pts is not None
|
||||
assert packet.size == len(b"header\x00\x00\x00\x01\x65")
|
||||
|
||||
def test_input_audio_track(self, mocker):
|
||||
packet_time, rate = 0.02, 16000
|
||||
@@ -93,11 +84,27 @@ class TestStreamSession:
|
||||
fake_msg = messaging.new_message("rawAudioData")
|
||||
fake_msg.rawAudioData.data = b"\x00" * 2 * sample_count
|
||||
fake_msg.rawAudioData.sampleRate = rate
|
||||
mocker.patch("iqpilot.system.webrtc.device.audio.messaging.recv_one_or_none", return_value=fake_msg)
|
||||
track = AudioInputStreamTrack(rate=rate)
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.sub_sock", return_value=mocker.Mock())
|
||||
track = AudioInputOpusProducer()
|
||||
track._source_rate = rate
|
||||
mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.recv_one_or_none", return_value=fake_msg)
|
||||
|
||||
for i in range(5):
|
||||
frame = self.loop.run_until_complete(track.recv())
|
||||
assert frame.rate == rate
|
||||
assert frame.samples == sample_count
|
||||
assert frame.pts == i * sample_count
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet is not None
|
||||
payload, pts = packet
|
||||
assert payload
|
||||
assert pts >= 0
|
||||
|
||||
def test_debug_video_track(self):
|
||||
track = DebugVideoStreamTrack("road")
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet.size > 0
|
||||
assert packet.pts == 0
|
||||
|
||||
def test_debug_audio_track(self):
|
||||
track = DebugAudioOpusProducer()
|
||||
packet = self.loop.run_until_complete(track.recv())
|
||||
assert packet is not None
|
||||
payload, pts = packet
|
||||
assert payload
|
||||
assert pts == 0
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import asyncio
|
||||
import json
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from iqpilot.cereal import log, messaging
|
||||
from iqpilot.system.webrtc.ui_stream import (
|
||||
UI_STREAM_SERVICES,
|
||||
@@ -270,11 +273,13 @@ class TestUIStreamFrame:
|
||||
|
||||
|
||||
class TestSessionWiring:
|
||||
def test_set_ui_stream_control_message(self):
|
||||
import asyncio
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_ui_stream_control_message(self, mocker):
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
from iqpilot.system.webrtc.webrtcd import StreamSession
|
||||
from iqpilot.system.webrtc.session import StreamSession
|
||||
|
||||
mocker.patch("iqpilot.system.webrtc.ui_stream.messaging.SubMaster", return_value=FakeSubMaster(make_readers()))
|
||||
|
||||
session = StreamSession.__new__(StreamSession)
|
||||
session.logger = logging.getLogger("webrtcd")
|
||||
@@ -287,14 +292,11 @@ class TestSessionWiring:
|
||||
get_messaging_channel=lambda: channel,
|
||||
)
|
||||
|
||||
async def go():
|
||||
await session.message_handler(b'{"type":"setUiStream","enabled":true}')
|
||||
assert session.ui_stream_runner is not None
|
||||
await asyncio.sleep(0.05)
|
||||
await session.message_handler(b'{"type":"setUiStream","enabled":false}')
|
||||
assert session.ui_stream_runner is None
|
||||
|
||||
asyncio.run(go())
|
||||
session.message_handler(b'{"type":"setUiStream","enabled":true}')
|
||||
assert session.ui_stream_runner is not None
|
||||
await asyncio.sleep(0.05)
|
||||
session.message_handler(b'{"type":"setUiStream","enabled":false}')
|
||||
assert session.ui_stream_runner is None
|
||||
|
||||
assert len(channel.sent) >= 1
|
||||
frame = json.loads(channel.sent[0])
|
||||
|
||||
@@ -1,16 +1,33 @@
|
||||
import pytest
|
||||
import asyncio
|
||||
import json
|
||||
# for aiortc and its dependencies
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
|
||||
from types import SimpleNamespace
|
||||
|
||||
from iqpilot.system.webrtc.webrtcd import get_stream
|
||||
|
||||
import aiortc
|
||||
from teleoprtc import WebRTCOfferBuilder
|
||||
from parameterized import parameterized_class
|
||||
import pytest
|
||||
|
||||
from iqpilot.system.webrtc.webrtcd import add_ice, get_stream
|
||||
|
||||
|
||||
class FakeSession:
|
||||
instances = []
|
||||
|
||||
def __init__(self, identifier="session"):
|
||||
self.identifier = identifier
|
||||
self.started = False
|
||||
self.stopped = False
|
||||
self.candidates = []
|
||||
self.instances.append(self)
|
||||
|
||||
async def get_answer(self):
|
||||
return SimpleNamespace(sdp="answer", type="answer")
|
||||
|
||||
def start(self):
|
||||
self.started = True
|
||||
|
||||
async def stop_async(self):
|
||||
self.stopped = True
|
||||
|
||||
async def add_ice_candidate(self, candidate):
|
||||
self.candidates.append(candidate)
|
||||
|
||||
|
||||
@parameterized_class(("in_services", "out_services"), [
|
||||
@@ -21,44 +38,38 @@ from parameterized import parameterized_class
|
||||
])
|
||||
@pytest.mark.asyncio
|
||||
class TestWebrtcdProc:
|
||||
async def assertCompletesWithTimeout(self, awaitable, timeout=10):
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
await awaitable
|
||||
except TimeoutError:
|
||||
pytest.fail("Timeout while waiting for awaitable to complete")
|
||||
|
||||
async def test_webrtcd(self, mocker):
|
||||
mock_request = mocker.MagicMock()
|
||||
async def connect(offer):
|
||||
body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
|
||||
mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
|
||||
response = await get_stream(mock_request)
|
||||
response_json = json.loads(response.text)
|
||||
return aiortc.RTCSessionDescription(**response_json)
|
||||
session = FakeSession()
|
||||
mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
|
||||
request = mocker.MagicMock()
|
||||
request.app = {"streams": {}, "debug": False}
|
||||
request.json = mocker.AsyncMock(return_value={
|
||||
"sdp": "offer",
|
||||
"cameras": ["road"],
|
||||
"bridge_services_in": self.in_services,
|
||||
"bridge_services_out": self.out_services,
|
||||
})
|
||||
|
||||
builder = WebRTCOfferBuilder(connect, ice_servers=[])
|
||||
builder.offer_to_receive_video_stream("road")
|
||||
builder.offer_to_receive_audio_stream()
|
||||
if len(self.in_services) > 0 or len(self.out_services) > 0:
|
||||
builder.add_messaging()
|
||||
response = await get_stream(request)
|
||||
|
||||
stream = builder.stream()
|
||||
assert response.status == 200
|
||||
assert json.loads(response.text) == {"sdp": "answer", "type": "answer"}
|
||||
assert request.app["streams"] == {session.identifier: session}
|
||||
assert session.started
|
||||
|
||||
await self.assertCompletesWithTimeout(stream.start())
|
||||
await self.assertCompletesWithTimeout(stream.wait_for_connection())
|
||||
async def test_replaces_session_and_routes_ice(self, mocker):
|
||||
previous = FakeSession("previous")
|
||||
session = FakeSession("current")
|
||||
mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
|
||||
request = mocker.MagicMock()
|
||||
request.app = {"streams": {previous.identifier: previous}, "debug": False}
|
||||
request.json = mocker.AsyncMock(return_value={"sdp": "offer", "cameras": ["road"]})
|
||||
|
||||
assert stream.has_incoming_video_track("road")
|
||||
assert stream.has_incoming_audio_track()
|
||||
assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
|
||||
response = await get_stream(request)
|
||||
|
||||
video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
|
||||
await self.assertCompletesWithTimeout(video_track.recv())
|
||||
await self.assertCompletesWithTimeout(audio_track.recv())
|
||||
|
||||
await self.assertCompletesWithTimeout(stream.stop())
|
||||
|
||||
# cleanup, very implementation specific, test may break if it changes
|
||||
assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
|
||||
_, session = mock_request.app["streams"].__setitem__.call_args.args
|
||||
await self.assertCompletesWithTimeout(session.post_run_cleanup())
|
||||
assert response.status == 200
|
||||
assert previous.stopped
|
||||
request.json = mocker.AsyncMock(return_value={"candidate": {"candidate": "candidate:1"}})
|
||||
ice_response = await add_ice(request)
|
||||
assert ice_response.status == 200
|
||||
assert session.candidates == [{"candidate": "candidate:1"}]
|
||||
|
||||
@@ -3,52 +3,27 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
from iqpilot.common.params import Params
|
||||
from typing import Any
|
||||
|
||||
import warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
warnings.filterwarnings("ignore", category=RuntimeWarning)
|
||||
|
||||
import capnp
|
||||
import aiortc.rtcrtpsender
|
||||
from aiohttp import web
|
||||
from aiortc.rtp import RTCP_PSFB_APP, RtcpPsfbPacket, unpack_remb_fci
|
||||
if TYPE_CHECKING:
|
||||
from aiortc.rtcdatachannel import RTCDataChannel
|
||||
|
||||
from iqpilot.system.webrtc.schema import generate_field
|
||||
from iqpilot.cereal import messaging, log
|
||||
|
||||
|
||||
_handle_rtcp_packet = aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet
|
||||
|
||||
|
||||
async def _handle_rtcp_packet_with_remb(self, packet):
|
||||
if isinstance(packet, RtcpPsfbPacket) and packet.fmt == RTCP_PSFB_APP:
|
||||
try:
|
||||
bitrate, ssrcs = unpack_remb_fci(packet.fci)
|
||||
if getattr(self, "_ssrc", None) in ssrcs:
|
||||
self._remb_bitrate = bitrate
|
||||
except ValueError:
|
||||
pass
|
||||
return await _handle_rtcp_packet(self, packet)
|
||||
|
||||
|
||||
aiortc.rtcrtpsender.RTCRtpSender._handle_rtcp_packet = _handle_rtcp_packet_with_remb
|
||||
|
||||
|
||||
class CerealOutgoingMessageProxy:
|
||||
def __init__(self, sm: messaging.SubMaster):
|
||||
self.sm = sm
|
||||
self.channels: list[RTCDataChannel] = []
|
||||
self.channels: list[Any] = []
|
||||
|
||||
def add_channel(self, channel: 'RTCDataChannel'):
|
||||
def add_channel(self, channel: Any):
|
||||
self.channels.append(channel)
|
||||
|
||||
def to_json(self, msg_content: Any):
|
||||
@@ -114,43 +89,6 @@ class AsyncTaskRunner:
|
||||
self.task = None
|
||||
|
||||
|
||||
class IncomingAudioCerealProxy(AsyncTaskRunner):
|
||||
def __init__(self, track: Any):
|
||||
super().__init__()
|
||||
from av.audio.resampler import AudioResampler
|
||||
from iqpilot.selfdrive.ui.soundd import SAMPLE_RATE as SOUND_SAMPLE_RATE
|
||||
from iqpilot.system.webrtc.device.audio import WEBRTC_AUDIO_SERVICE
|
||||
|
||||
self.track = track
|
||||
self.service = WEBRTC_AUDIO_SERVICE
|
||||
self.pm = messaging.PubMaster([self.service])
|
||||
self.resampler = AudioResampler(format="s16", layout="mono", rate=SOUND_SAMPLE_RATE)
|
||||
|
||||
def _publish(self, frame: Any) -> None:
|
||||
data = frame.to_ndarray().tobytes()
|
||||
if not data:
|
||||
return
|
||||
|
||||
msg = messaging.new_message(self.service, valid=True)
|
||||
msg.webrtcAudioData.data = data
|
||||
msg.webrtcAudioData.sampleRate = frame.sample_rate
|
||||
self.pm.send(self.service, msg)
|
||||
|
||||
async def run(self):
|
||||
from aiortc.mediastreams import MediaStreamError
|
||||
|
||||
while True:
|
||||
try:
|
||||
frame = await self.track.recv()
|
||||
for resampled_frame in self.resampler.resample(frame):
|
||||
self._publish(resampled_frame)
|
||||
except MediaStreamError:
|
||||
break
|
||||
except Exception:
|
||||
self.logger.exception("Incoming audio cereal proxy failure")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
|
||||
class CerealProxyRunner:
|
||||
def __init__(self, proxy: CerealOutgoingMessageProxy):
|
||||
self.proxy = proxy
|
||||
@@ -169,110 +107,14 @@ class CerealProxyRunner:
|
||||
self.task = None
|
||||
|
||||
async def run(self):
|
||||
from aiortc.exceptions import InvalidStateError
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.proxy.update()
|
||||
except InvalidStateError:
|
||||
self.logger.warning("Cereal outgoing proxy invalid state (connection closed)")
|
||||
break
|
||||
except Exception:
|
||||
self.logger.exception("Cereal outgoing proxy failure")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
class LivestreamBitrateController:
|
||||
"""Adaptive bitrate for the livestream encoder using browser REMB feedback."""
|
||||
|
||||
# Match comma's rung choices more closely. A steadier capped stream tends to look better than
|
||||
# an occasionally-higher bitrate stream that induces queueing, jitter, and frame pacing swings.
|
||||
bitrates = [500_000, 1_500_000, int(os.environ.get("STREAM_BITRATE", 5_000_000))]
|
||||
label_to_bitrate = {"low": bitrates[0], "med": bitrates[1], "high": bitrates[-1]}
|
||||
|
||||
sample_interval = 1.0
|
||||
lower_factor = 0.9
|
||||
probe_after = 10
|
||||
settle_samples = 3
|
||||
|
||||
def __init__(self, peer_connection: Any):
|
||||
self.pc = peer_connection
|
||||
self.params = Params()
|
||||
self.task: asyncio.Task | None = None
|
||||
|
||||
# Start conservative and probe UP only when REMB proves headroom. Previously this started at
|
||||
# the top rung (5 Mbps); with no REMB feedback (e.g. transport-cc-only receivers, or a flaky
|
||||
# uplink that never delivers RTCP), _bandwidth_estimate() returns None and run() hits
|
||||
# `if estimate is None: continue` — so the level never moves and the encoder stays pinned at
|
||||
# 5 Mbps, flooding a marginal cellular uplink until webrtcd's send buffer balloons and trips the
|
||||
# device's lowMemory soft-disable. The med rung is carriable on typical cellular; healthy links
|
||||
# with working REMB still probe up to high within ~probe_after seconds.
|
||||
self.level = min(1, len(self.bitrates) - 1)
|
||||
self.stable = 0
|
||||
self.settle = 0
|
||||
self._auto = True
|
||||
self.current_bitrate = self.bitrates[self.level]
|
||||
self._publish(self.bitrates[self.level])
|
||||
|
||||
def start(self):
|
||||
if self.task is None:
|
||||
self.task = asyncio.create_task(self.run())
|
||||
|
||||
def stop(self):
|
||||
if self.task is not None and not self.task.done():
|
||||
self.task.cancel()
|
||||
self.task = None
|
||||
|
||||
async def run(self):
|
||||
while True:
|
||||
await asyncio.sleep(self.sample_interval)
|
||||
if not self._auto:
|
||||
continue
|
||||
estimate = self._bandwidth_estimate()
|
||||
if estimate is None:
|
||||
continue
|
||||
|
||||
if self.settle > 0:
|
||||
self.settle -= 1
|
||||
continue
|
||||
|
||||
if estimate < self.bitrates[self.level] * self.lower_factor:
|
||||
while self.level > 0 and estimate < self.bitrates[self.level] * self.lower_factor:
|
||||
self.level -= 1
|
||||
self.stable = 0
|
||||
self._publish(self.bitrates[self.level])
|
||||
elif self.level < len(self.bitrates) - 1:
|
||||
self.stable += 1
|
||||
if self.stable >= self.probe_after:
|
||||
self.level += 1
|
||||
self.stable = 0
|
||||
self.settle = self.settle_samples
|
||||
self._publish(self.bitrates[self.level])
|
||||
else:
|
||||
self.stable = 0
|
||||
|
||||
def _bandwidth_estimate(self) -> int | None:
|
||||
estimate = None
|
||||
for sender in self.pc.getSenders():
|
||||
bitrate = getattr(sender, "_remb_bitrate", None)
|
||||
if bitrate is not None:
|
||||
estimate = bitrate if estimate is None else min(estimate, bitrate)
|
||||
return estimate
|
||||
|
||||
def set_quality(self, quality: str):
|
||||
if quality in self.label_to_bitrate:
|
||||
self._auto = False
|
||||
self._publish(self.label_to_bitrate[quality])
|
||||
elif quality == "auto":
|
||||
self._auto = True
|
||||
|
||||
def _publish(self, bitrate: int):
|
||||
# Param is registered as INT — must pass a Python int, not str. Passing str throws
|
||||
# TypeError in Params.put (type mismatch) and crashes StreamSession.__init__ → HTTP 500.
|
||||
self.current_bitrate = int(bitrate)
|
||||
self.params.put("LivestreamEncoderBitrate", int(bitrate))
|
||||
|
||||
|
||||
class DynamicPubMaster(messaging.PubMaster):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -285,239 +127,11 @@ class DynamicPubMaster(messaging.PubMaster):
|
||||
self.sock[service] = messaging.pub_sock(service)
|
||||
|
||||
|
||||
class StreamSession:
|
||||
shared_pub_master = DynamicPubMaster([])
|
||||
|
||||
def __init__(self, sdp: str, cameras: list[str], incoming_services: list[str], outgoing_services: list[str],
|
||||
ice_servers: list[dict[str, Any]] | None = None, debug_mode: bool = False, ui_stream: bool = False):
|
||||
from aiortc.mediastreams import VideoStreamTrack, AudioStreamTrack
|
||||
from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
|
||||
from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
|
||||
from teleoprtc import WebRTCAnswerBuilder
|
||||
from teleoprtc.info import parse_info_from_offer
|
||||
|
||||
config = parse_info_from_offer(sdp)
|
||||
builder = WebRTCAnswerBuilder(sdp, ice_servers=ice_servers or [])
|
||||
|
||||
assert len(cameras) == config.n_expected_camera_tracks, "Incoming stream has misconfigured number of video tracks"
|
||||
self.video_tracks: list[LiveStreamVideoStreamTrack] = []
|
||||
for cam in cameras:
|
||||
track = LiveStreamVideoStreamTrack(cam) if not debug_mode else VideoStreamTrack()
|
||||
if isinstance(track, LiveStreamVideoStreamTrack):
|
||||
self.video_tracks.append(track)
|
||||
builder.add_video_stream(cam, track)
|
||||
# Audio init may fail if openpilot is using the audio subsystem - skip gracefully
|
||||
if config.expected_audio_track:
|
||||
try:
|
||||
self.audio_input_track = AudioInputStreamTrack() if not debug_mode else AudioStreamTrack()
|
||||
builder.add_audio_stream(self.audio_input_track)
|
||||
self.audio_send_enabled = True
|
||||
except Exception as e:
|
||||
logging.warning(f"Could not init audio input (audio in use?): {e}")
|
||||
if config.incoming_audio_track:
|
||||
builder.offer_to_receive_audio_stream()
|
||||
|
||||
self.stream = builder.stream()
|
||||
self.identifier = str(uuid.uuid4())
|
||||
|
||||
self.incoming_bridge: CerealIncomingMessageProxy | None = None
|
||||
self.incoming_bridge_services = incoming_services
|
||||
self.outgoing_bridge: CerealOutgoingMessageProxy | None = None
|
||||
self.outgoing_bridge_runner: CerealProxyRunner | None = None
|
||||
if len(incoming_services) > 0:
|
||||
self.incoming_bridge = CerealIncomingMessageProxy(self.shared_pub_master)
|
||||
if len(outgoing_services) > 0:
|
||||
self.outgoing_bridge = CerealOutgoingMessageProxy(messaging.SubMaster(outgoing_services))
|
||||
self.outgoing_bridge_runner = CerealProxyRunner(self.outgoing_bridge)
|
||||
|
||||
self.ui_stream_requested = ui_stream
|
||||
self.ui_stream_runner: CerealProxyRunner | None = None
|
||||
|
||||
self.incoming_audio_proxy: IncomingAudioCerealProxy | None = None
|
||||
self.audio_input_track: AudioInputStreamTrack | AudioStreamTrack | None = None
|
||||
self.audio_send_enabled = False
|
||||
self.audio_recv_requested = bool(config.incoming_audio_track)
|
||||
self.audio_send_requested = bool(config.expected_audio_track)
|
||||
self.run_task: asyncio.Task | None = None
|
||||
# Adaptive bitrate controller for the livestream encoder (no-op in debug mode).
|
||||
self.bitrate_controller: LivestreamBitrateController | None = None
|
||||
if not debug_mode and len(self.video_tracks) > 0:
|
||||
self.bitrate_controller = LivestreamBitrateController(self.stream.peer_connection)
|
||||
self.logger = logging.getLogger("webrtcd")
|
||||
self.logger.info("New stream session (%s), cameras %s, audio in %s out %s, incoming services %s, outgoing services %s",
|
||||
self.identifier, cameras, config.incoming_audio_track, config.expected_audio_track, incoming_services, outgoing_services)
|
||||
|
||||
def start(self):
|
||||
self.run_task = asyncio.create_task(self.run())
|
||||
|
||||
async def stop_async(self):
|
||||
if self.run_task is not None and not self.run_task.done():
|
||||
self.run_task.cancel()
|
||||
try:
|
||||
await self.run_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
self.logger.exception("Stream session stop task failure")
|
||||
self.run_task = None
|
||||
await self.post_run_cleanup()
|
||||
|
||||
def stop(self):
|
||||
# Backwards-compatible sync wrapper. Prefer `await stop_async()` from async contexts.
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
# If we're already in an event loop, schedule async shutdown and return.
|
||||
loop.create_task(self.stop_async())
|
||||
return
|
||||
except RuntimeError:
|
||||
pass
|
||||
asyncio.run(self.stop_async())
|
||||
|
||||
async def get_answer(self):
|
||||
return await self.stream.start()
|
||||
|
||||
async def message_handler(self, message: bytes):
|
||||
# Control messages are handled in-process and don't require an incoming cereal bridge.
|
||||
try:
|
||||
payload = json.loads(message) if isinstance(message, (bytes, str)) else None
|
||||
except (ValueError, TypeError):
|
||||
payload = None
|
||||
if isinstance(payload, dict) and payload.get("type") == "timingSei":
|
||||
enabled = bool(payload.get("enabled", False))
|
||||
for track in self.video_tracks:
|
||||
track.timing_sei_enabled = enabled
|
||||
self.logger.info("timing SEI %s", "enabled" if enabled else "disabled")
|
||||
return
|
||||
if isinstance(payload, dict) and payload.get("type") == "setQuality":
|
||||
if self.bitrate_controller is not None:
|
||||
quality = str(payload.get("quality", "auto"))
|
||||
self.bitrate_controller.set_quality(quality)
|
||||
self.logger.info("livestream quality set to %s", quality)
|
||||
return
|
||||
if isinstance(payload, dict) and payload.get("type") == "setAudioEnabled":
|
||||
enabled = bool(payload.get("enabled", True))
|
||||
if hasattr(self.audio_input_track, "enable"):
|
||||
self.audio_input_track.enable(enabled)
|
||||
self.audio_send_enabled = enabled
|
||||
self.logger.info("livestream audio send %s", "enabled" if enabled else "disabled")
|
||||
return
|
||||
if isinstance(payload, dict) and payload.get("type") == "setUiStream":
|
||||
enabled = bool(payload.get("enabled", False))
|
||||
self.set_ui_stream(enabled)
|
||||
self.logger.info("ui stream %s", "enabled" if enabled else "disabled")
|
||||
return
|
||||
if isinstance(payload, dict) and payload.get("type") == "switchCamera":
|
||||
camera = str(payload.get("camera", ""))
|
||||
# Single-track model: repoint the (one) video track at the requested camera.
|
||||
for track in self.video_tracks:
|
||||
track.switch_camera(camera)
|
||||
return
|
||||
|
||||
if self.incoming_bridge is None:
|
||||
return
|
||||
try:
|
||||
self.incoming_bridge.send(message)
|
||||
except Exception:
|
||||
self.logger.exception("Cereal incoming proxy failure")
|
||||
|
||||
def set_ui_stream(self, enabled: bool):
|
||||
if enabled:
|
||||
if self.ui_stream_runner is not None or not self.stream.has_messaging_channel():
|
||||
return
|
||||
from iqpilot.system.webrtc.ui_stream import UIStreamMessageProxy
|
||||
bitrate_getter = None
|
||||
if self.bitrate_controller is not None:
|
||||
controller = self.bitrate_controller
|
||||
|
||||
def bitrate_getter():
|
||||
return controller.current_bitrate
|
||||
proxy = UIStreamMessageProxy(bitrate_getter=bitrate_getter)
|
||||
proxy.add_channel(self.stream.get_messaging_channel())
|
||||
self.ui_stream_runner = CerealProxyRunner(proxy)
|
||||
self.ui_stream_runner.start()
|
||||
elif self.ui_stream_runner is not None:
|
||||
self.ui_stream_runner.stop()
|
||||
self.ui_stream_runner = None
|
||||
|
||||
async def add_ice_candidate(self, cand: Any):
|
||||
"""Add a trickled ICE candidate from the client to the live peer connection."""
|
||||
if not isinstance(cand, dict):
|
||||
return
|
||||
cand_str = cand.get("candidate") or ""
|
||||
if not cand_str:
|
||||
return # end-of-candidates marker; aiortc needs no explicit signal
|
||||
try:
|
||||
from aiortc.sdp import candidate_from_sdp
|
||||
sdp_str = cand_str.split(":", 1)[-1] if cand_str.startswith("candidate:") else cand_str
|
||||
ice = candidate_from_sdp(sdp_str)
|
||||
ice.sdpMid = cand.get("sdpMid")
|
||||
ice.sdpMLineIndex = cand.get("sdpMLineIndex")
|
||||
await self.stream.peer_connection.addIceCandidate(ice)
|
||||
except Exception:
|
||||
self.logger.exception("Failed to add ICE candidate")
|
||||
|
||||
async def run(self):
|
||||
try:
|
||||
await self.stream.wait_for_connection()
|
||||
if self.stream.has_messaging_channel():
|
||||
# Always install the handler so control messages (e.g. timing SEI toggle) work
|
||||
# even when no incoming cereal bridge service was requested.
|
||||
self.stream.set_message_handler(self.message_handler)
|
||||
if self.incoming_bridge is not None:
|
||||
await self.shared_pub_master.add_services_if_needed(self.incoming_bridge_services)
|
||||
if self.outgoing_bridge_runner is not None:
|
||||
channel = self.stream.get_messaging_channel()
|
||||
self.outgoing_bridge_runner.proxy.add_channel(channel)
|
||||
self.outgoing_bridge_runner.start()
|
||||
if self.ui_stream_requested:
|
||||
self.set_ui_stream(True)
|
||||
if self.audio_recv_requested and self.stream.has_incoming_audio_track():
|
||||
track = self.stream.get_incoming_audio_track(buffered=False)
|
||||
self.incoming_audio_proxy = IncomingAudioCerealProxy(track)
|
||||
self.incoming_audio_proxy.start()
|
||||
self.logger.info("Stream session (%s) incoming audio proxy started", self.identifier)
|
||||
else:
|
||||
self.logger.info("Stream session (%s) no incoming audio track from client", self.identifier)
|
||||
if self.bitrate_controller is not None:
|
||||
self.bitrate_controller.start()
|
||||
self.logger.info(
|
||||
"Stream session (%s) audio state send_requested=%s send_enabled=%s recv_requested=%s recv_active=%s",
|
||||
self.identifier,
|
||||
self.audio_send_requested,
|
||||
self.audio_send_enabled,
|
||||
self.audio_recv_requested,
|
||||
self.incoming_audio_proxy is not None,
|
||||
)
|
||||
self.logger.info("Stream session (%s) connected", self.identifier)
|
||||
|
||||
await self.stream.wait_for_disconnection()
|
||||
await self.post_run_cleanup()
|
||||
|
||||
self.logger.info("Stream session (%s) ended", self.identifier)
|
||||
except Exception:
|
||||
self.logger.exception("Stream session failure")
|
||||
|
||||
async def post_run_cleanup(self):
|
||||
if self.bitrate_controller is not None:
|
||||
self.bitrate_controller.stop()
|
||||
await self.stream.stop()
|
||||
if self.ui_stream_runner is not None:
|
||||
self.ui_stream_runner.stop()
|
||||
self.ui_stream_runner = None
|
||||
if self.outgoing_bridge is not None:
|
||||
self.outgoing_bridge_runner.stop()
|
||||
if self.incoming_audio_proxy is not None:
|
||||
await self.incoming_audio_proxy.stop()
|
||||
|
||||
|
||||
def _is_retryable_stream_error(e: Exception) -> bool:
|
||||
# Transient failures seen during answer generation: SDP/candidate parse issues
|
||||
# (typically browser mDNS .local host candidates aiortc can't resolve) and
|
||||
# socket-level hiccups while gathering. Anything else is a real error.
|
||||
return isinstance(e, (ValueError, OSError))
|
||||
|
||||
|
||||
async def _cleanup_failed_session(session: 'StreamSession | None', logger: logging.Logger) -> None:
|
||||
async def _cleanup_failed_session(session: Any | None, logger: logging.Logger) -> None:
|
||||
if session is None:
|
||||
return
|
||||
try:
|
||||
@@ -543,15 +157,7 @@ class StreamRequestBody:
|
||||
|
||||
|
||||
def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: bool):
|
||||
if Params().get_bool("Konn3ktLibdatachannelWebRTC"):
|
||||
try:
|
||||
from iqpilot.system.webrtc.webrtcd_ldc import StreamSessionLibdatachannel
|
||||
return StreamSessionLibdatachannel(
|
||||
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
|
||||
ui_stream=body.ui_stream,
|
||||
)
|
||||
except Exception:
|
||||
logging.getLogger("webrtcd").exception("libdatachannel unavailable; falling back to aiortc")
|
||||
from iqpilot.system.webrtc.session import StreamSession
|
||||
return StreamSession(
|
||||
offer_sdp, body.cameras, body.bridge_services_in, body.bridge_services_out, body.iceServers, debug_mode,
|
||||
ui_stream=body.ui_stream,
|
||||
@@ -561,7 +167,7 @@ def _new_stream_session(offer_sdp: str, body: StreamRequestBody, debug_mode: boo
|
||||
async def get_stream(request: 'web.Request'):
|
||||
stream_dict, debug_mode = request.app['streams'], request.app['debug']
|
||||
logger = logging.getLogger("webrtcd")
|
||||
session: StreamSession | None = None
|
||||
session: Any | None = None
|
||||
try:
|
||||
raw_body = await request.json()
|
||||
body = StreamRequestBody(**raw_body)
|
||||
|
||||
Reference in New Issue
Block a user