IQ.Pilot Release Commit @ 3fe374f

iqpilot/system/webrtc/tests/test_native_session.py

@@ -0,0 +1,96 @@
+from types import SimpleNamespace
+
+import pytest
+
+from iqpilot.system.webrtc import session as session_module
+
+
+class FakeStream:
+  def get_receiver_report_stats(self):
+    return {}
+
+
+class FakeBuilder:
+  instance = None
+
+  def __init__(self, sdp, bind_address=None, ice_servers=None):
+    self.sdp = sdp
+    self.bind_address = bind_address
+    self.ice_servers = ice_servers
+    self.video = []
+    self.audio = []
+    self.receive_audio = False
+    self.result = FakeStream()
+    FakeBuilder.instance = self
+
+  def add_video_stream(self, camera, track):
+    self.video.append((camera, track))
+
+  def add_audio_stream(self, track):
+    self.audio.append(track)
+
+  def offer_to_receive_audio_stream(self):
+    self.receive_audio = True
+
+  def stream(self):
+    return self.result
+
+
+class FakeVideoTrack:
+  def __init__(self, camera):
+    self.camera = camera
+    self.timing_sei_enabled = False
+    self.switched = None
+
+  def switch_camera(self, camera):
+    self.switched = camera
+
+
+class FakeAudioProducer:
+  def __init__(self):
+    self.enabled = True
+
+  def enable(self, enabled):
+    self.enabled = enabled
+
+
+@pytest.fixture
+def native_session(mocker):
+  config = SimpleNamespace(
+    n_expected_camera_tracks=1,
+    expected_audio_track=True,
+    incoming_audio_track=True,
+    incoming_datachannel=True,
+  )
+  mocker.patch("iqpilot.system.webrtc.rtc.info.parse_info_from_offer", return_value=config)
+  mocker.patch("iqpilot.system.webrtc.rtc.builder.WebRTCAnswerBuilder", FakeBuilder)
+  mocker.patch("iqpilot.system.webrtc.device.native_video.LiveStreamVideoStreamTrack", FakeVideoTrack)
+  mocker.patch("iqpilot.system.webrtc.device.native_audio.AudioInputOpusProducer", FakeAudioProducer)
+  mocker.patch.object(session_module, "_default_route_ip", return_value="192.0.2.1")
+  mocker.patch.object(session_module, "Params", return_value=mocker.Mock())
+  return session_module.StreamSession(
+    "offer", ["road"], [], [], [{"urls": "turn:example.com"}], ui_stream=False,
+  )
+
+
+def test_native_session_builds_duplex_audio(native_session):
+  builder = FakeBuilder.instance
+  assert builder is not None
+  assert builder.bind_address == "192.0.2.1"
+  assert builder.ice_servers == [{"urls": "turn:example.com"}]
+  assert [camera for camera, _ in builder.video] == ["road"]
+  assert builder.audio == [native_session.audio_output]
+  assert builder.receive_audio
+  assert native_session.audio_recv_requested
+
+
+def test_native_session_controls(native_session, mocker):
+  native_session.bitrate_controller = mocker.Mock()
+  native_session.message_handler('{"type":"timingSei","enabled":true}')
+  assert native_session.video_tracks[0].timing_sei_enabled
+  native_session.message_handler('{"type":"switchCamera","camera":"driver"}')
+  assert native_session.video_tracks[0].switched == "driver"
+  native_session.message_handler('{"type":"setAudioEnabled","enabled":false}')
+  assert not native_session.audio_output.enabled
+  native_session.message_handler('{"type":"setQuality","quality":"low"}')
+  native_session.bitrate_controller.set_quality.assert_called_once_with("low")

iqpilot/system/webrtc/tests/test_rtc.py

@@ -0,0 +1,116 @@
+import pytest
+
+from iqpilot.system.webrtc.rtc.info import parse_info_from_offer
+from iqpilot.system.webrtc.rtc.stream import WebRTCBaseStream
+from iqpilot.system.webrtc.rtc.tracks import TiciTrackWrapper, TiciVideoStreamTrack, parse_video_track_id, video_track_id
+
+
+def sdp_with_media(media):
+  mids = " ".join(str(i) for i in range(len(media)))
+  sections = []
+  for index, (kind, direction) in enumerate(media):
+    if kind == "video":
+      payload, mapping = "96", "H264/90000"
+      protocol = "UDP/TLS/RTP/SAVPF"
+    elif kind == "audio":
+      payload, mapping = "111", "opus/48000/2"
+      protocol = "UDP/TLS/RTP/SAVPF"
+    else:
+      sections.extend([
+        "m=application 9 UDP/DTLS/SCTP webrtc-datachannel",
+        "c=IN IP4 0.0.0.0",
+        f"a=mid:{index}",
+        "a=sctp-port:5000",
+      ])
+      continue
+    sections.extend([
+      f"m={kind} 9 {protocol} {payload}",
+      "c=IN IP4 0.0.0.0",
+      f"a=mid:{index}",
+      f"a={direction}",
+      f"a=rtpmap:{payload} {mapping}",
+      "a=rtcp-mux",
+    ])
+  lines = [
+    "v=0",
+    "o=- 1 1 IN IP4 0.0.0.0",
+    "s=-",
+    "t=0 0",
+    f"a=group:BUNDLE {mids}",
+    *sections,
+  ]
+  return "\r\n".join(lines) + "\r\n"
+
+
+@pytest.mark.parametrize("direction,expected_outgoing,expected_incoming", [
+  ("recvonly", True, False),
+  ("sendonly", False, True),
+  ("sendrecv", True, True),
+  ("inactive", False, False),
+])
+def test_audio_directions(direction, expected_outgoing, expected_incoming):
+  info = parse_info_from_offer(sdp_with_media([("audio", direction)]))
+  assert info.expected_audio_track == expected_outgoing
+  assert info.incoming_audio_track == expected_incoming
+
+
+def test_video_and_data_channel_metadata():
+  info = parse_info_from_offer(sdp_with_media([
+    ("video", "recvonly"),
+    ("video", "sendrecv"),
+    ("application", "sendrecv"),
+  ]))
+  assert info.n_expected_camera_tracks == 2
+  assert info.incoming_datachannel
+
+
+def test_explicit_empty_ice_servers_disable_defaults():
+  assert WebRTCBaseStream._make_ice_servers([]) == []
+
+
+def test_default_ice_server():
+  servers = WebRTCBaseStream._make_ice_servers(None)
+  assert len(servers) == 1
+  assert servers[0].hostname == "stun.l.google.com"
+  assert servers[0].port == 19302
+
+
+def test_authenticated_ice_servers():
+  servers = WebRTCBaseStream._make_ice_servers([{
+    "urls": ["turn:relay.example.com:3478", "stun:stun.example.com:3478"],
+    "username": "user",
+    "credential": "secret",
+  }])
+  assert [(server.hostname, server.port) for server in servers] == [
+    ("relay.example.com", 3478),
+    ("stun.example.com", 3478),
+  ]
+  assert all(server.username == "user" and server.password == "secret" for server in servers)
+
+
+def test_track_id_roundtrip():
+  assert parse_video_track_id(video_track_id("driver", "track")) == ("driver", "track")
+
+
+def test_invalid_track_id():
+  with pytest.raises(ValueError):
+    parse_video_track_id("driver")
+
+
+def test_track_wrapper_preserves_camera():
+  class Track:
+    kind = "video"
+    id = "source"
+
+    async def recv(self):
+      return b"frame"
+
+  wrapper = TiciTrackWrapper("road", Track())
+  assert parse_video_track_id(wrapper.id)[0] == "road"
+  wrapper.stop()
+  assert wrapper.readyState == "ended"
+
+
+def test_track_stores_frame_period():
+  track = TiciVideoStreamTrack("wideRoad", 0.05)
+  assert track._dt == 0.05

iqpilot/system/webrtc/tests/test_rtc_integration.py

@@ -0,0 +1,79 @@
+import asyncio
+import os
+
+import pytest
+
+from iqpilot.system.webrtc.rtc import WebRTCOfferBuilder
+from iqpilot.system.webrtc.rtc.stream import RTCSessionDescription
+from iqpilot.system.webrtc.session import StreamSession
+
+
+@pytest.mark.asyncio
+async def test_native_video_audio_and_data_channel():
+  if not os.environ.get("CI"):
+    return
+
+  answer_session = None
+  video_received = asyncio.Event()
+  audio_received = asyncio.Event()
+
+  async def connect(offer):
+    nonlocal answer_session
+    answer_session = StreamSession(offer.sdp, offer.video, [], [], [], debug_mode=True)
+    answer = await answer_session.get_answer()
+    answer_session.start()
+    return RTCSessionDescription(answer.sdp, answer.type)
+
+  builder = WebRTCOfferBuilder(connect, ice_servers=[])
+  builder.offer_to_receive_video_stream("road")
+  builder.offer_to_receive_audio_stream()
+  builder.add_messaging()
+  stream = builder.stream()
+
+  try:
+    await asyncio.wait_for(stream.start(), 10)
+    stream.get_incoming_video_track("road").on_frame(lambda *_: video_received.set())
+    stream.get_incoming_audio_track().on_frame(lambda *_: audio_received.set())
+    await asyncio.wait_for(stream.wait_for_connection(), 10)
+    await asyncio.wait_for(video_received.wait(), 10)
+    await asyncio.wait_for(audio_received.wait(), 10)
+    stream.get_messaging_channel().send('{"type":"timingSei","enabled":true}')
+    await asyncio.sleep(0.1)
+    assert answer_session is not None
+    assert answer_session.video_tracks[0].timing_sei_enabled
+  finally:
+    await stream.stop()
+    if answer_session is not None:
+      await answer_session.stop_async()
+
+
+@pytest.mark.asyncio
+async def test_native_duplex_audio_negotiation():
+  if not os.environ.get("CI"):
+    return
+
+  answer_session = None
+
+  async def connect(offer):
+    nonlocal answer_session
+    sendrecv_offer = offer.sdp.replace("a=recvonly", "a=sendrecv", 1)
+    answer_session = StreamSession(sendrecv_offer, offer.video, [], [], [], debug_mode=True)
+    answer = await answer_session.get_answer()
+    answer_session.start()
+    return RTCSessionDescription(answer.sdp, answer.type)
+
+  builder = WebRTCOfferBuilder(connect, ice_servers=[])
+  builder.offer_to_receive_audio_stream()
+  stream = builder.stream()
+
+  try:
+    await asyncio.wait_for(stream.start(), 10)
+    await asyncio.wait_for(stream.wait_for_connection(), 10)
+    assert stream.has_incoming_audio_track()
+    assert answer_session is not None
+    assert answer_session.audio_recv_requested
+    assert answer_session.audio_output is not None
+  finally:
+    await stream.stop()
+    if answer_session is not None:
+      await answer_session.stop_async()

iqpilot/system/webrtc/tests/test_stream_session.py

@@ -1,19 +1,14 @@
 import asyncio
+import gc
 import json
-import time
-# for aiortc and its dependencies
-import warnings
-warnings.filterwarnings("ignore", category=DeprecationWarning)
-warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
 
-from aiortc import RTCDataChannel
-from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
 import capnp
 from iqpilot.cereal import messaging, log
 
 from iqpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
-from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
-from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
+from iqpilot.system.webrtc.device.native_audio import AudioInputOpusProducer, DebugAudioOpusProducer
+from iqpilot.system.webrtc.device.native_video import DebugVideoStreamTrack, LiveStreamVideoStreamTrack
+from iqpilot.system.webrtc.rtc.tracks import VIDEO_TIME_BASE
 
 
 class TestStreamSession:
@@ -23,6 +18,7 @@ class TestStreamSession:
   def teardown_method(self):
     self.loop.stop()
     self.loop.close()
+    gc.collect()
 
   def test_outgoing_proxy(self, mocker):
     test_msg = log.Event.new_message()
@@ -32,12 +28,12 @@ class TestStreamSession:
     expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
     expected_json = json.dumps(expected_dict).encode()
 
-    channel = mocker.Mock(spec=RTCDataChannel)
-    mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
-    def mocked_update(t):
-      mocked_submaster.update_msgs(0, [test_msg])
-
-    mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
+    channel = mocker.Mock()
+    mocked_submaster = mocker.MagicMock()
+    mocked_submaster.updated = {"customReservedRawData0": True}
+    mocked_submaster.logMonoTime = {"customReservedRawData0": 123}
+    mocked_submaster.valid = {"customReservedRawData0": True}
+    mocked_submaster.__getitem__.return_value = test_msg.customReservedRawData0
     proxy = CerealOutgoingMessageProxy(mocked_submaster)
     proxy.add_channel(channel)
 
@@ -47,9 +43,9 @@ class TestStreamSession:
 
   def test_incoming_proxy(self, mocker):
     tested_msgs = [
-      {"type": "customReservedRawData0", "data": "test"}, # primitive
-      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
-      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
+      {"type": "customReservedRawData0", "data": "test"},
+      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]},
+      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}},
     ]
 
     mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
@@ -72,20 +68,15 @@ class TestStreamSession:
     fake_msg.livestreamDriverEncodeData.header = b"header"
     fake_msg.livestreamDriverEncodeData.data = b"\x00\x00\x00\x01\x65"
 
-    mocker.patch("iqpilot.system.webrtc.device.video.messaging.recv_one_or_none", return_value=fake_msg)
+    mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.sub_sock", return_value=mocker.Mock())
+    mocker.patch("iqpilot.system.webrtc.device.native_video.messaging.recv_one_or_none", return_value=fake_msg)
     track = LiveStreamVideoStreamTrack("driver")
 
     assert track.id.startswith("driver")
-    assert track.codec_preference() == "H264"
-
-    for i in range(5):
-      packet = self.loop.run_until_complete(track.recv())
-      assert packet.time_base == VIDEO_TIME_BASE
-      if i == 0:
-        start_ns = time.monotonic_ns()
-        start_pts = packet.pts
-      assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
-      assert packet.size == len(b"header\x00\x00\x00\x01\x65")
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet.time_base == VIDEO_TIME_BASE
+    assert packet.pts is not None
+    assert packet.size == len(b"header\x00\x00\x00\x01\x65")
 
   def test_input_audio_track(self, mocker):
     packet_time, rate = 0.02, 16000
@@ -93,11 +84,27 @@ class TestStreamSession:
     fake_msg = messaging.new_message("rawAudioData")
     fake_msg.rawAudioData.data = b"\x00" * 2 * sample_count
     fake_msg.rawAudioData.sampleRate = rate
-    mocker.patch("iqpilot.system.webrtc.device.audio.messaging.recv_one_or_none", return_value=fake_msg)
-    track = AudioInputStreamTrack(rate=rate)
+    mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.sub_sock", return_value=mocker.Mock())
+    track = AudioInputOpusProducer()
+    track._source_rate = rate
+    mocker.patch("iqpilot.system.webrtc.device.native_audio.messaging.recv_one_or_none", return_value=fake_msg)
 
-    for i in range(5):
-      frame = self.loop.run_until_complete(track.recv())
-      assert frame.rate == rate
-      assert frame.samples == sample_count
-      assert frame.pts == i * sample_count
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet is not None
+    payload, pts = packet
+    assert payload
+    assert pts >= 0
+
+  def test_debug_video_track(self):
+    track = DebugVideoStreamTrack("road")
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet.size > 0
+    assert packet.pts == 0
+
+  def test_debug_audio_track(self):
+    track = DebugAudioOpusProducer()
+    packet = self.loop.run_until_complete(track.recv())
+    assert packet is not None
+    payload, pts = packet
+    assert payload
+    assert pts == 0

iqpilot/system/webrtc/tests/test_ui_stream.py

@@ -1,6 +1,9 @@
+import asyncio
 import json
 import math
 
+import pytest
+
 from iqpilot.cereal import log, messaging
 from iqpilot.system.webrtc.ui_stream import (
   UI_STREAM_SERVICES,
@@ -270,11 +273,13 @@ class TestUIStreamFrame:
 
 
 class TestSessionWiring:
-  def test_set_ui_stream_control_message(self):
-    import asyncio
+  @pytest.mark.asyncio
+  async def test_set_ui_stream_control_message(self, mocker):
     import logging
     from types import SimpleNamespace
-    from iqpilot.system.webrtc.webrtcd import StreamSession
+    from iqpilot.system.webrtc.session import StreamSession
+
+    mocker.patch("iqpilot.system.webrtc.ui_stream.messaging.SubMaster", return_value=FakeSubMaster(make_readers()))
 
     session = StreamSession.__new__(StreamSession)
     session.logger = logging.getLogger("webrtcd")
@@ -287,14 +292,11 @@ class TestSessionWiring:
       get_messaging_channel=lambda: channel,
     )
 
-    async def go():
-      await session.message_handler(b'{"type":"setUiStream","enabled":true}')
-      assert session.ui_stream_runner is not None
-      await asyncio.sleep(0.05)
-      await session.message_handler(b'{"type":"setUiStream","enabled":false}')
-      assert session.ui_stream_runner is None
-
-    asyncio.run(go())
+    session.message_handler(b'{"type":"setUiStream","enabled":true}')
+    assert session.ui_stream_runner is not None
+    await asyncio.sleep(0.05)
+    session.message_handler(b'{"type":"setUiStream","enabled":false}')
+    assert session.ui_stream_runner is None
 
     assert len(channel.sent) >= 1
     frame = json.loads(channel.sent[0])

iqpilot/system/webrtc/tests/test_webrtcd.py

@@ -1,16 +1,33 @@
-import pytest
-import asyncio
 import json
-# for aiortc and its dependencies
-import warnings
-warnings.filterwarnings("ignore", category=DeprecationWarning)
-warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+from types import SimpleNamespace
 
-from iqpilot.system.webrtc.webrtcd import get_stream
-
-import aiortc
-from teleoprtc import WebRTCOfferBuilder
 from parameterized import parameterized_class
+import pytest
+
+from iqpilot.system.webrtc.webrtcd import add_ice, get_stream
+
+
+class FakeSession:
+  instances = []
+
+  def __init__(self, identifier="session"):
+    self.identifier = identifier
+    self.started = False
+    self.stopped = False
+    self.candidates = []
+    self.instances.append(self)
+
+  async def get_answer(self):
+    return SimpleNamespace(sdp="answer", type="answer")
+
+  def start(self):
+    self.started = True
+
+  async def stop_async(self):
+    self.stopped = True
+
+  async def add_ice_candidate(self, candidate):
+    self.candidates.append(candidate)
 
 
 @parameterized_class(("in_services", "out_services"), [
@@ -21,44 +38,38 @@ from parameterized import parameterized_class
 ])
 @pytest.mark.asyncio
 class TestWebrtcdProc:
-  async def assertCompletesWithTimeout(self, awaitable, timeout=10):
-    try:
-      async with asyncio.timeout(timeout):
-        await awaitable
-    except TimeoutError:
-      pytest.fail("Timeout while waiting for awaitable to complete")
-
   async def test_webrtcd(self, mocker):
-    mock_request = mocker.MagicMock()
-    async def connect(offer):
-      body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
-      mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
-      response = await get_stream(mock_request)
-      response_json = json.loads(response.text)
-      return aiortc.RTCSessionDescription(**response_json)
+    session = FakeSession()
+    mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
+    request = mocker.MagicMock()
+    request.app = {"streams": {}, "debug": False}
+    request.json = mocker.AsyncMock(return_value={
+      "sdp": "offer",
+      "cameras": ["road"],
+      "bridge_services_in": self.in_services,
+      "bridge_services_out": self.out_services,
+    })
 
-    builder = WebRTCOfferBuilder(connect, ice_servers=[])
-    builder.offer_to_receive_video_stream("road")
-    builder.offer_to_receive_audio_stream()
-    if len(self.in_services) > 0 or len(self.out_services) > 0:
-      builder.add_messaging()
+    response = await get_stream(request)
 
-    stream = builder.stream()
+    assert response.status == 200
+    assert json.loads(response.text) == {"sdp": "answer", "type": "answer"}
+    assert request.app["streams"] == {session.identifier: session}
+    assert session.started
 
-    await self.assertCompletesWithTimeout(stream.start())
-    await self.assertCompletesWithTimeout(stream.wait_for_connection())
+  async def test_replaces_session_and_routes_ice(self, mocker):
+    previous = FakeSession("previous")
+    session = FakeSession("current")
+    mocker.patch("iqpilot.system.webrtc.webrtcd._new_stream_session", return_value=session)
+    request = mocker.MagicMock()
+    request.app = {"streams": {previous.identifier: previous}, "debug": False}
+    request.json = mocker.AsyncMock(return_value={"sdp": "offer", "cameras": ["road"]})
 
-    assert stream.has_incoming_video_track("road")
-    assert stream.has_incoming_audio_track()
-    assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
+    response = await get_stream(request)
 
-    video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
-    await self.assertCompletesWithTimeout(video_track.recv())
-    await self.assertCompletesWithTimeout(audio_track.recv())
-
-    await self.assertCompletesWithTimeout(stream.stop())
-
-    # cleanup, very implementation specific, test may break if it changes
-    assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
-    _, session = mock_request.app["streams"].__setitem__.call_args.args
-    await self.assertCompletesWithTimeout(session.post_run_cleanup())
+    assert response.status == 200
+    assert previous.stopped
+    request.json = mocker.AsyncMock(return_value={"candidate": {"candidate": "candidate:1"}})
+    ice_response = await add_ice(request)
+    assert ice_response.status == 200
+    assert session.candidates == [{"candidate": "candidate:1"}]