IQ.Pilot Release Commit @ bec7652

iqpilot/system/webrtc/tests/test_stream_session.py

@@ -0,0 +1,103 @@
+import asyncio
+import json
+import time
+# for aiortc and its dependencies
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+from aiortc import RTCDataChannel
+from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
+import capnp
+from iqpilot.cereal import messaging, log
+
+from iqpilot.system.webrtc.webrtcd import CerealOutgoingMessageProxy, CerealIncomingMessageProxy
+from iqpilot.system.webrtc.device.video import LiveStreamVideoStreamTrack
+from iqpilot.system.webrtc.device.audio import AudioInputStreamTrack
+
+
+class TestStreamSession:
+  def setup_method(self):
+    self.loop = asyncio.new_event_loop()
+
+  def teardown_method(self):
+    self.loop.stop()
+    self.loop.close()
+
+  def test_outgoing_proxy(self, mocker):
+    test_msg = log.Event.new_message()
+    test_msg.logMonoTime = 123
+    test_msg.valid = True
+    test_msg.customReservedRawData0 = b"test"
+    expected_dict = {"type": "customReservedRawData0", "logMonoTime": 123, "valid": True, "data": "test"}
+    expected_json = json.dumps(expected_dict).encode()
+
+    channel = mocker.Mock(spec=RTCDataChannel)
+    mocked_submaster = messaging.SubMaster(["customReservedRawData0"])
+    def mocked_update(t):
+      mocked_submaster.update_msgs(0, [test_msg])
+
+    mocker.patch.object(messaging.SubMaster, "update", side_effect=mocked_update)
+    proxy = CerealOutgoingMessageProxy(mocked_submaster)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    channel.send.assert_called_once_with(expected_json)
+
+  def test_incoming_proxy(self, mocker):
+    tested_msgs = [
+      {"type": "customReservedRawData0", "data": "test"}, # primitive
+      {"type": "can", "data": [{"address": 0, "dat": "", "src": 0}]}, # list
+      {"type": "testJoystick", "data": {"axes": [0, 0], "buttons": [False]}}, # dict
+    ]
+
+    mocked_pubmaster = mocker.MagicMock(spec=messaging.PubMaster)
+
+    proxy = CerealIncomingMessageProxy(mocked_pubmaster)
+
+    for msg in tested_msgs:
+      proxy.send(json.dumps(msg).encode())
+
+      mocked_pubmaster.send.assert_called_once()
+      mt, md = mocked_pubmaster.send.call_args.args
+      assert mt == msg["type"]
+      assert isinstance(md, capnp._DynamicStructBuilder)
+      assert hasattr(md, msg["type"])
+
+      mocked_pubmaster.reset_mock()
+
+  def test_livestream_track(self, mocker):
+    fake_msg = messaging.new_message("livestreamDriverEncodeData")
+    fake_msg.livestreamDriverEncodeData.header = b"header"
+    fake_msg.livestreamDriverEncodeData.data = b"\x00\x00\x00\x01\x65"
+
+    mocker.patch("iqpilot.system.webrtc.device.video.messaging.recv_one_or_none", return_value=fake_msg)
+    track = LiveStreamVideoStreamTrack("driver")
+
+    assert track.id.startswith("driver")
+    assert track.codec_preference() == "H264"
+
+    for i in range(5):
+      packet = self.loop.run_until_complete(track.recv())
+      assert packet.time_base == VIDEO_TIME_BASE
+      if i == 0:
+        start_ns = time.monotonic_ns()
+        start_pts = packet.pts
+      assert abs(i + packet.pts - (start_pts + (((time.monotonic_ns() - start_ns) * VIDEO_CLOCK_RATE) // 1_000_000_000))) < 450 #5ms
+      assert packet.size == len(b"header\x00\x00\x00\x01\x65")
+
+  def test_input_audio_track(self, mocker):
+    packet_time, rate = 0.02, 16000
+    sample_count = int(packet_time * rate)
+    fake_msg = messaging.new_message("rawAudioData")
+    fake_msg.rawAudioData.data = b"\x00" * 2 * sample_count
+    fake_msg.rawAudioData.sampleRate = rate
+    mocker.patch("iqpilot.system.webrtc.device.audio.messaging.recv_one_or_none", return_value=fake_msg)
+    track = AudioInputStreamTrack(rate=rate)
+
+    for i in range(5):
+      frame = self.loop.run_until_complete(track.recv())
+      assert frame.rate == rate
+      assert frame.samples == sample_count
+      assert frame.pts == i * sample_count

iqpilot/system/webrtc/tests/test_ui_stream.py

@@ -0,0 +1,301 @@
+import json
+import math
+
+from iqpilot.cereal import log, messaging
+from iqpilot.system.webrtc.ui_stream import (
+  UI_STREAM_SERVICES,
+  UIStreamMessageProxy,
+  compute_ui_status,
+  frame_to_str,
+  MAX_BUFFERED_BYTES,
+)
+
+OpenpilotState = log.SelfdriveState.OpenpilotState
+
+
+def make_readers(**overrides):
+  readers = {}
+  for service in UI_STREAM_SERVICES:
+    if service == "onroadEvents":
+      msg = messaging.new_message(service, 0)
+    else:
+      msg = messaging.new_message(service)
+    readers[service] = msg
+  readers.update(overrides)
+  return {s: getattr(m, s) for s, m in readers.items()}
+
+
+class FakeSubMaster:
+  def __init__(self, readers, updated=None, valid=None):
+    self.readers = readers
+    self.updated = updated or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.valid = valid or dict.fromkeys(UI_STREAM_SERVICES, True)
+    self.logMonoTime = dict.fromkeys(UI_STREAM_SERVICES, 42)
+    self.update_calls = 0
+
+  def __getitem__(self, service):
+    return self.readers[service]
+
+  def update(self, timeout):
+    self.update_calls += 1
+
+
+class FakeChannel:
+  def __init__(self, buffered_amount=0):
+    self.bufferedAmount = buffered_amount
+    self.sent = []
+
+  def send(self, data):
+    self.sent.append(data)
+
+
+def make_proxy(sm, **kwargs):
+  return UIStreamMessageProxy(sm=sm, **kwargs)
+
+
+class TestComputeUiStatus:
+  def _msgs(self):
+    ss = messaging.new_message("selfdriveState")
+    iq = messaging.new_message("iqState")
+    ev = messaging.new_message("onroadEvents", 0)
+    return ss.selfdriveState, iq.iqState, ev.onroadEvents
+
+  def test_disengaged(self):
+    ss, iq, ev = self._msgs()
+    assert compute_ui_status(ss, iq, ev) == "disengaged"
+
+  def test_engaged_no_guidance(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+  def test_pre_enabled_is_override(self):
+    ss, iq, ev = self._msgs()
+    ss.state = OpenpilotState.preEnabled
+    assert compute_ui_status(ss, iq, ev) == "override"
+
+  def test_lat_only(self):
+    ss, iq, ev = self._msgs()
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "lat_only"
+
+  def test_long_only(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    assert compute_ui_status(ss, iq, ev) == "long_only"
+
+  def test_both_engaged(self):
+    ss, iq, ev = self._msgs()
+    ss.enabled = True
+    iq.aol.available = True
+    iq.aol.enabled = True
+    assert compute_ui_status(ss, iq, ev) == "engaged"
+
+
+class TestUIStreamFrame:
+  def test_frame_shape_and_json(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) for i in range(33)]
+    model.position.y = [0.123456] * 33
+    model.position.z = [0.0] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [1.0, 2.0]
+      lane.y = [0.1, 0.2]
+      lane.z = [0.0, 0.0]
+    model.laneLineProbs = [0.9, 0.8, 0.7, 0.6]
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [1.0]
+      edge.y = [2.0]
+      edge.z = [0.0]
+    model.roadEdgeStds = [0.1, 0.2]
+    model.acceleration.x = [0.5] * 33
+
+    cs_msg = messaging.new_message("carState")
+    cs_msg.carState.vEgo = 12.345
+    cs_msg.carState.leftBlinker = True
+
+    readers = make_readers(modelV2=model_msg, carState=cs_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"
+    data = frame["data"]
+    assert len(data["modelV2"]["position"]["x"]) == 33
+    assert data["modelV2"]["position"]["y"][0] == 0.12
+    assert len(data["modelV2"]["laneLines"]) == 4
+    assert data["carState"]["vEgo"] == 12.35
+    assert data["carState"]["leftBlinker"] is True
+    assert data["uiStatus"] == "disengaged"
+    assert data["selfdriveState"]["alertSize"] == "none"
+    assert "hasLongitudinalControl" in data["init"]
+    assert "cameraOffset" in data["init"]
+    assert "isMetric" in data["init"]
+
+  def test_nan_scrubbed(self):
+    model_msg = messaging.new_message("modelV2")
+    model_msg.modelV2.position.x = [math.nan, math.inf, 1.0]
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    raw = channel.sent[0]
+    assert "NaN" not in raw and "Infinity" not in raw
+    frame = json.loads(raw)
+    assert frame["data"]["modelV2"]["position"]["x"] == [0.0, 0.0, 1.0]
+
+  def test_backpressure_drops_frames(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel(buffered_amount=MAX_BUFFERED_BYTES + 1)
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert proxy.dropped_frames == 1
+
+  def test_no_send_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    proxy._last_emit_time = float("inf")
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert channel.sent == []
+    assert sm.update_calls == 1
+
+  def test_heartbeat_without_model_update(self):
+    readers = make_readers()
+    updated = dict.fromkeys(UI_STREAM_SERVICES, False)
+    sm = FakeSubMaster(readers, updated=updated)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    proxy.update()
+
+    assert len(channel.sent) == 1
+    frame = json.loads(channel.sent[0])
+    assert frame["data"]["modelV2"] is None
+
+  def test_low_bandwidth_decimation(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 500_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 2
+
+  def test_full_rate_at_high_bitrate(self):
+    readers = make_readers()
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm, bitrate_getter=lambda: 5_000_000)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+
+    for _ in range(4):
+      proxy.update()
+
+    assert len(channel.sent) == 4
+
+  def test_sticky_status_when_engaged_like(self):
+    ss_msg = messaging.new_message("selfdriveState")
+    ss_msg.selfdriveState.enabled = True
+    iq_msg = messaging.new_message("iqState")
+    iq_msg.iqState.aol.available = True
+    iq_msg.iqState.aol.enabled = True
+
+    readers = make_readers(selfdriveState=ss_msg, iqState=iq_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    channel = FakeChannel()
+    proxy.add_channel(channel)
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+    iq_msg.iqState.aol.available = False
+    proxy.update()
+    assert json.loads(channel.sent[-1])["data"]["uiStatus"] == "engaged"
+
+  def test_frame_size_budget(self):
+    model_msg = messaging.new_message("modelV2")
+    model = model_msg.modelV2
+    model.position.x = [float(i) * 3.03 for i in range(33)]
+    model.position.y = [1.234567] * 33
+    model.position.z = [0.456789] * 33
+    model.init("laneLines", 4)
+    for lane in model.laneLines:
+      lane.x = [float(i) * 3.03 for i in range(33)]
+      lane.y = [1.234567] * 33
+      lane.z = [0.456789] * 33
+    model.laneLineProbs = [0.9] * 4
+    model.init("roadEdges", 2)
+    for edge in model.roadEdges:
+      edge.x = [float(i) * 3.03 for i in range(33)]
+      edge.y = [1.234567] * 33
+      edge.z = [0.456789] * 33
+    model.acceleration.x = [1.23] * 33
+
+    readers = make_readers(modelV2=model_msg)
+    sm = FakeSubMaster(readers)
+    proxy = make_proxy(sm)
+    frame = proxy._build_frame()
+    encoded = frame_to_str(frame)
+    assert len(encoded) < 8 * 1024
+
+
+class TestSessionWiring:
+  def test_set_ui_stream_control_message(self):
+    import asyncio
+    import logging
+    from types import SimpleNamespace
+    from iqpilot.system.webrtc.webrtcd import StreamSession
+
+    session = StreamSession.__new__(StreamSession)
+    session.logger = logging.getLogger("webrtcd")
+    session.ui_stream_runner = None
+    session.bitrate_controller = None
+    session.incoming_bridge = None
+    channel = FakeChannel()
+    session.stream = SimpleNamespace(
+      has_messaging_channel=lambda: True,
+      get_messaging_channel=lambda: channel,
+    )
+
+    async def go():
+      await session.message_handler(b'{"type":"setUiStream","enabled":true}')
+      assert session.ui_stream_runner is not None
+      await asyncio.sleep(0.05)
+      await session.message_handler(b'{"type":"setUiStream","enabled":false}')
+      assert session.ui_stream_runner is None
+
+    asyncio.run(go())
+
+    assert len(channel.sent) >= 1
+    frame = json.loads(channel.sent[0])
+    assert frame["type"] == "uiStream"

iqpilot/system/webrtc/tests/test_webrtcd.py

@@ -0,0 +1,64 @@
+import pytest
+import asyncio
+import json
+# for aiortc and its dependencies
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
+
+from iqpilot.system.webrtc.webrtcd import get_stream
+
+import aiortc
+from teleoprtc import WebRTCOfferBuilder
+from parameterized import parameterized_class
+
+
+@parameterized_class(("in_services", "out_services"), [
+  (["testJoystick"], ["carState"]),
+  ([], ["carState"]),
+  (["testJoystick"], []),
+  ([], []),
+])
+@pytest.mark.asyncio
+class TestWebrtcdProc:
+  async def assertCompletesWithTimeout(self, awaitable, timeout=10):
+    try:
+      async with asyncio.timeout(timeout):
+        await awaitable
+    except TimeoutError:
+      pytest.fail("Timeout while waiting for awaitable to complete")
+
+  async def test_webrtcd(self, mocker):
+    mock_request = mocker.MagicMock()
+    async def connect(offer):
+      body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
+      mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
+      response = await get_stream(mock_request)
+      response_json = json.loads(response.text)
+      return aiortc.RTCSessionDescription(**response_json)
+
+    builder = WebRTCOfferBuilder(connect, ice_servers=[])
+    builder.offer_to_receive_video_stream("road")
+    builder.offer_to_receive_audio_stream()
+    if len(self.in_services) > 0 or len(self.out_services) > 0:
+      builder.add_messaging()
+
+    stream = builder.stream()
+
+    await self.assertCompletesWithTimeout(stream.start())
+    await self.assertCompletesWithTimeout(stream.wait_for_connection())
+
+    assert stream.has_incoming_video_track("road")
+    assert stream.has_incoming_audio_track()
+    assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
+
+    video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
+    await self.assertCompletesWithTimeout(video_track.recv())
+    await self.assertCompletesWithTimeout(audio_track.recv())
+
+    await self.assertCompletesWithTimeout(stream.stop())
+
+    # cleanup, very implementation specific, test may break if it changes
+    assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
+    _, session = mock_request.app["streams"].__setitem__.call_args.args
+    await self.assertCompletesWithTimeout(session.post_run_cleanup())