IQ.Pilot Release Commit @ bec7652

artifacts/package_sources/teleoprtc/examples/face_detection/README.md

@@ -0,0 +1,10 @@
+## Face detection demo
+
+Run simple face detection model on video stream from driver camera of comma three.
+
+This example streams video frames, runs face-detection model and displays window with live detection results (bounding boxes).
+
+```sh
+# pass the ip address of comma three, if running remotely (by default localhost)
+python3 face_detection.py [--host comma-ip-address]
+```

artifacts/package_sources/teleoprtc/examples/face_detection/face_detection.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+
+import aiortc
+import aiohttp
+import cv2
+import pygame
+
+from teleoprtc import WebRTCOfferBuilder, StreamingOffer
+
+
+def pygame_should_quit():
+  for event in pygame.event.get():
+    if event.type == pygame.QUIT:
+      return True
+  return False
+
+
+class WebrtcdConnectionProvider:
+  """
+  Connection provider reaching webrtcd server on comma three
+  """
+  def __init__(self, host, port=5001):
+    self.url = f"http://{host}:{port}/stream"
+
+  async def __call__(self, offer: StreamingOffer) -> aiortc.RTCSessionDescription:
+    async with aiohttp.ClientSession() as session:
+      body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': [], 'bridge_services_out': []}
+      async with session.post(self.url, json=body) as resp:
+        payload = await resp.json()
+        answer = aiortc.RTCSessionDescription(**payload)
+        return answer
+
+
+class FaceDetector:
+  """
+  Simple face detector using opencv
+  """
+  def __init__(self):
+    self.classifier = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+
+  def detect(self, array):
+    gray_array = cv2.cvtColor(array, cv2.COLOR_RGB2GRAY)
+    faces = self.classifier.detectMultiScale(gray_array, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
+    return faces
+
+  def draw(self, array, faces):
+    for (x, y, w, h) in faces:
+      cv2.rectangle(array, (x, y), (x + w, y + h), (0, 255, 0), 2)
+    return array
+
+
+async def run_face_detection(stream):
+  # setup pygame window
+  pygame.init()
+  screen_width, screen_height = 1280, 720
+  screen = pygame.display.set_mode((screen_width, screen_height))
+  pygame.display.set_caption("Face detection demo")
+  surface = pygame.Surface((screen_width, screen_height))
+
+  # get the driver camera video track from the stream
+  # generally its better to reuse the track object instead of getting it every time
+  track = stream.get_incoming_video_track("driver", buffered=False)
+  # cv2 face detector
+  detector = FaceDetector()
+  while stream.is_connected_and_ready and not pygame_should_quit():
+    try:
+      # receive frame as pyAV VideoFrame, convert to rgb24 numpy array
+      frame = await track.recv()
+      array = frame.to_ndarray(format="rgb24")
+
+      # detect faces and draw rects around them
+      resized_array = cv2.resize(array, (screen_width, screen_height))
+      faces = detector.detect(resized_array)
+      detector.draw(resized_array, faces)
+
+      # display the image
+      pygame.surfarray.blit_array(surface, resized_array.swapaxes(0, 1))
+      screen.blit(surface, (0, 0))
+      pygame.display.flip()
+
+      print("Received frame from", "driver", frame.time)
+    except aiortc.mediastreams.MediaStreamError:
+      break
+
+  pygame.quit()
+  await stream.stop()
+
+
+async def run(args):
+  # build your own the offer stream
+  builder = WebRTCOfferBuilder(WebrtcdConnectionProvider(args.host))
+  # request video stream from drivers camera
+  builder.offer_to_receive_video_stream("driver")
+  # add cereal messaging streaming support
+  builder.add_messaging()
+
+  stream = builder.stream()
+
+  # start the stream then wait for connection
+  # server will receive the offer and attempt to fulfill it
+  await stream.start()
+  await stream.wait_for_connection()
+  # all the tracks and channel are ready to be used at this point
+
+  assert stream.has_incoming_video_track("driver") and stream.has_messaging_channel()
+
+  # run face detection loop on the drivers camera
+  await run_face_detection(stream)
+
+
+if __name__=='__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--host", default="localhost", help="Host for webrtcd server")
+
+  args = parser.parse_args()
+  asyncio.run(run(args))

artifacts/package_sources/teleoprtc/examples/videostream_cli/README.md

@@ -0,0 +1 @@
+# VideoStream CLI

artifacts/package_sources/teleoprtc/examples/videostream_cli/cli.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import argparse
+import asyncio
+import dataclasses
+import json
+import logging
+
+import aiortc
+from aiortc.mediastreams import VideoStreamTrack, AudioStreamTrack
+
+from teleoprtc import WebRTCOfferBuilder, WebRTCAnswerBuilder
+from teleoprtc.stream import StreamingOffer
+from teleoprtc.info import parse_info_from_offer
+
+
+async def async_input():
+  return await asyncio.to_thread(input)
+
+
+async def StdioConnectionProvider(offer: StreamingOffer) -> aiortc.RTCSessionDescription:
+  print("-- Please send this JSON to server --")
+  print(json.dumps(dataclasses.asdict(offer)))
+  print("-- Press enter when the answer is ready --")
+  raw_payload = await async_input()
+  payload = json.loads(raw_payload)
+  answer = aiortc.RTCSessionDescription(**payload)
+
+  return answer
+
+
+async def run_answer(args):
+  streams = []
+  while True:
+    print("-- Please enter a JSON from client --")
+    raw_payload = await async_input()
+
+    payload = json.loads(raw_payload)
+    offer = StreamingOffer(**payload)
+    info = parse_info_from_offer(offer.sdp)
+    assert len(offer.video) == info.n_expected_camera_tracks
+    video_tracks = [VideoStreamTrack() for _ in offer.video]
+    audio_tracks = [AudioStreamTrack()] if info.expected_audio_track else []
+
+    stream_builder = WebRTCAnswerBuilder(offer.sdp)
+    for cam, track in zip(offer.video, video_tracks, strict=True):
+      stream_builder.add_video_stream(cam, track)
+    for track in audio_tracks:
+      stream_builder.add_audio_stream(track)
+    stream = stream_builder.stream()
+    answer = await stream.start()
+    streams.append(stream)
+
+    print("-- Please send this JSON to client --")
+    print(json.dumps({"sdp": answer.sdp, "type": answer.type}))
+
+    await stream.wait_for_connection()
+
+
+async def run_offer(args):
+  stream_builder = WebRTCOfferBuilder(StdioConnectionProvider)
+  for cam in args.cameras:
+    stream_builder.offer_to_receive_video_stream(cam)
+  if args.audio:
+    stream_builder.offer_to_receive_audio_stream()
+  if args.messaging:
+    stream_builder.add_messaging()
+  stream = stream_builder.stream()
+  _ = await stream.start()
+  await stream.wait_for_connection()
+  print("Connection established and all tracks are ready")
+
+  video_tracks = [stream.get_incoming_video_track(cam, False) for cam in args.cameras]
+  audio_track = None
+  if stream.has_incoming_audio_track():
+    audio_track = stream.get_incoming_audio_track(False)
+  while True:
+    try:
+      frames = await asyncio.gather(*[track.recv() for track in video_tracks])
+      for key, frame in zip(args.cameras, frames, strict=True):
+        print("Received frame from", key, frame.time)
+      if audio_track:
+        frame = await audio_track.recv()
+        print("Received frame from audio", frame.time)
+    except aiortc.mediastreams.MediaStreamError:
+      return
+    print("=====================================")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  subparsers = parser.add_subparsers(dest="command", required=True)
+
+  offer_parser = subparsers.add_parser("offer", description="Create offer stream")
+  offer_parser.add_argument("--audio", action="store_true", help="Offer to receive audio")
+  offer_parser.add_argument("--messaging", action="store_true", help="Add messaging support")
+  offer_parser.add_argument("cameras", metavar="CAMERA", type=str, nargs="+", default=[], help="Camera types to stream")
+
+  answer_parser = subparsers.add_parser("answer", description="Create answer stream")
+
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.CRITICAL, handlers=[logging.StreamHandler()])
+  logger = logging.getLogger("WebRTCStream")
+  logger.setLevel(logging.DEBUG)
+
+  loop = asyncio.get_event_loop()
+  if args.command == "offer":
+    loop.run_until_complete(run_offer(args))
+  elif args.command == "answer":
+    loop.run_until_complete(run_answer(args))