IQ.Pilot Prebuilt Release @ ab07000

teleoprtc_repo/.github/workflows/release.yaml

@@ -0,0 +1,28 @@
+name: release
+
+on:
+  workflow_dispatch:
+
+jobs:
+  publish_pypi:
+    name: "Publish package on PyPI"
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.repository == 'commaai/teleoprtc'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-tags: true
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Bump version and tag
+      run: |
+        git config --global user.name "Vehicle Researcher"
+        git config --global user.email "user@comma.ai"
+
+        bash scripts/bump_tag.sh
+
+        git push --no-verify --force-with-lease --tags origin master
+    - name: Build and publish
+      run: |
+        bash scripts/publish_pypi.sh "${{ secrets.PYPI_PAT }}"

teleoprtc_repo/.github/workflows/tests.yaml

@@ -0,0 +1,36 @@
+name: tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install aiortc dependencies
+      run: |
+        sudo apt update
+        sudo apt install libavdevice-dev libavfilter-dev libopus-dev libvpx-dev libsrtp2-dev pkg-config
+    - name: Install package
+      run: pip install -e .[dev]
+    - name: Unit Tests
+      run: pytest
+  static_analysis:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+    - name: Install pre-commit
+      run: pip install pre-commit
+    - name: Static analysis
+      run: |
+        pre-commit run --all

teleoprtc_repo/.gitignore

@@ -0,0 +1,163 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+*.swp
+*.swo

teleoprtc_repo/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-ast
+    -   id: check-json
+    -   id: check-xml
+    -   id: check-yaml
+    -   id: check-merge-conflict
+    -   id: check-symlinks
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.7.1
+    hooks:
+    -   id: mypy
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.2
+    hooks:
+    -   id: ruff

teleoprtc_repo/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023, Comma.ai, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

teleoprtc_repo/README.md

@@ -0,0 +1,4 @@
+# teleoprtc
+
+Set of abstractions for webRTC communication with [openpilot](https://github.com/commaai/openpilot).
+

teleoprtc_repo/examples/face_detection/README.md

@@ -0,0 +1,10 @@
+## Face detection demo
+
+Run simple face detection model on video stream from driver camera of comma three.
+
+This example streams video frames, runs face-detection model and displays window with live detection results (bounding boxes).
+
+```sh
+# pass the ip address of comma three, if running remotely (by default localhost)
+python3 face_detection.py [--host comma-ip-address]
+```

teleoprtc_repo/examples/face_detection/face_detection.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import argparse
+import asyncio
+
+import aiortc
+import aiohttp
+import cv2
+import pygame
+
+from teleoprtc import WebRTCOfferBuilder, StreamingOffer
+
+
+def pygame_should_quit():
+  for event in pygame.event.get():
+    if event.type == pygame.QUIT:
+      return True
+  return False
+
+
+class WebrtcdConnectionProvider:
+  """
+  Connection provider reaching webrtcd server on comma three
+  """
+  def __init__(self, host, port=5001):
+    self.url = f"http://{host}:{port}/stream"
+
+  async def __call__(self, offer: StreamingOffer) -> aiortc.RTCSessionDescription:
+    async with aiohttp.ClientSession() as session:
+      body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': [], 'bridge_services_out': []}
+      async with session.post(self.url, json=body) as resp:
+        payload = await resp.json()
+        answer = aiortc.RTCSessionDescription(**payload)
+        return answer
+
+
+class FaceDetector:
+  """
+  Simple face detector using opencv
+  """
+  def __init__(self):
+    self.classifier = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+
+  def detect(self, array):
+    gray_array = cv2.cvtColor(array, cv2.COLOR_RGB2GRAY)
+    faces = self.classifier.detectMultiScale(gray_array, scaleFactor=1.1, minNeighbors=5, minSize=(30, 30))
+    return faces
+
+  def draw(self, array, faces):
+    for (x, y, w, h) in faces:
+      cv2.rectangle(array, (x, y), (x + w, y + h), (0, 255, 0), 2)
+    return array
+
+
+async def run_face_detection(stream):
+  # setup pygame window
+  pygame.init()
+  screen_width, screen_height = 1280, 720
+  screen = pygame.display.set_mode((screen_width, screen_height))
+  pygame.display.set_caption("Face detection demo")
+  surface = pygame.Surface((screen_width, screen_height))
+
+  # get the driver camera video track from the stream
+  # generally its better to reuse the track object instead of getting it every time
+  track = stream.get_incoming_video_track("driver", buffered=False)
+  # cv2 face detector
+  detector = FaceDetector()
+  while stream.is_connected_and_ready and not pygame_should_quit():
+    try:
+      # receive frame as pyAV VideoFrame, convert to rgb24 numpy array
+      frame = await track.recv()
+      array = frame.to_ndarray(format="rgb24")
+
+      # detect faces and draw rects around them
+      resized_array = cv2.resize(array, (screen_width, screen_height))
+      faces = detector.detect(resized_array)
+      detector.draw(resized_array, faces)
+
+      # display the image
+      pygame.surfarray.blit_array(surface, resized_array.swapaxes(0, 1))
+      screen.blit(surface, (0, 0))
+      pygame.display.flip()
+
+      print("Received frame from", "driver", frame.time)
+    except aiortc.mediastreams.MediaStreamError:
+      break
+
+  pygame.quit()
+  await stream.stop()
+
+
+async def run(args):
+  # build your own the offer stream
+  builder = WebRTCOfferBuilder(WebrtcdConnectionProvider(args.host))
+  # request video stream from drivers camera
+  builder.offer_to_receive_video_stream("driver")
+  # add cereal messaging streaming support
+  builder.add_messaging()
+
+  stream = builder.stream()
+
+  # start the stream then wait for connection
+  # server will receive the offer and attempt to fulfill it
+  await stream.start()
+  await stream.wait_for_connection()
+  # all the tracks and channel are ready to be used at this point
+
+  assert stream.has_incoming_video_track("driver") and stream.has_messaging_channel()
+
+  # run face detection loop on the drivers camera
+  await run_face_detection(stream)
+
+
+if __name__=='__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--host", default="localhost", help="Host for webrtcd server")
+
+  args = parser.parse_args()
+  asyncio.run(run(args))

teleoprtc_repo/examples/videostream_cli/README.md

@@ -0,0 +1 @@
+# VideoStream CLI

teleoprtc_repo/examples/videostream_cli/cli.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+
+import argparse
+import asyncio
+import dataclasses
+import json
+import logging
+
+import aiortc
+from aiortc.mediastreams import VideoStreamTrack, AudioStreamTrack
+
+from teleoprtc import WebRTCOfferBuilder, WebRTCAnswerBuilder
+from teleoprtc.stream import StreamingOffer
+from teleoprtc.info import parse_info_from_offer
+
+
+async def async_input():
+  return await asyncio.to_thread(input)
+
+
+async def StdioConnectionProvider(offer: StreamingOffer) -> aiortc.RTCSessionDescription:
+  print("-- Please send this JSON to server --")
+  print(json.dumps(dataclasses.asdict(offer)))
+  print("-- Press enter when the answer is ready --")
+  raw_payload = await async_input()
+  payload = json.loads(raw_payload)
+  answer = aiortc.RTCSessionDescription(**payload)
+
+  return answer
+
+
+async def run_answer(args):
+  streams = []
+  while True:
+    print("-- Please enter a JSON from client --")
+    raw_payload = await async_input()
+
+    payload = json.loads(raw_payload)
+    offer = StreamingOffer(**payload)
+    info = parse_info_from_offer(offer.sdp)
+    assert len(offer.video) == info.n_expected_camera_tracks
+    video_tracks = [VideoStreamTrack() for _ in offer.video]
+    audio_tracks = [AudioStreamTrack()] if info.expected_audio_track else []
+
+    stream_builder = WebRTCAnswerBuilder(offer.sdp)
+    for cam, track in zip(offer.video, video_tracks, strict=True):
+      stream_builder.add_video_stream(cam, track)
+    for track in audio_tracks:
+      stream_builder.add_audio_stream(track)
+    stream = stream_builder.stream()
+    answer = await stream.start()
+    streams.append(stream)
+
+    print("-- Please send this JSON to client --")
+    print(json.dumps({"sdp": answer.sdp, "type": answer.type}))
+
+    await stream.wait_for_connection()
+
+
+async def run_offer(args):
+  stream_builder = WebRTCOfferBuilder(StdioConnectionProvider)
+  for cam in args.cameras:
+    stream_builder.offer_to_receive_video_stream(cam)
+  if args.audio:
+    stream_builder.offer_to_receive_audio_stream()
+  if args.messaging:
+    stream_builder.add_messaging()
+  stream = stream_builder.stream()
+  _ = await stream.start()
+  await stream.wait_for_connection()
+  print("Connection established and all tracks are ready")
+
+  video_tracks = [stream.get_incoming_video_track(cam, False) for cam in args.cameras]
+  audio_track = None
+  if stream.has_incoming_audio_track():
+    audio_track = stream.get_incoming_audio_track(False)
+  while True:
+    try:
+      frames = await asyncio.gather(*[track.recv() for track in video_tracks])
+      for key, frame in zip(args.cameras, frames, strict=True):
+        print("Received frame from", key, frame.time)
+      if audio_track:
+        frame = await audio_track.recv()
+        print("Received frame from audio", frame.time)
+    except aiortc.mediastreams.MediaStreamError:
+      return
+    print("=====================================")
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  subparsers = parser.add_subparsers(dest="command", required=True)
+
+  offer_parser = subparsers.add_parser("offer", description="Create offer stream")
+  offer_parser.add_argument("--audio", action="store_true", help="Offer to receive audio")
+  offer_parser.add_argument("--messaging", action="store_true", help="Add messaging support")
+  offer_parser.add_argument("cameras", metavar="CAMERA", type=str, nargs="+", default=[], help="Camera types to stream")
+
+  answer_parser = subparsers.add_parser("answer", description="Create answer stream")
+
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.CRITICAL, handlers=[logging.StreamHandler()])
+  logger = logging.getLogger("WebRTCStream")
+  logger.setLevel(logging.DEBUG)
+
+  loop = asyncio.get_event_loop()
+  if args.command == "offer":
+    loop.run_until_complete(run_offer(args))
+  elif args.command == "answer":
+    loop.run_until_complete(run_answer(args))

teleoprtc_repo/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "teleoprtc"
+version = "1.0.1"
+authors = [{ name="Vehicle Researcher", email="user@comma.ai" }]
+description = "Comma webRTC abstractions"
+readme = "README.md"
+license = { file="LICENSE" }
+requires-python = ">=3.8"
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+]
+dependencies = [
+  "aiortc>=1.6.0",
+  "aiohttp>=3.7.0",
+  "av>=11.0.0,<13.0.0",
+  "numpy>=1.19.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "parameterized>=0.8",
+  "pre-commit",
+  "pytest",
+  "pytest-asyncio",
+  "pytest-xdist"
+]
+
+[project.urls]
+"Homepage" = "https://github.com/commaai/teleoprtc"
+"Bug Tracker" = "https://github.com/commaai/teleoprtc/issues"
+
+# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
+[tool.ruff]
+line-length = 160
+target-version="py38"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF008", "RUF100", "A", "B", "TID251"]
+ignore = ["W292", "E741", "E402", "C408", "ISC003", "B027", "B024"]
+flake8-implicit-str-concat.allow-multiline=false
+
+[tool.ruff.lint.flake8-tidy-imports.banned-api]
+"unittest".msg = "Use pytest"
+
+[tool.pytest.ini_options]
+addopts = "--durations=10 -n auto"

teleoprtc_repo/scripts/bump_tag.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+BRANCH="$(git branch --show-current)"
+TOML_VERSION="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+LATEST_TAG_VERSION="$(git tag --list | sort -V -r | head -n 1)"
+TAGGED_VERSION=""
+
+if [[ "$BRANCH" != "master" ]]; then
+  echo "Not on master branch."
+  exit 1
+fi
+
+if [[ "$TOML_VERSION" == "$LATEST_TAG_VERSION" ]]; then
+  TAGGED_VERSION=$(echo "$TOML_VERSION" | python3 -c "v = input().split('.'); v[-1]=str(int(v[-1])+1); print('.'.join(v))")
+  sed -i "s/version = \"$TOML_VERSION\"/version = \"$TAGGED_VERSION\"/" pyproject.toml
+elif [[ -z "$LATEST_TAG_VERSION" ]] || printf "$LATEST_TAG_VERSION\n$TOML_VERSION" | sort -V -C; then
+  TAGGED_VERSION="$TOML_VERSION"
+else
+  echo "Version in pyproject.toml is lower than the latest tag version."
+  exit 1
+fi
+
+echo "Tagging $TAGGED_VERSION..."
+if [[ -n "$(git ls-files -m | grep pyproject.toml)" ]]; then
+  echo "Commiting pyproject.toml..."
+  git add pyproject.toml
+  git commit --no-verify -m "Bump version to $TAGGED_VERSION"
+fi
+git tag "$TAGGED_VERSION"

teleoprtc_repo/scripts/publish_pypi.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -e
+
+if [[ -z "$1" ]]; then
+  echo "Usage: $0 <PyPI token>"
+  exit 1
+fi
+PYPI_TOKEN="$1"
+
+# install required packages
+pip install --upgrade twine build
+
+# build the package
+python3 -m build
+
+# upload to PyPI
+REPOSITORY=""
+if [[ -n "$TEST_UPLOAD" ]]; then
+    REPOSITORY="--repository testpypi"
+fi
+
+python3 -m twine upload $REPOSITORY --username __token__ --password "$PYPI_TOKEN" dist/*

teleoprtc_repo/teleoprtc/__init__.py

@@ -0,0 +1,2 @@
+from teleoprtc.builder import WebRTCOfferBuilder, WebRTCAnswerBuilder  # noqa
+from teleoprtc.stream import WebRTCBaseStream, StreamingOffer, ConnectionProvider, MessageHandler  # noqa

teleoprtc_repo/teleoprtc/builder.py

@@ -0,0 +1,83 @@
+import abc
+from typing import Dict, List, Optional
+
+import aiortc
+
+from teleoprtc.stream import WebRTCBaseStream, WebRTCOfferStream, WebRTCAnswerStream, ConnectionProvider
+from teleoprtc.tracks import TiciVideoStreamTrack, TiciTrackWrapper
+
+
+class WebRTCStreamBuilder(abc.ABC):
+  @abc.abstractmethod
+  def stream(self) -> WebRTCBaseStream:
+    raise NotImplementedError
+
+
+class WebRTCOfferBuilder(WebRTCStreamBuilder):
+  def __init__(self, connection_provider: ConnectionProvider, ice_servers: Optional[List[dict]] = None):
+    self.connection_provider = connection_provider
+    self.ice_servers = ice_servers
+    self.requested_camera_types: List[str] = []
+    self.requested_audio = False
+    self.audio_tracks: List[aiortc.MediaStreamTrack] = []
+    self.messaging_enabled = False
+
+  def offer_to_receive_video_stream(self, camera_type: str):
+    assert camera_type in ["driver", "wideRoad", "road"]
+    self.requested_camera_types.append(camera_type)
+
+  def offer_to_receive_audio_stream(self):
+    self.requested_audio = True
+
+  def add_audio_stream(self, track: aiortc.MediaStreamTrack):
+    assert len(self.audio_tracks) == 0
+    self.audio_tracks = [track]
+
+  def add_messaging(self):
+    self.messaging_enabled = True
+
+  def stream(self) -> WebRTCBaseStream:
+    return WebRTCOfferStream(
+      self.connection_provider,
+      consumed_camera_types=self.requested_camera_types,
+      consume_audio=self.requested_audio,
+      video_producer_tracks=[],
+      audio_producer_tracks=self.audio_tracks,
+      should_add_data_channel=self.messaging_enabled,
+      ice_servers=self.ice_servers,
+    )
+
+
+class WebRTCAnswerBuilder(WebRTCStreamBuilder):
+  def __init__(self, offer_sdp: str, ice_servers: Optional[List[dict]] = None):
+    self.offer_sdp = offer_sdp
+    self.ice_servers = ice_servers
+    self.video_tracks: Dict[str, aiortc.MediaStreamTrack] = dict()
+    self.requested_audio = False
+    self.audio_tracks: List[aiortc.MediaStreamTrack] = []
+
+  def offer_to_receive_audio_stream(self):
+    self.requested_audio = True
+
+  def add_video_stream(self, camera_type: str, track: aiortc.MediaStreamTrack):
+    assert camera_type not in self.video_tracks
+    assert camera_type in ["driver", "wideRoad", "road"]
+    if not isinstance(track, TiciVideoStreamTrack):
+      track = TiciTrackWrapper(camera_type, track)
+    self.video_tracks[camera_type] = track
+
+  def add_audio_stream(self, track: aiortc.MediaStreamTrack):
+    assert len(self.audio_tracks) == 0
+    self.audio_tracks = [track]
+
+  def stream(self) -> WebRTCBaseStream:
+    description = aiortc.RTCSessionDescription(sdp=self.offer_sdp, type="offer")
+    return WebRTCAnswerStream(
+      description,
+      consumed_camera_types=[],
+      consume_audio=self.requested_audio,
+      video_producer_tracks=list(self.video_tracks.values()),
+      audio_producer_tracks=self.audio_tracks,
+      should_add_data_channel=False,
+      ice_servers=self.ice_servers,
+    )

teleoprtc_repo/teleoprtc/info.py

@@ -0,0 +1,26 @@
+import dataclasses
+
+import aiortc
+
+
+@dataclasses.dataclass
+class StreamingMediaInfo:
+  n_expected_camera_tracks: int
+  expected_audio_track: bool
+  incoming_audio_track: bool
+  incoming_datachannel: bool
+
+
+def parse_info_from_offer(sdp: str) -> StreamingMediaInfo:
+  """
+  helper function to parse info about outgoing and incoming streams from an offer sdp
+  """
+  desc = aiortc.sdp.SessionDescription.parse(sdp)
+  audio_tracks = [m for m in desc.media if m.kind == "audio"]
+  video_tracks = [m for m in desc.media if m.kind == "video" and m.direction in ["recvonly", "sendrecv"]]
+  application_tracks = [m for m in desc.media if m.kind == "application"]
+  has_incoming_audio_track = next((t for t in audio_tracks if t.direction in ["sendonly", "sendrecv"]), None) is not None
+  has_incoming_datachannel = len(application_tracks) > 0
+  expects_outgoing_audio_track = next((t for t in audio_tracks if t.direction in ["recvonly", "sendrecv"]), None) is not None
+
+  return StreamingMediaInfo(len(video_tracks), expects_outgoing_audio_track, has_incoming_audio_track, has_incoming_datachannel)

teleoprtc_repo/teleoprtc/stream.py

@@ -0,0 +1,337 @@
+import abc
+import asyncio
+import dataclasses
+import logging
+from typing import Any, Awaitable, Callable, Dict, List, Optional
+
+import aiortc
+from aiortc.contrib.media import MediaRelay
+
+from teleoprtc.tracks import parse_video_track_id
+
+
+@dataclasses.dataclass
+class StreamingOffer:
+  sdp: str
+  video: List[str]
+
+
+ConnectionProvider = Callable[[StreamingOffer], Awaitable[aiortc.RTCSessionDescription]]
+MessageHandler = Callable[[bytes], Awaitable[None]]
+
+
+def _rtc_configuration_from_ice_servers(ice_servers: Optional[List[Dict[str, Any]]]) -> Optional[aiortc.RTCConfiguration]:
+  if not ice_servers:
+    return None
+
+  servers: List[aiortc.RTCIceServer] = []
+  for s in ice_servers:
+    if not isinstance(s, dict):
+      continue
+    urls = s.get("urls") or s.get("url")
+    if not urls:
+      continue
+    username = s.get("username")
+    credential = s.get("credential")
+    servers.append(aiortc.RTCIceServer(urls=urls, username=username, credential=credential))
+
+  if not servers:
+    return None
+
+  return aiortc.RTCConfiguration(iceServers=servers)
+
+
+class WebRTCBaseStream(abc.ABC):
+  def __init__(self,
+               consumed_camera_types: List[str],
+               consume_audio: bool,
+               video_producer_tracks: List[aiortc.MediaStreamTrack],
+               audio_producer_tracks: List[aiortc.MediaStreamTrack],
+               should_add_data_channel: bool,
+               ice_servers: Optional[List[Dict[str, Any]]] = None):
+    configuration = _rtc_configuration_from_ice_servers(ice_servers)
+    self.peer_connection = aiortc.RTCPeerConnection(configuration=configuration) if configuration else aiortc.RTCPeerConnection()
+    self.media_relay = MediaRelay()
+    self.expected_incoming_camera_types = consumed_camera_types
+    self.expected_incoming_audio = consume_audio
+    self.expected_number_of_incoming_media: Optional[int] = None
+
+    self.incoming_camera_tracks: Dict[str, aiortc.MediaStreamTrack] = dict()
+    self.incoming_audio_tracks: List[aiortc.MediaStreamTrack] = []
+    self.outgoing_video_tracks: List[aiortc.MediaStreamTrack] = video_producer_tracks
+    self.outgoing_audio_tracks: List[aiortc.MediaStreamTrack] = audio_producer_tracks
+
+    self.should_add_data_channel = should_add_data_channel
+    self.messaging_channel: Optional[aiortc.RTCDataChannel] = None
+    self.incoming_message_handlers: List[MessageHandler] = []
+
+    self.incoming_media_ready_event = asyncio.Event()
+    self.messaging_channel_ready_event = asyncio.Event()
+    self.connection_attempted_event = asyncio.Event()
+    self.connection_stopped_event = asyncio.Event()
+
+    self.peer_connection.on("connectionstatechange", self._on_connectionstatechange)
+    self.peer_connection.on("datachannel", self._on_incoming_datachannel)
+    self.peer_connection.on("track", self._on_incoming_track)
+
+    self.logger = logging.getLogger("WebRTCStream")
+
+  async def _wait_for_ice_gathering_complete(self, timeout_s: float = 5.0):
+    # Non-trickle ICE: wait for candidates to be included in SDP.
+    if self.peer_connection.iceGatheringState == "complete":
+      return
+
+    loop = asyncio.get_running_loop()
+    end = loop.time() + timeout_s
+    while self.peer_connection.iceGatheringState != "complete" and loop.time() < end:
+      await asyncio.sleep(0.05)
+
+  def _log_debug(self, msg: Any, *args):
+    self.logger.debug(f"{type(self)}() {msg}", *args)
+
+  @property
+  def _number_of_incoming_media(self) -> int:
+    media = len(self.incoming_camera_tracks) + len(self.incoming_audio_tracks)
+    # if stream does not add data_channel, then it means its incoming
+    media += int(self.messaging_channel is not None) if not self.should_add_data_channel else 0
+    return media
+
+  def _add_consumer_transceivers(self):
+    for _ in self.expected_incoming_camera_types:
+      self.peer_connection.addTransceiver("video", direction="recvonly")
+    if self.expected_incoming_audio:
+      self.peer_connection.addTransceiver("audio", direction="recvonly")
+
+  def _find_trackless_transceiver(self, kind: str) -> Optional[aiortc.RTCRtpTransceiver]:
+    transceivers = self.peer_connection.getTransceivers()
+    target_transceiver = None
+    for t in transceivers:
+      if t.kind == kind and t.sender.track is None:
+        target_transceiver = t
+        break
+
+    return target_transceiver
+
+  def _add_producer_tracks(self):
+    for track in self.outgoing_video_tracks:
+      target_transceiver = self._find_trackless_transceiver(track.kind)
+      if target_transceiver is None:
+        self.peer_connection.addTransceiver(track.kind, direction="sendonly")
+
+      sender = self.peer_connection.addTrack(track)
+      if hasattr(track, "codec_preference") and track.codec_preference() is not None:
+        transceiver = next(t for t in self.peer_connection.getTransceivers() if t.sender == sender)
+        self._force_codec(transceiver, track.codec_preference(), "video")
+    for track in self.outgoing_audio_tracks:
+      target_transceiver = self._find_trackless_transceiver(track.kind)
+      if target_transceiver is None:
+        self.peer_connection.addTransceiver(track.kind, direction="sendonly")
+
+      self.peer_connection.addTrack(track)
+
+  def _add_messaging_channel(self, channel: Optional[aiortc.RTCDataChannel] = None):
+    if not channel:
+      channel = self.peer_connection.createDataChannel("data", ordered=True)
+
+    for handler in self.incoming_message_handlers:
+      channel.on("message", handler)
+
+    if channel.readyState == "open":
+      self.messaging_channel_ready_event.set()
+    else:
+      channel.on("open", lambda: self.messaging_channel_ready_event.set())
+    self.messaging_channel = channel
+
+  def _force_codec(self, transceiver: aiortc.RTCRtpTransceiver, codec: str, stream_type: str):
+    codec_mime = f"{stream_type}/{codec.upper()}"
+    rtp_codecs = aiortc.RTCRtpSender.getCapabilities(stream_type).codecs
+    rtp_codec = [c for c in rtp_codecs if c.mimeType == codec_mime]
+    transceiver.setCodecPreferences(rtp_codec)
+
+  def _on_connectionstatechange(self):
+    self._log_debug("connection state is %s", self.peer_connection.connectionState)
+    if self.peer_connection.connectionState in ['connected', 'failed']:
+      self.connection_attempted_event.set()
+    if self.peer_connection.connectionState in ['disconnected', 'closed', 'failed']:
+      self.connection_stopped_event.set()
+
+  def _on_incoming_track(self, track: aiortc.MediaStreamTrack):
+    self._log_debug("got track: %s %s", track.kind, track.id)
+    if track.kind == "video":
+      camera_type, _ = parse_video_track_id(track.id)
+      if camera_type in self.expected_incoming_camera_types:
+        self.incoming_camera_tracks[camera_type] = track
+    elif track.kind == "audio":
+      if self.expected_incoming_audio:
+        self.incoming_audio_tracks.append(track)
+    self._on_after_media()
+
+  def _on_incoming_datachannel(self, channel: aiortc.RTCDataChannel):
+    self._log_debug("got data channel: %s", channel.label)
+    if channel.label == "data" and self.messaging_channel is None:
+      self._add_messaging_channel(channel)
+    self._on_after_media()
+
+  def _on_after_media(self):
+    if self._number_of_incoming_media == self.expected_number_of_incoming_media:
+      self.incoming_media_ready_event.set()
+
+  def _parse_incoming_streams(self, remote_sdp: str):
+    desc = aiortc.sdp.SessionDescription.parse(remote_sdp)
+    sending_medias = [m for m in desc.media if m.direction in ["sendonly", "sendrecv"]]
+    incoming_media_count = len(sending_medias)
+    if not self.should_add_data_channel:
+      channel_medias = [m for m in desc.media if m.kind == "application"]
+      incoming_media_count += len(channel_medias)
+    self.expected_number_of_incoming_media = incoming_media_count
+
+  def has_incoming_video_track(self, camera_type: str) -> bool:
+    return camera_type in self.incoming_camera_tracks
+
+  def has_incoming_audio_track(self) -> bool:
+    return len(self.incoming_audio_tracks) > 0
+
+  def has_messaging_channel(self) -> bool:
+    return self.messaging_channel is not None
+
+  def get_incoming_video_track(self, camera_type: str, buffered: bool = False) -> aiortc.MediaStreamTrack:
+    assert camera_type in self.incoming_camera_tracks, "Video tracks are not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+
+    track = self.incoming_camera_tracks[camera_type]
+    relay_track = self.media_relay.subscribe(track, buffered=buffered)
+    return relay_track
+
+  def get_incoming_audio_track(self, buffered: bool = False) -> aiortc.MediaStreamTrack:
+    assert len(self.incoming_audio_tracks) > 0, "Audio tracks are not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+
+    track = self.incoming_audio_tracks[0]
+    relay_track = self.media_relay.subscribe(track, buffered=buffered)
+    return relay_track
+
+  def get_messaging_channel(self) -> aiortc.RTCDataChannel:
+    assert self.messaging_channel is not None, "Messaging channel is not enabled on this stream"
+    assert self.is_started, "Stream must be started"
+
+    return self.messaging_channel
+
+  def set_message_handler(self, message_handler: MessageHandler):
+    self.incoming_message_handlers.append(message_handler)
+    if self.messaging_channel is not None:
+      self.messaging_channel.on("message", message_handler)
+
+  @property
+  def is_started(self) -> bool:
+    return self.peer_connection is not None and \
+           self.peer_connection.localDescription is not None and \
+           self.peer_connection.remoteDescription is not None and \
+           self.peer_connection.connectionState != "closed"
+
+  @property
+  def is_connected_and_ready(self) -> bool:
+    return self.peer_connection is not None and \
+           self.peer_connection.connectionState == "connected" and \
+           (self.expected_number_of_incoming_media == 0 or self.incoming_media_ready_event.is_set())
+
+  async def wait_for_connection(self):
+    assert self.is_started
+    await self.connection_attempted_event.wait()
+    if self.peer_connection.connectionState != 'connected':
+      raise ValueError("Connection failed.")
+    if self.expected_number_of_incoming_media:
+      await self.incoming_media_ready_event.wait()
+    if self.messaging_channel is not None:
+      await self.messaging_channel_ready_event.wait()
+
+  async def wait_for_disconnection(self):
+    assert self.is_connected_and_ready, "Stream is not connected/ready yet (make sure wait_for_connection was awaited)"
+    await self.connection_stopped_event.wait()
+
+  async def stop(self):
+    await self.peer_connection.close()
+
+  @abc.abstractmethod
+  async def start(self) -> aiortc.RTCSessionDescription:
+    raise NotImplementedError
+
+
+class WebRTCOfferStream(WebRTCBaseStream):
+  def __init__(self, session_provider: ConnectionProvider, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.session_provider = session_provider
+
+  async def start(self) -> aiortc.RTCSessionDescription:
+    self._add_consumer_transceivers()
+    if self.should_add_data_channel:
+      self._add_messaging_channel()
+    self._add_producer_tracks()
+
+    offer = await self.peer_connection.createOffer()
+    await self.peer_connection.setLocalDescription(offer)
+    await self._wait_for_ice_gathering_complete()
+    actual_offer = self.peer_connection.localDescription
+
+    streaming_offer = StreamingOffer(
+      sdp=actual_offer.sdp,
+      video=list(self.expected_incoming_camera_types),
+    )
+    remote_answer = await self.session_provider(streaming_offer)
+    self._parse_incoming_streams(remote_sdp=remote_answer.sdp)
+    await self.peer_connection.setRemoteDescription(remote_answer)
+    actual_answer = self.peer_connection.remoteDescription
+
+    return actual_answer
+
+
+class WebRTCAnswerStream(WebRTCBaseStream):
+  def __init__(self, session: aiortc.RTCSessionDescription, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.session = session
+
+  def _probe_video_codecs(self) -> List[str]:
+    codecs = []
+    for track in self.outgoing_video_tracks:
+      if hasattr(track, "codec_preference") and track.codec_preference() is not None:
+        codecs.append(track.codec_preference())
+
+    return codecs
+
+  def _override_incoming_video_codecs(self, remote_sdp: str, codecs: List[str]) -> str:
+    desc = aiortc.sdp.SessionDescription.parse(remote_sdp)
+    codec_mimes = [f"video/{c}" for c in codecs]
+    for m in desc.media:
+      if m.kind != "video":
+        continue
+
+      preferred_codecs: List[aiortc.RTCRtpCodecParameters] = [c for c in m.rtp.codecs if c.mimeType in codec_mimes]
+      if len(preferred_codecs) == 0:
+        raise ValueError(f"None of {preferred_codecs} codecs is supported in remote SDP")
+
+      m.rtp.codecs = preferred_codecs
+      m.fmt = [c.payloadType for c in preferred_codecs]
+
+    return str(desc)
+
+  async def start(self) -> aiortc.RTCSessionDescription:
+    assert self.peer_connection.remoteDescription is None, "Connection already established"
+
+    self._add_consumer_transceivers()
+
+    # since we sent already encoded frames in some cases (e.g. livestream video tracks are in H264), we need to force aiortc to actually use it
+    # we do that by overriding supported codec information on incoming sdp
+    preferred_codecs = self._probe_video_codecs()
+    if len(preferred_codecs) > 0:
+      self.session.sdp = self._override_incoming_video_codecs(self.session.sdp, preferred_codecs)
+
+    self._parse_incoming_streams(remote_sdp=self.session.sdp)
+    await self.peer_connection.setRemoteDescription(self.session)
+
+    self._add_producer_tracks()
+
+    answer = await self.peer_connection.createAnswer()
+    await self.peer_connection.setLocalDescription(answer)
+    await self._wait_for_ice_gathering_complete()
+    actual_answer = self.peer_connection.localDescription
+
+    return actual_answer

teleoprtc_repo/teleoprtc/tracks.py

@@ -0,0 +1,76 @@
+import asyncio
+import logging
+import time
+import fractions
+from typing import Any, Optional, Tuple
+
+import aiortc
+from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
+
+
+def video_track_id(camera_type: str, track_id: str) -> str:
+  return f"{camera_type}:{track_id}"
+
+
+def parse_video_track_id(track_id: str) -> Tuple[str, str]:
+  parts = track_id.split(":")
+  if len(parts) != 2:
+    raise ValueError(f"Invalid video track id: {track_id}")
+
+  camera_type, track_id = parts
+  return camera_type, track_id
+
+
+class TiciVideoStreamTrack(aiortc.MediaStreamTrack):
+  """
+  Abstract video track which associates video track with camera_type
+  """
+  kind = "video"
+
+  def __init__(self, camera_type: str, dt: float, time_base: fractions.Fraction = VIDEO_TIME_BASE, clock_rate: int = VIDEO_CLOCK_RATE):
+    assert camera_type in ["driver", "wideRoad", "road"]
+    super().__init__()
+    # override track id to include camera type - client needs that for identification
+    self._id: str = video_track_id(camera_type, self._id)
+    self._dt: float = dt
+    self._time_base: fractions.Fraction = time_base
+    self._clock_rate: int = clock_rate
+    self._start: Optional[float] = None
+    self._logger = logging.getLogger("WebRTCStream")
+
+  def log_debug(self, msg: Any, *args):
+    self._logger.debug(f"{type(self)}() {msg}", *args)
+
+  async def next_pts(self, current_pts) -> float:
+    pts: float = current_pts + self._dt * self._clock_rate
+
+    data_time = pts * self._time_base
+    if self._start is None:
+      self._start = time.time() - data_time
+    else:
+      wait_time = self._start + data_time - time.time()
+      await asyncio.sleep(wait_time)
+
+    return pts
+
+  def codec_preference(self) -> Optional[str]:
+    return None
+
+
+class TiciTrackWrapper(aiortc.MediaStreamTrack):
+  """
+  Associates video track with camera_type
+  """
+  def __init__(self, camera_type: str, track: aiortc.MediaStreamTrack):
+    assert track.kind == "video"
+    assert not isinstance(track, TiciVideoStreamTrack)
+    super().__init__()
+    self._id = video_track_id(camera_type, track.id)
+    self._track = track
+
+  @property
+  def kind(self) -> str:
+    return self._track.kind
+
+  async def recv(self):
+    return await self._track.recv()

teleoprtc_repo/tests/test_info.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+from teleoprtc.info import parse_info_from_offer
+
+
+def lf2crlf(x):
+    return x.replace("\n", "\r\n")
+
+
+class TestStream:
+  def test_double_video_tracks(self):
+    sdp = """v=0
+o=- 3910210993 3910210993 IN IP4 0.0.0.0
+s=-
+t=0 0
+a=group:BUNDLE 0 1
+a=msid-semantic:WMS *
+m=video 9 UDP/TLS/RTP/SAVPF 97 98 99 100 101 102
+c=IN IP4 0.0.0.0
+a=recvonly
+a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=mid:0
+a=msid:e123f852-010c-4b7b-8761-71b72fbfd013 2b75cb0e-6b34-48d6-8bf9-21b809f2e08e
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-mux
+a=ssrc-group:FID 1048118556 4149054509
+a=ssrc:1048118556 cname:61992fce-bab5-42a0-ab8c-7112adfb1857
+a=ssrc:4149054509 cname:61992fce-bab5-42a0-ab8c-7112adfb1857
+a=rtpmap:97 VP8/90000
+a=rtcp-fb:97 nack
+a=rtcp-fb:97 nack pli
+a=rtcp-fb:97 goog-remb
+a=rtpmap:98 rtx/90000
+a=fmtp:98 apt=97
+a=rtpmap:99 H264/90000
+a=rtcp-fb:99 nack
+a=rtcp-fb:99 nack pli
+a=rtcp-fb:99 goog-remb
+a=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f
+a=rtpmap:100 rtx/90000
+a=fmtp:100 apt=99
+a=rtpmap:101 H264/90000
+a=rtcp-fb:101 nack
+a=rtcp-fb:101 nack pli
+a=rtcp-fb:101 goog-remb
+a=fmtp:101 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f
+a=rtpmap:102 rtx/90000
+a=fmtp:102 apt=101
+a=ice-ufrag:jxQW
+a=ice-pwd:KpJ0tfaY2RxnIYpTHqPSSv
+a=fingerprint:sha-256 70:3A:2D:37:3C:52:96:0E:10:F6:4D:7A:EB:18:38:1B:FD:CA:A5:90:D7:6C:DA:A9:39:76:C9:2F:FB:FF:56:0C
+a=setup:actpass
+m=video 9 UDP/TLS/RTP/SAVPF 97 98 99 100 101 102
+c=IN IP4 0.0.0.0
+a=recvonly
+a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=mid:1
+a=msid:e123f852-010c-4b7b-8761-71b72fbfd013 311db759-8d51-479c-a5b4-5c8d055c43ec
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-mux
+a=ssrc-group:FID 4096183284 2713379498
+a=ssrc:4096183284 cname:61992fce-bab5-42a0-ab8c-7112adfb1857
+a=ssrc:2713379498 cname:61992fce-bab5-42a0-ab8c-7112adfb1857
+a=rtpmap:97 VP8/90000
+a=rtcp-fb:97 nack
+a=rtcp-fb:97 nack pli
+a=rtcp-fb:97 goog-remb
+a=rtpmap:98 rtx/90000
+a=fmtp:98 apt=97
+a=rtpmap:99 H264/90000
+a=rtcp-fb:99 nack
+a=rtcp-fb:99 nack pli
+a=rtcp-fb:99 goog-remb
+a=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f
+a=rtpmap:100 rtx/90000
+a=fmtp:100 apt=99
+a=rtpmap:101 H264/90000
+a=rtcp-fb:101 nack
+a=rtcp-fb:101 nack pli
+a=rtcp-fb:101 goog-remb
+a=fmtp:101 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f
+a=rtpmap:102 rtx/90000
+a=fmtp:102 apt=101
+a=ice-ufrag:1234
+a=ice-pwd:1234
+a=fingerprint:sha-256 70:3A:2D:37:3C:52:96:0E:10:F6:4D:7A:EB:18:38:1B:FD:CA:A5:90:D7:6C:DA:A9:39:76:C9:2F:FB:FF:56:0C
+a=setup:actpass"""
+    info = parse_info_from_offer(lf2crlf(sdp))
+    assert info.n_expected_camera_tracks == 2
+    assert not info.expected_audio_track
+    assert not info.incoming_audio_track
+    assert not info.incoming_datachannel
+
+  def test_recvonly_audio(self):
+    sdp = """v=0
+o=- 3910210904 3910210904 IN IP4 0.0.0.0
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=msid-semantic:WMS *
+m=audio 9 UDP/TLS/RTP/SAVPF 96 0 8
+c=IN IP4 0.0.0.0
+a=recvonly
+a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:2 urn:ietf:params:rtp-hdrext:ssrc-audio-level
+a=mid:0
+a=msid:eb1d3f1a-569a-465f-b419-319477bfded6 e44eecb2-1a04-4547-97d8-481389f50d5b
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-mux
+a=ssrc:1233332626 cname:ca4dede8-4994-4a6d-9ae3-923b28177ca5
+a=rtpmap:96 opus/48000/2
+a=rtpmap:0 PCMU/8000
+a=rtpmap:8 PCMA/8000
+a=ice-ufrag:1234
+a=ice-pwd:1234
+a=fingerprint:sha-256 40:4B:14:CF:70:B8:67:E1:B1:FF:7E:F9:22:6E:60:7D:73:B5:1E:38:4B:10:20:9C:CD:1C:47:02:52:ED:45:25
+a=setup:actpass"""
+    info = parse_info_from_offer(lf2crlf(sdp))
+    assert info.n_expected_camera_tracks == 0
+    assert info.expected_audio_track
+    assert not info.incoming_audio_track
+    assert not info.incoming_datachannel
+
+  def test_incoming_datachanel(self):
+    sdp = """v=0
+o=- 3910211092 3910211092 IN IP4 0.0.0.0
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=msid-semantic:WMS *
+m=application 9 DTLS/SCTP 5000
+c=IN IP4 0.0.0.0
+a=mid:0
+a=sctpmap:5000 webrtc-datachannel 65535
+a=max-message-size:65536
+a=ice-ufrag:1234
+a=ice-pwd:1234
+a=fingerprint:sha-256 9B:C0:F3:35:8E:05:A1:15:DB:F8:39:0E:B0:E0:0C:EB:82:E4:B9:26:18:A6:43:2D:B9:9A:23:96:0A:59:B6:58
+a=setup:actpass"""
+    info = parse_info_from_offer(lf2crlf(sdp))
+    assert info.n_expected_camera_tracks == 0
+    assert not info.expected_audio_track
+    assert not info.incoming_audio_track
+    assert info.incoming_datachannel

teleoprtc_repo/tests/test_integration.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+
+import pytest
+import asyncio
+import sys
+
+from aiortc.mediastreams import AudioStreamTrack, VideoStreamTrack
+from parameterized import parameterized
+
+from teleoprtc.builder import WebRTCOfferBuilder, WebRTCAnswerBuilder
+from teleoprtc.stream import StreamingOffer
+from teleoprtc.info import parse_info_from_offer
+
+
+if sys.version_info >= (3, 11):
+  timeout = asyncio.timeout
+else:
+  class Timeout:
+    def __init__(self, delay: float):
+      self._delay = delay
+      self._task = None
+      self._timeout_handle = None
+
+    def _timeout(self):
+      if self._task:
+        self._task.cancel()
+
+    async def __aenter__(self):
+      self._task = asyncio.current_task()
+      loop = asyncio.events.get_running_loop()
+      self._timeout_handle = loop.call_later(self._delay, self._timeout)
+      return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+      if self._timeout_handle:
+        self._timeout_handle.cancel()
+      if exc_type is asyncio.CancelledError and self._task and self._task.cancelled():
+        raise asyncio.TimeoutError from exc
+      return False
+
+  def timeout(delay):
+    return Timeout(delay)
+
+
+class SimpleAnswerProvider:
+  def __init__(self):
+    self.stream = None
+
+  async def __call__(self, offer: StreamingOffer):
+    assert self.stream is None, "This may only be called once"
+
+    info = parse_info_from_offer(offer.sdp)
+
+    builder = WebRTCAnswerBuilder(offer.sdp)
+    for cam in offer.video:
+      builder.add_video_stream(cam, VideoStreamTrack())
+    if info.expected_audio_track:
+      builder.add_audio_stream(AudioStreamTrack())
+    if info.incoming_audio_track:
+      builder.offer_to_receive_audio_stream()
+
+    self.stream = builder.stream()
+    answer = await self.stream.start()
+
+    return answer
+
+
+@pytest.mark.asyncio
+class TestStreamIntegration:
+  @parameterized.expand([
+    # name, recv_cameras, recv_audio, messaging
+    ("multi_camera", ["driver", "wideRoad", "road"], False, False),
+    ("camera_and_audio", ["driver"], True, False),
+    ("camera_and__messaging", ["driver"], False, True),
+    ("camera_and_audio_and_messaging", ["driver", "wideRoad", "road"], True, True),
+  ])
+  async def test_multi_camera(self, name, cameras, recv_audio, add_messaging):
+    simple_answerer = SimpleAnswerProvider()
+    offer_builder = WebRTCOfferBuilder(simple_answerer)
+    for cam in cameras:
+      offer_builder.offer_to_receive_video_stream(cam)
+    if recv_audio:
+      offer_builder.offer_to_receive_audio_stream()
+    if add_messaging:
+      offer_builder.add_messaging()
+    stream = offer_builder.stream()
+
+    _ = await stream.start()
+    assert stream.is_started
+
+    try:
+      async with timeout(2):
+        await stream.wait_for_connection()
+    except TimeoutError:
+      pytest.fail("Timed out waiting for connection")
+    assert stream.is_connected_and_ready
+
+    assert stream.has_messaging_channel() == add_messaging
+    if stream.has_messaging_channel():
+      channel = stream.get_messaging_channel()
+      assert channel is not None
+      assert channel.readyState == "open"
+
+    assert stream.has_incoming_audio_track() == recv_audio
+    if stream.has_incoming_audio_track():
+      track = stream.get_incoming_audio_track(False)
+      assert track is not None
+      assert track.readyState == "live"
+      assert track.kind == "audio"
+      # test audio recv
+      try:
+        async with timeout(1):
+          await track.recv()
+      except TimeoutError:
+        pytest.fail("Timed out waiting for audio frame")
+
+    for cam in cameras:
+      assert stream.has_incoming_video_track(cam)
+      if stream.has_incoming_video_track(cam):
+        track = stream.get_incoming_video_track(cam, False)
+        assert track is not None
+        assert track.readyState == "live"
+        assert track.kind == "video"
+        # test video recv
+        try:
+          async with timeout(1):
+            await stream.get_incoming_video_track(cam, False).recv()
+        except TimeoutError:
+          pytest.fail("Timed out waiting for video frame")
+
+    await stream.stop()
+    await simple_answerer.stream.stop()
+    assert not stream.is_started
+    assert not stream.is_connected_and_ready

teleoprtc_repo/tests/test_stream.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+
+import pytest
+
+import aiortc
+from aiortc.mediastreams import AudioStreamTrack
+
+from teleoprtc.builder import WebRTCOfferBuilder, WebRTCAnswerBuilder
+from teleoprtc.info import parse_info_from_offer
+from teleoprtc.tracks import TiciVideoStreamTrack
+
+
+class OfferCapture:
+  def __init__(self):
+    self.offer = None
+
+  async def __call__(self, offer):
+    self.offer = offer
+    raise Exception("Offer captured")
+
+
+class DummyH264VideoStreamTrack(TiciVideoStreamTrack):
+  kind = "video"
+
+  async def recv(self):
+    raise NotImplementedError()
+
+  def codec_preference(self):
+    return "H264"
+
+
+@pytest.mark.asyncio
+class TestOfferStream:
+  async def test_offer_stream_sdp_recvonly_audio(self):
+    capture = OfferCapture()
+    builder = WebRTCOfferBuilder(capture)
+    builder.offer_to_receive_audio_stream()
+    stream = builder.stream()
+
+    try:
+      _ = await stream.start()
+    except Exception:
+      pass
+
+    info = parse_info_from_offer(capture.offer.sdp)
+    assert info.expected_audio_track
+    assert not info.incoming_audio_track
+
+  async def test_offer_stream_sdp_sendonly_audio(self):
+    capture = OfferCapture()
+    builder = WebRTCOfferBuilder(capture)
+    builder.add_audio_stream(AudioStreamTrack())
+    stream = builder.stream()
+
+    try:
+      _ = await stream.start()
+    except Exception:
+      pass
+
+    info = parse_info_from_offer(capture.offer.sdp)
+    assert not info.expected_audio_track
+    assert info.incoming_audio_track
+
+  async def test_offer_stream_sdp_channel(self):
+    capture = OfferCapture()
+    builder = WebRTCOfferBuilder(capture)
+    builder.add_messaging()
+    stream = builder.stream()
+
+    try:
+      _ = await stream.start()
+    except Exception:
+      pass
+
+    info = parse_info_from_offer(capture.offer.sdp)
+    assert info.incoming_datachannel
+
+
+@pytest.mark.asyncio
+class TestAnswerStream:
+  async def test_codec_preference(self):
+    offer_sdp = """v=0
+o=- 3910274679 3910274679 IN IP4 0.0.0.0
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=msid-semantic:WMS *
+m=video 1337 UDP/TLS/RTP/SAVPF 97 98 99 100 101 102
+c=IN IP4 0.0.0.0
+a=recvonly
+a=mid:0
+a=msid:34803878-98f8-4245-b45c-f773e5f926df 881dbc20-356a-499c-b4e8-695303bb901d
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-mux
+a=ssrc-group:FID 1303546896 3784011659
+a=ssrc:1303546896 cname:a59185ac-c115-48d3-b39b-db7d615a6966
+a=ssrc:3784011659 cname:a59185ac-c115-48d3-b39b-db7d615a6966
+a=rtpmap:97 VP8/90000
+a=rtcp-fb:97 nack
+a=rtcp-fb:97 nack pli
+a=rtcp-fb:97 goog-remb
+a=rtpmap:99 H264/90000
+a=rtcp-fb:99 nack
+a=rtcp-fb:99 nack pli
+a=rtcp-fb:99 goog-remb
+a=fmtp:99 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f
+a=ice-ufrag:1234
+a=ice-pwd:1234
+a=fingerprint:sha-256 15:F3:F0:23:67:44:EE:2C:AA:8C:D9:50:95:26:42:7C:67:EA:1F:D2:92:C5:97:01:7B:2E:57:C9:A3:13:00:4A
+a=setup:actpass"""
+
+    builder = WebRTCAnswerBuilder(offer_sdp)
+    builder.add_video_stream("road", DummyH264VideoStreamTrack("road", 0.05))
+    stream = builder.stream()
+    answer = await stream.start()
+
+    sdp_desc = aiortc.sdp.SessionDescription.parse(answer.sdp)
+    video_desc = [m for m in sdp_desc.media if m.kind == "video"][0]
+    codecs = video_desc.rtp.codecs
+    assert codecs[0].mimeType == "video/H264"
+
+  async def test_fail_if_preferred_codec_not_in_offer(self):
+    offer_sdp = """v=0
+o=- 3910274679 3910274679 IN IP4 0.0.0.0
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=msid-semantic:WMS *
+m=video 1337 UDP/TLS/RTP/SAVPF 97 98 99 100 101 102
+c=IN IP4 0.0.0.0
+a=recvonly
+a=mid:0
+a=msid:34803878-98f8-4245-b45c-f773e5f926df 881dbc20-356a-499c-b4e8-695303bb901d
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-mux
+a=ssrc-group:FID 1303546896 3784011659
+a=ssrc:1303546896 cname:a59185ac-c115-48d3-b39b-db7d615a6966
+a=ssrc:3784011659 cname:a59185ac-c115-48d3-b39b-db7d615a6966
+a=rtpmap:97 VP8/90000
+a=rtcp-fb:97 nack
+a=rtcp-fb:97 nack pli
+a=rtcp-fb:97 goog-remb
+a=ice-ufrag:1234
+a=ice-pwd:1234
+a=fingerprint:sha-256 15:F3:F0:23:67:44:EE:2C:AA:8C:D9:50:95:26:42:7C:67:EA:1F:D2:92:C5:97:01:7B:2E:57:C9:A3:13:00:4A
+a=setup:actpass"""
+
+    builder = WebRTCAnswerBuilder(offer_sdp)
+    builder.add_video_stream("road", DummyH264VideoStreamTrack("road", 0.05))
+    stream = builder.stream()
+
+    with pytest.raises(ValueError):
+      _ = await stream.start()

teleoprtc_repo/tests/test_track.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import pytest
+
+import aiortc
+
+from teleoprtc.tracks import video_track_id, parse_video_track_id, TiciVideoStreamTrack, TiciTrackWrapper
+
+
+class TestTracks:
+  def test_track_id(self):
+    expected_camera_type, expected_track_id = "driver", "test"
+    track_id = video_track_id(expected_camera_type, expected_track_id)
+    camera_type, track_id = parse_video_track_id(track_id)
+    assert expected_camera_type == camera_type
+    assert expected_track_id == track_id
+
+  def test_track_id_invalid(self):
+    with pytest.raises(ValueError):
+      parse_video_track_id("test")
+
+  def test_tici_track_id(self):
+    class VideoStream(TiciVideoStreamTrack):
+      async def recv(self):
+        raise NotImplementedError()
+
+    track = VideoStream("driver", 0.1)
+    camera_type, _ = parse_video_track_id(track.id)
+    assert "driver" == camera_type
+
+  def test_tici_wrapper_id(self):
+    track = TiciTrackWrapper("driver", aiortc.mediastreams.VideoStreamTrack())
+    camera_type, _ = parse_video_track_id(track.id)
+    assert "driver" == camera_type