IQ.Pilot Prebuilt Release @ 27f668a

iqpilot/system/loggerd/.gitignore

@@ -0,0 +1,4 @@
+loggerd
+encoderd
+bootlog
+tests/test_logger

iqpilot/system/loggerd/config.py

@@ -0,0 +1,31 @@
+import os
+from iqpilot.system.hardware.hw import Paths
+
+
+CAMERA_FPS = 20
+SEGMENT_LENGTH = 60
+
+
+PATH_DICT = {
+  "internal": Paths.log_root(),
+  "external": Paths.log_root_external()
+}
+
+def get_available_percent(default: float, path_type="internal") -> float:
+  try:
+    statvfs = os.statvfs(PATH_DICT[path_type])
+    available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
+  except (OSError, KeyError):
+    available_percent = default
+
+  return available_percent
+
+
+def get_available_bytes(default: int, path_type="internal") -> int:
+  try:
+    statvfs = os.statvfs(PATH_DICT[path_type])
+    available_bytes = statvfs.f_bavail * statvfs.f_frsize
+  except (OSError, KeyError):
+    available_bytes = default
+
+  return available_bytes

iqpilot/system/loggerd/crash_recovery.py

@@ -0,0 +1,45 @@
+import os
+
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.system.hardware.hw import Paths
+
+PRESERVE_ATTR_NAME = b"user.preserve"
+PRESERVE_ATTR_VALUE = b"1"
+
+
+def recover_unclean_segments(log_root: str | None = None) -> list[str]:
+  # Segments with leftover .lock files are from a loggerd that never closed
+  # cleanly (power cut, crash). The video/log data in them is valid up to the
+  # last durable sync. Clear the stale locks so the deleter can manage them
+  # again, and preserve them: footage from an unclean shutdown is exactly the
+  # footage a dashcam must not throw away.
+  root = log_root if log_root is not None else Paths.log_root()
+  recovered = []
+  try:
+    dirs = os.listdir(root)
+  except OSError:
+    return recovered
+
+  for d in dirs:
+    seg_path = os.path.join(root, d)
+    if not os.path.isdir(seg_path):
+      continue
+    try:
+      locks = [f for f in os.listdir(seg_path) if f.endswith(".lock")]
+      if not locks:
+        continue
+      for lock in locks:
+        os.unlink(os.path.join(seg_path, lock))
+      setxattr = getattr(os, "setxattr", None)  # not available on darwin
+      if setxattr is not None:
+        try:
+          setxattr(seg_path, PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE)
+        except OSError:
+          pass
+      recovered.append(d)
+    except OSError:
+      cloudlog.exception(f"crash_recovery: failed to recover {seg_path}")
+
+  if recovered:
+    cloudlog.event("crash_recovery.recovered_unclean_segments", segments=sorted(recovered), error=True)
+  return recovered

iqpilot/system/loggerd/deleter.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+import os
+import time
+import shutil
+import threading
+from pathlib import Path
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.common.swaglog import cloudlog
+from iqpilot.system.loggerd.config import get_available_bytes, get_available_percent
+from iqpilot.system.loggerd.uploader_common import listdir_by_creation
+from iqpilot.system.loggerd.xattr_cache import getxattr
+
+MIN_BYTES = 5 * 1024 * 1024 * 1024
+MIN_PERCENT = 10
+
+DELETE_LAST = ['boot', 'crash']
+
+PRESERVE_ATTR_NAME = 'user.preserve'
+PRESERVE_ATTR_VALUE = b'1'
+PRESERVE_COUNT = 5
+
+
+def has_preserve_xattr(d: str) -> bool:
+  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+
+def get_preserved_segments(dirs_by_creation: list[str]) -> set[str]:
+  # skip deleting most recent N preserved segments (and their prior segment)
+  preserved = set()
+  for n, d in enumerate(filter(has_preserve_xattr, reversed(dirs_by_creation))):
+    if n == PRESERVE_COUNT:
+      break
+    date_str, _, seg_str = d.rpartition("--")
+
+    # ignore non-segment directories
+    if not date_str:
+      continue
+    try:
+      seg_num = int(seg_str)
+    except ValueError:
+      continue
+
+    # preserve segment and two prior
+    for _seg_num in range(max(0, seg_num - 2), seg_num + 1):
+      preserved.add(f"{date_str}--{_seg_num}")
+
+  return preserved
+
+
+def deleter_thread(exit_event: threading.Event):
+  while not exit_event.is_set():
+    out_of_bytes = get_available_bytes(default=MIN_BYTES + 1) < MIN_BYTES
+    out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
+
+    if out_of_percent or out_of_bytes:
+      dirs = listdir_by_creation(Paths.log_root())
+      preserved_dirs = get_preserved_segments(dirs)
+
+      # remove the earliest directory we can
+      for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
+        delete_path = os.path.join(Paths.log_root(), delete_dir)
+
+        if any(name.endswith(".lock") for name in os.listdir(delete_path)):
+          continue
+
+        if Path(Paths.log_root_external()).is_mount():
+          out_of_bytes_external = get_available_bytes(default=MIN_BYTES + 1, path_type="external") < MIN_BYTES
+          out_of_percent_external = get_available_percent(default=MIN_PERCENT + 1, path_type="external") < MIN_PERCENT
+
+          if out_of_percent_external or out_of_bytes_external:
+            dirs_external = listdir_by_creation(Paths.log_root_external())
+
+            # remove the earliest external directory we can
+            for delete_dir_external in sorted(dirs_external):
+              delete_path_external = os.path.join(Paths.log_root_external(), delete_dir_external)
+              try:
+                cloudlog.warning(f"deleting {delete_path_external}")
+                shutil.rmtree(delete_path_external)
+                break
+              except OSError:
+                cloudlog.exception(f"issue deleting {delete_path_external}")
+
+          # move directory from internal to external
+          path_external = os.path.join(Paths.log_root_external(), delete_dir)
+          try:
+            cloudlog.warning(f"moving {delete_path} to {path_external}")
+            start = time.monotonic()
+            shutil.move(delete_path, path_external)
+            cloudlog.warning(f"moved {delete_path} to {path_external} in {time.monotonic() - start:.2f}s")
+            break
+          except Exception:
+            cloudlog.error(f"issue moving {delete_path} to {path_external}")
+            try:
+              cloudlog.warning(f"deleting {delete_path}")
+              shutil.rmtree(delete_path)
+              break
+            except OSError:
+              cloudlog.exception(f"issue deleting {delete_path}")
+          continue
+
+        try:
+          cloudlog.info(f"deleting {delete_path}")
+          shutil.rmtree(delete_path)
+          break
+        except OSError:
+          cloudlog.exception(f"issue deleting {delete_path}")
+      exit_event.wait(.1)
+    else:
+      exit_event.wait(30)
+
+
+def main():
+  deleter_thread(threading.Event())
+
+
+if __name__ == "__main__":
+  main()

iqpilot/system/loggerd/tests/deleter_tests_common.py

@@ -0,0 +1,57 @@
+import os
+import random
+import shutil
+from pathlib import Path
+
+import iqpilot.system.loggerd.deleter as deleter
+from iqpilot.common.params import Params
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.system.loggerd.xattr_cache import setxattr
+
+
+def create_random_file(file_path: Path, size_mb: float, lock: bool = False) -> None:
+  file_path.parent.mkdir(parents=True, exist_ok=True)
+
+  if lock:
+    lock_path = str(file_path) + ".lock"
+    os.close(os.open(lock_path, os.O_CREAT | os.O_EXCL))
+
+  chunks = 128
+  chunk_bytes = int(size_mb * 1024 * 1024 / chunks)
+  data = os.urandom(chunk_bytes)
+
+  with open(file_path, "wb") as f:
+    for _ in range(chunks):
+      f.write(data)
+
+
+class DeleterTestCase:
+  f_type = "UNKNOWN"
+
+  root: Path
+  seg_num: int
+  seg_format: str
+  seg_format2: str
+  seg_dir: str
+
+  def setup_method(self):
+    shutil.rmtree(Paths.log_root(), ignore_errors=True)
+    Path(Paths.log_root()).mkdir(parents=True, exist_ok=True)
+    self.seg_num = random.randint(1, 300)
+    self.seg_format = "00000004--0ac3964c96--{}"
+    self.seg_format2 = "00000005--4c4e99b08b--{}"
+    self.seg_dir = self.seg_format.format(self.seg_num)
+
+    self.params = Params()
+    self.params.put("IsOffroad", True)
+    self.params.put("DongleId", "0000000000000000")
+
+  def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False,
+                          preserve_xattr: bytes | None = None) -> Path:
+    file_path = Path(Paths.log_root()) / f_dir / fn
+    create_random_file(file_path, size_mb, lock)
+
+    if preserve_xattr is not None:
+      setxattr(str(file_path.parent), deleter.PRESERVE_ATTR_NAME, preserve_xattr)
+
+    return file_path

iqpilot/system/loggerd/tests/test_deleter.py

@@ -0,0 +1,117 @@
+import time
+import threading
+from collections import namedtuple
+from pathlib import Path
+from collections.abc import Sequence
+
+import iqpilot.system.loggerd.deleter as deleter
+from iqpilot.common.timeout import Timeout, TimeoutException
+from iqpilot.system.loggerd.tests.deleter_tests_common import DeleterTestCase
+
+Stats = namedtuple("Stats", ['f_bavail', 'f_blocks', 'f_frsize'])
+
+
+class TestDeleter(DeleterTestCase):
+  def fake_statvfs(self, d):
+    return self.fake_stats
+
+  def setup_method(self):
+    self.f_type = "fcamera.hevc"
+    super().setup_method()
+    self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096)
+    deleter.os.statvfs = self.fake_statvfs
+
+  def start_thread(self):
+    self.end_event = threading.Event()
+    self.del_thread = threading.Thread(target=deleter.deleter_thread, args=[self.end_event])
+    self.del_thread.daemon = True
+    self.del_thread.start()
+
+  def join_thread(self):
+    self.end_event.set()
+    self.del_thread.join()
+
+  def test_delete(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type, 1)
+
+    self.start_thread()
+
+    try:
+      with Timeout(2, "Timeout waiting for file to be deleted"):
+        while f_path.exists():
+          time.sleep(0.01)
+    finally:
+      self.join_thread()
+
+  def assertDeleteOrder(self, f_paths: Sequence[Path], timeout: int = 5) -> None:
+    deleted_order = []
+
+    self.start_thread()
+    try:
+      with Timeout(timeout, "Timeout waiting for files to be deleted"):
+        while True:
+          for f in f_paths:
+            if not f.exists() and f not in deleted_order:
+              deleted_order.append(f)
+          if len(deleted_order) == len(f_paths):
+            break
+          time.sleep(0.01)
+    except TimeoutException:
+      print("Not deleted:", [f for f in f_paths if f not in deleted_order])
+      raise
+    finally:
+      self.join_thread()
+
+    assert deleted_order == f_paths, "Files not deleted in expected order"
+
+  def test_delete_order(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+    ])
+
+  def test_delete_many_preserved(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(1), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data(self.seg_format.format(2), self.f_type),
+    ] + [
+      self.make_file_with_data(self.seg_format2.format(i), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE)
+      for i in range(5)
+    ])
+
+  def test_delete_last(self):
+    self.assertDeleteOrder([
+      self.make_file_with_data(self.seg_format.format(1), self.f_type),
+      self.make_file_with_data(self.seg_format2.format(0), self.f_type),
+      self.make_file_with_data(self.seg_format.format(0), self.f_type, preserve_xattr=deleter.PRESERVE_ATTR_VALUE),
+      self.make_file_with_data("boot", self.seg_format[:-4]),
+      self.make_file_with_data("crash", self.seg_format2[:-4]),
+    ])
+
+  def test_no_delete_when_available_space(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type)
+
+    block_size = 4096
+    available = (10 * 1024 * 1024 * 1024) / block_size  # 10GB free
+    self.fake_stats = Stats(f_bavail=available, f_blocks=10, f_frsize=block_size)
+
+    self.start_thread()
+    start_time = time.monotonic()
+    while f_path.exists() and time.monotonic() - start_time < 2:
+      time.sleep(0.01)
+    self.join_thread()
+
+    assert f_path.exists(), "File deleted with available space"
+
+  def test_no_delete_with_lock_file(self):
+    f_path = self.make_file_with_data(self.seg_dir, self.f_type, lock=True)
+
+    self.start_thread()
+    start_time = time.monotonic()
+    while f_path.exists() and time.monotonic() - start_time < 2:
+      time.sleep(0.01)
+    self.join_thread()
+
+    assert f_path.exists(), "File deleted when locked"

iqpilot/system/loggerd/tests/test_encoder.py

@@ -0,0 +1,152 @@
+import math
+import os
+import pytest
+import random
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+from parameterized import parameterized
+from tqdm import trange
+
+from iqpilot.common.params import Params
+from iqpilot.common.timeout import Timeout
+from iqpilot.system.hardware import TICI
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.tools.lib.logreader import LogReader
+from iqpilot.system.hardware.hw import Paths
+
+SEGMENT_LENGTH = 2
+FULL_SIZE = 2507572
+def hevc_size(w): return FULL_SIZE // 2 if w <= 1344 else FULL_SIZE
+CAMERAS = [
+  ("fcamera.hevc", 20, hevc_size, "roadEncodeIdx"),
+  ("dcamera.hevc", 20, hevc_size, "driverEncodeIdx"),
+  ("ecamera.hevc", 20, hevc_size, "wideRoadEncodeIdx"),
+  ("qcamera.ts", 20, lambda x: 130000, None),
+]
+
+# we check frame count, so we don't have to be too strict on size
+FILE_SIZE_TOLERANCE = 0.7
+
+
+@pytest.mark.tici # TODO: all of loggerd should work on PC
+class TestEncoder:
+
+  def setup_method(self):
+    self._clear_logs()
+    os.environ["LOGGERD_TEST"] = "1"
+    os.environ["LOGGERD_SEGMENT_LENGTH"] = str(SEGMENT_LENGTH)
+
+  def teardown_method(self):
+    self._clear_logs()
+
+  def _clear_logs(self):
+    if os.path.exists(Paths.log_root()):
+      shutil.rmtree(Paths.log_root())
+
+  def _get_latest_segment_path(self):
+    last_route = sorted(Path(Paths.log_root()).iterdir())[-1]
+    return os.path.join(Paths.log_root(), last_route)
+
+  # TODO: this should run faster than real time
+  @parameterized.expand([(True, ), (False, )])
+  def test_log_rotation(self, record_front):
+    Params().put_bool("RecordFront", record_front)
+
+    managed_processes['sensord'].start()
+    managed_processes['loggerd'].start()
+    managed_processes['encoderd'].start()
+
+    time.sleep(1.0)
+    managed_processes['camerad'].start()
+
+    num_segments = int(os.getenv("SEGMENTS", random.randint(2, 8)))
+
+    # wait for loggerd to make the dir for first segment
+    route_prefix_path = None
+    with Timeout(int(SEGMENT_LENGTH*3)):
+      while route_prefix_path is None:
+        try:
+          route_prefix_path = self._get_latest_segment_path().rsplit("--", 1)[0]
+        except Exception:
+          time.sleep(0.1)
+
+    def check_seg(i):
+      # check each camera file size
+      counts = []
+      first_frames = []
+      for camera, fps, size_lambda, encode_idx_name in CAMERAS:
+        if not record_front and "dcamera" in camera:
+          continue
+
+        file_path = f"{route_prefix_path}--{i}/{camera}"
+
+        # check file exists
+        assert os.path.exists(file_path), f"segment #{i}: '{file_path}' missing"
+
+        # TODO: this ffprobe call is really slow
+        # get width and check frame count
+        cmd = f"ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets,width -of csv=p=0 {file_path}"
+        if TICI:
+          cmd = "LD_LIBRARY_PATH=/usr/local/lib " + cmd
+
+        expected_frames = fps * SEGMENT_LENGTH
+        probe = subprocess.check_output(cmd, shell=True, encoding='utf8').split('\n')[0].strip().split(',')
+        frame_width, frame_count = int(probe[0]), int(probe[1])
+        counts.append(frame_count)
+
+        assert frame_count == expected_frames, \
+                         f"segment #{i}: {camera} failed frame count check: expected {expected_frames}, got {frame_count}"
+
+        # sanity check file size
+        file_size = os.path.getsize(file_path)
+        target_size = size_lambda(frame_width)
+        assert math.isclose(file_size, target_size, rel_tol=FILE_SIZE_TOLERANCE), \
+                        f"{file_path} size {file_size} isn't close to target size {target_size}"
+
+        # Check encodeIdx
+        if encode_idx_name is not None:
+          rlog_path = f"{route_prefix_path}--{i}/rlog.zst"
+          msgs = [m for m in LogReader(rlog_path) if m.which() == encode_idx_name]
+          encode_msgs = [getattr(m, encode_idx_name) for m in msgs]
+
+          valid = [m.valid for m in msgs]
+          segment_idxs = [m.segmentId for m in encode_msgs]
+          encode_idxs = [m.encodeId for m in encode_msgs]
+          frame_idxs = [m.frameId for m in encode_msgs]
+
+          # Check frame count
+          assert frame_count == len(segment_idxs)
+          assert frame_count == len(encode_idxs)
+
+          # Check for duplicates or skips
+          assert 0 == segment_idxs[0]
+          assert len(set(segment_idxs)) == len(segment_idxs)
+
+          assert all(valid)
+
+          assert expected_frames * i == encode_idxs[0]
+          first_frames.append(frame_idxs[0])
+          assert len(set(encode_idxs)) == len(encode_idxs)
+
+      assert 1 == len(set(first_frames))
+
+      if TICI:
+        expected_frames = fps * SEGMENT_LENGTH
+        assert min(counts) == expected_frames
+      shutil.rmtree(f"{route_prefix_path}--{i}")
+
+    try:
+      for i in trange(num_segments):
+        # poll for next segment
+        with Timeout(int(SEGMENT_LENGTH*10), error_msg=f"timed out waiting for segment {i}"):
+          while Path(f"{route_prefix_path}--{i+1}") not in Path(Paths.log_root()).iterdir():
+            time.sleep(0.1)
+        check_seg(i)
+    finally:
+      managed_processes['loggerd'].stop()
+      managed_processes['encoderd'].stop()
+      managed_processes['camerad'].stop()
+      managed_processes['sensord'].stop()

iqpilot/system/loggerd/tests/test_loggerd.py

@@ -0,0 +1,377 @@
+import numpy as np
+import os
+import re
+import random
+import string
+import subprocess
+import time
+from collections import defaultdict
+from pathlib import Path
+import pytest
+
+import iqpilot.cereal.messaging as messaging
+from iqpilot.cereal import log
+from iqpilot.cereal.services import SERVICE_LIST
+from iqpilot.common.basedir import BASEDIR
+from iqpilot.common.params import Params
+from iqpilot.common.timeout import Timeout
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.system.hardware import TICI
+from iqpilot.system.loggerd.xattr_cache import getxattr
+from iqpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE
+from iqpilot.system.manager.process_config import managed_processes
+from iqpilot.system.version import get_version
+from iqpilot.tools.lib.route import RE
+from iqpilot.tools.lib.logreader import LogReader
+from iqpilot.cereal.visionipc import VisionStreamType
+from msgq.visionipc import VisionIpcServer
+
+SentinelType = log.Sentinel.SentinelType
+
+CEREAL_SERVICES = [f for f in log.Event.schema.union_fields if f in SERVICE_LIST
+                   and SERVICE_LIST[f].should_log and "encode" not in f.lower()]
+
+
+class TestLoggerd:
+  def _get_latest_log_dir(self):
+    log_dirs = sorted(Path(Paths.log_root()).iterdir(), key=lambda f: f.stat().st_mtime)
+    return log_dirs[-1]
+
+  def _get_log_dir(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_dir():
+          return path
+    return None
+
+  def _get_log_fn(self, x):
+    for l in x.splitlines():
+      for p in l.split(' '):
+        path = Path(p.strip())
+        if path.is_file():
+          return path
+    return None
+
+  def _gen_bootlog(self):
+    with Timeout(5):
+      out = subprocess.check_output("./bootlog", cwd=os.path.join(BASEDIR, "iqpilot/system/loggerd"), encoding='utf-8')
+
+    log_fn = self._get_log_fn(out)
+
+    # check existence
+    assert log_fn is not None
+
+    return log_fn
+
+  def _check_init_data(self, msgs):
+    msg = msgs[0]
+    assert msg.which() == 'initData'
+
+  def _check_sentinel(self, msgs, route):
+    start_type = SentinelType.startOfRoute if route else SentinelType.startOfSegment
+    assert msgs[1].sentinel.type == start_type
+
+    end_type = SentinelType.endOfRoute if route else SentinelType.endOfSegment
+    assert msgs[-1].sentinel.type == end_type
+
+  def _publish_random_messages(self, services: list[str]) -> dict[str, list]:
+    pm = messaging.PubMaster(services)
+
+    managed_processes["loggerd"].start()
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+
+    sent_msgs = defaultdict(list)
+    for i in range(random.randint(2, 10) * 100):
+      for s in services:
+        try:
+          m = messaging.new_message(s)
+        except Exception:
+          m = messaging.new_message(s, random.randint(2, 10))
+        pm.send(s, m)
+        sent_msgs[s].append(m)
+
+      if (i + 1) % 100 == 0:
+        for s in services:
+          assert pm.wait_for_readers_to_update(s, timeout=5)
+
+    for s in services:
+      assert pm.wait_for_readers_to_update(s, timeout=5)
+    assert managed_processes["loggerd"].stop(timeout=30) == 0
+
+    return sent_msgs
+
+  def _publish_camera_and_audio_messages(self, num_segs=1, segment_length=5):
+    # Use small frame sizes for testing (width, height, size, stride, uv_offset)
+    # NV12 format: size = stride * height * 1.5, uv_offset = stride * height
+    w, h = 320, 240
+    frame_spec = (w, h, w * h * 3 // 2, w, w * h)
+    streams = [
+      (VisionStreamType.VISION_STREAM_ROAD, frame_spec, "roadCameraState"),
+      (VisionStreamType.VISION_STREAM_DRIVER, frame_spec, "driverCameraState"),
+      (VisionStreamType.VISION_STREAM_WIDE_ROAD, frame_spec, "wideRoadCameraState"),
+    ]
+
+    sm = messaging.SubMaster(["roadEncodeData"])
+    pm = messaging.PubMaster([s for _, _, s in streams] + ["rawAudioData"])
+    vipc_server = VisionIpcServer("camerad")
+    for stream_type, frame_spec, _ in streams:
+      vipc_server.create_buffers_with_sizes(stream_type, 40, *(frame_spec))
+    vipc_server.start_listener()
+
+    encoderd_ret = None
+    loggerd_ret = None
+    try:
+      os.environ["LOGGERD_TEST"] = "1"
+      os.environ["LOGGERD_SEGMENT_LENGTH"] = str(segment_length)
+      managed_processes["loggerd"].start()
+      managed_processes["encoderd"].start()
+      for _, _, state in streams:
+        assert pm.wait_for_readers_to_update(state, timeout=5)
+
+      fps = 20
+      for n in range(1, int(num_segs * segment_length * fps) + 1):
+        # send video
+        for stream_type, frame_spec, state in streams:
+          dat = np.empty(frame_spec[2], dtype=np.uint8)
+          vipc_server.send(stream_type, dat[:].flatten().tobytes(), n, n / fps, n / fps)
+
+          camera_state = messaging.new_message(state)
+          frame = getattr(camera_state, state)
+          frame.frameId = n
+          pm.send(state, camera_state)
+
+        # send audio
+        msg = messaging.new_message('rawAudioData')
+        msg.rawAudioData.data = bytes(800 * 2) # 800 samples of int16
+        msg.rawAudioData.sampleRate = 16000
+        pm.send('rawAudioData', msg)
+
+        for _, _, state in streams:
+          assert pm.wait_for_readers_to_update(state, timeout=5, dt=0.001)
+
+        sm.update(100)
+    finally:
+      encoderd_ret = managed_processes["encoderd"].stop(timeout=30)
+      loggerd_ret = managed_processes["loggerd"].stop(timeout=30)
+      del vipc_server
+
+    assert encoderd_ret == 0
+    assert loggerd_ret == 0
+
+  def test_init_data_values(self):
+    os.environ["CLEAN"] = random.choice(["0", "1"])
+
+    dongle  = ''.join(random.choice(string.printable) for n in range(random.randint(1, 100)))
+    fake_params = [
+      # param, initData field, value
+      ("DongleId", "dongleId", dongle),
+      ("GitCommit", "gitCommit", "commit"),
+      ("GitCommitDate", "gitCommitDate", "date"),
+      ("GitBranch", "gitBranch", "branch"),
+      ("GitRemote", "gitRemote", "remote"),
+    ]
+    params = Params()
+    for k, _, v in fake_params:
+      params.put(k, v)
+    params.put("AccessToken", "abc")
+
+    lr = list(LogReader(str(self._gen_bootlog())))
+    initData = lr[0].initData
+
+    assert initData.dirty != bool(os.environ["CLEAN"])
+    assert initData.version == get_version()
+
+    if TICI:
+      assert initData._has("ufsHealth")
+      assert initData.ufsHealth.preEolInfo in (1, 2, 3)
+      assert 1 <= initData.ufsHealth.lifeTimeEstimateA <= 11
+      assert 1 <= initData.ufsHealth.lifeTimeEstimateB <= 11
+      assert len(initData.ufsHealth.vendorHealthReport) == 32
+    else:
+      assert not initData._has("ufsHealth")
+
+    if os.path.isfile("/proc/cmdline"):
+      with open("/proc/cmdline") as f:
+        assert list(initData.kernelArgs) == f.read().strip().split(" ")
+
+      with open("/proc/version") as f:
+        assert initData.kernelVersion == f.read()
+
+    # check params
+    logged_params = {entry.key: entry.value for entry in initData.params.entries}
+    expected_params = {k for k, _, __ in fake_params} | {'AccessToken', 'BootCount'}
+    assert set(logged_params.keys()) == expected_params, set(logged_params.keys()) ^ expected_params
+    assert logged_params['AccessToken'] == b'', f"DONT_LOG param value was logged: {repr(logged_params['AccessToken'])}"
+    for param_key, initData_key, v in fake_params:
+      assert getattr(initData, initData_key) == v
+      assert logged_params[param_key].decode() == v
+
+  @pytest.mark.xdist_group("camera_encoder_tests")  # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
+  def test_rotation(self):
+    Params().put_bool("RecordFront", True)
+
+    expected_files = {"rlog.zst", "qlog.zst", "qcamera.ts", "fcamera.hevc", "dcamera.hevc", "ecamera.hevc"}
+
+    num_segs = random.randint(2, 3)
+    length = random.randint(4, 5) # H264 encoder uses 40 lookahead frames and does B-frame reordering, so minimum 3 seconds before qcam output
+
+    self._publish_camera_and_audio_messages(num_segs=num_segs, segment_length=length)
+
+    route_path = str(self._get_latest_log_dir()).rsplit("--", 1)[0]
+    for n in range(num_segs):
+      p = Path(f"{route_path}--{n}")
+      logged = {f.name for f in p.iterdir() if f.is_file()}
+      diff = logged ^ expected_files
+      assert len(diff) == 0, f"didn't get all expected files. seg={n} {route_path=}, {diff=}\n{logged=} {expected_files=}"
+
+  def test_bootlog(self):
+    # generate bootlog with fake launch log
+    launch_log = ''.join(str(random.choice(string.printable)) for _ in range(100))
+    with open("/tmp/launch_log", "w") as f:
+      f.write(launch_log)
+
+    bootlog_path = self._gen_bootlog()
+    lr = list(LogReader(str(bootlog_path)))
+
+    # check length
+    assert len(lr) == 2  # boot + initData
+
+    self._check_init_data(lr)
+
+    # check msgs
+    bootlog_msgs = [m for m in lr if m.which() == 'boot']
+    assert len(bootlog_msgs) == 1
+
+    # sanity check values
+    boot = bootlog_msgs.pop().boot
+    assert abs(boot.wallTimeNanos - time.time_ns()) < 5*1e9 # within 5s
+    assert boot.launchLog == launch_log
+
+    if TICI:
+      for fn in ["console-ramoops", "pmsg-ramoops-0"]:
+        path = Path(os.path.join("/sys/fs/pstore/", fn))
+        if path.is_file():
+          with open(path, "rb") as f:
+            expected_val = f.read()
+          bootlog_val = [e.value for e in boot.pstore.entries if e.key == fn][0]
+          assert expected_val == bootlog_val
+    else:
+      assert len(boot.pstore.entries) == 0
+
+    # next one should increment by one
+    bl1 = re.match(RE.LOG_ID_V2, bootlog_path.name)
+    bl2 = re.match(RE.LOG_ID_V2, self._gen_bootlog().name)
+    assert bl1.group('uid') != bl2.group('uid')
+    assert int(bl1.group('count')) == 0 and int(bl2.group('count')) == 1
+
+  def test_qlog(self):
+    qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is not None]
+    no_qlog_services = [s for s in CEREAL_SERVICES if SERVICE_LIST[s].decimation is None]
+
+    services = random.sample(qlog_services, random.randint(2, min(10, len(qlog_services)))) + \
+               random.sample(no_qlog_services, random.randint(2, min(10, len(no_qlog_services))))
+    sent_msgs = self._publish_random_messages(services)
+
+    qlog_path = os.path.join(self._get_latest_log_dir(), "qlog.zst")
+    lr = list(LogReader(qlog_path))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    recv_msgs = defaultdict(list)
+    for m in lr:
+      recv_msgs[m.which()].append(m)
+
+    for s, msgs in sent_msgs.items():
+      recv_cnt = len(recv_msgs[s])
+
+      if s in no_qlog_services:
+        # check services with no specific decimation aren't in qlog
+        assert recv_cnt == 0, f"got {recv_cnt} {s} msgs in qlog"
+      else:
+        # check logged message count matches decimation
+        expected_cnt = (len(msgs) - 1) // SERVICE_LIST[s].decimation + 1
+        assert recv_cnt == expected_cnt, f"expected {expected_cnt} msgs for {s}, got {recv_cnt}"
+
+  def test_rlog(self):
+    services = random.sample(CEREAL_SERVICES, random.randint(5, 10))
+    sent_msgs = self._publish_random_messages(services)
+
+    lr = list(LogReader(os.path.join(self._get_latest_log_dir(), "rlog.zst")))
+
+    # check initData and sentinel
+    self._check_init_data(lr)
+    self._check_sentinel(lr, True)
+
+    # check all messages were logged and in order
+    lr = lr[2:-1] # slice off initData and both sentinels
+    for m in lr:
+      sent = sent_msgs[m.which()].pop(0)
+      sent.clear_write_flag()
+      assert sent.to_bytes() == m.as_builder().to_bytes()
+
+  def test_preserving_bookmarked_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) | {"userBookmark"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+
+  def test_not_preserving_nonbookmarked_segments(self):
+    services = set(random.sample(CEREAL_SERVICES, random.randint(5, 10))) - {"userBookmark", "audioFeedback"}
+    self._publish_random_messages(services)
+
+    segment_dir = self._get_latest_log_dir()
+    assert getxattr(segment_dir, PRESERVE_ATTR_NAME) is None
+
+  @pytest.mark.xdist_group("camera_encoder_tests")  # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
+  @pytest.mark.parametrize("record_front", [True, False])
+  def test_record_front(self, record_front):
+    params = Params()
+    params.put_bool("RecordFront", record_front)
+
+    self._publish_camera_and_audio_messages()
+
+    dcamera_hevc_exists = os.path.exists(os.path.join(self._get_latest_log_dir(), 'dcamera.hevc'))
+    assert dcamera_hevc_exists == record_front
+
+  @pytest.mark.xdist_group("camera_encoder_tests")  # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
+  @pytest.mark.parametrize("record_audio", [True, False])
+  def test_record_audio(self, record_audio):
+    params = Params()
+    params.put_bool("RecordAudio", record_audio)
+
+    self._publish_camera_and_audio_messages()
+
+    qcamera_ts_path = os.path.join(self._get_latest_log_dir(), 'qcamera.ts')
+    ffprobe_cmd = f"ffprobe -i {qcamera_ts_path} -show_streams -select_streams a -loglevel error"
+    has_audio_stream = subprocess.run(ffprobe_cmd, shell=True, capture_output=True).stdout.strip() != b''
+    assert has_audio_stream == record_audio
+
+    raw_audio_in_rlog = any(m.which() == 'rawAudioData' for m in LogReader(os.path.join(self._get_latest_log_dir(), 'rlog.zst')))
+    assert raw_audio_in_rlog == record_audio
+
+  @pytest.mark.xdist_group("camera_encoder_tests")  # setting xdist group ensures tests are run in same worker, prevents encoderd from crashing
+  def test_record_audio_init_failure_fails_open(self):
+    params = Params()
+    params.put_bool("RecordAudio", True)
+
+    os.environ["LOGGERD_TEST_AUDIO_INIT_FAIL"] = "1"
+    try:
+      self._publish_camera_and_audio_messages()
+    finally:
+      os.environ.pop("LOGGERD_TEST_AUDIO_INIT_FAIL", None)
+
+    latest_log_dir = self._get_latest_log_dir()
+    qcamera_ts_path = os.path.join(latest_log_dir, 'qcamera.ts')
+    assert os.path.exists(qcamera_ts_path)
+
+    ffprobe_cmd = f"ffprobe -i {qcamera_ts_path} -show_streams -select_streams a -loglevel error"
+    has_audio_stream = subprocess.run(ffprobe_cmd, shell=True, capture_output=True).stdout.strip() != b''
+    assert has_audio_stream is False
+
+    raw_audio_in_rlog = any(m.which() == 'rawAudioData' for m in LogReader(os.path.join(latest_log_dir, 'rlog.zst')))
+    assert raw_audio_in_rlog is True

iqpilot/system/loggerd/tests/vidc_debug.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+cd /sys/kernel/debug/tracing
+echo "" > trace
+echo 1 > tracing_on
+echo 1 > /sys/kernel/debug/tracing/events/msm_vidc/enable
+
+echo 0xff > /sys/module/videobuf2_core/parameters/debug
+echo 0x7fffffff > /sys/kernel/debug/msm_vidc/debug_level
+echo 0xff > /sys/devices/platform/soc/aa00000.qcom,vidc/video4linux/video33/dev_debug
+
+cat /sys/kernel/debug/tracing/trace_pipe

iqpilot/system/loggerd/uploader_common.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+import os
+
+from iqpilot.common.swaglog import cloudlog
+
+
+def get_directory_sort(d: str) -> list[str]:
+  prefix = ["0"] if d.startswith("2024-") else ["1"]
+  return prefix + [s.rjust(10, "0") for s in d.rsplit("--", 1)]
+
+
+def listdir_by_creation(d: str) -> list[str]:
+  if not os.path.isdir(d):
+    return []
+
+  try:
+    paths = [f for f in os.listdir(d) if os.path.isdir(os.path.join(d, f))]
+    return sorted(paths, key=get_directory_sort)
+  except OSError:
+    cloudlog.exception("uploader_common.listdir_by_creation_failed")
+    return []

iqpilot/system/loggerd/xattr_cache.py

@@ -0,0 +1,28 @@
+import errno
+import os
+
+import xattr
+
+_cached_attributes: dict[tuple[str, str], tuple[tuple[int, int, int], bytes | None]] = {}
+
+def getxattr(path: str, attr_name: str) -> bytes | None:
+  key = (path, attr_name)
+  st = os.stat(path)
+  identity = (st.st_dev, st.st_ino, st.st_ctime_ns)
+  cached = _cached_attributes.get(key)
+  if cached is None or cached[0] != identity:
+    try:
+      response = xattr.getxattr(path, attr_name)
+    except OSError as e:
+      # ENODATA (Linux) or ENOATTR (macOS) means attribute hasn't been set
+      if e.errno == errno.ENODATA or (hasattr(errno, 'ENOATTR') and e.errno == errno.ENOATTR):
+        response = None
+      else:
+        raise
+    _cached_attributes[key] = (identity, response)
+  return _cached_attributes[key][1]
+
+def setxattr(path: str, attr_name: str, attr_value: bytes) -> None:
+  xattr.setxattr(path, attr_name, attr_value)
+  st = os.stat(path)
+  _cached_attributes[(path, attr_name)] = ((st.st_dev, st.st_ino, st.st_ctime_ns), attr_value)