IQ.Pilot Release Commit @ bec7652

iqpilot/tools/lib/tests/test_caching.py

@@ -0,0 +1,191 @@
+import http.server
+import multiprocessing
+import os
+import shutil
+import socket
+import tempfile
+import pytest
+
+from iqpilot.selfdrive.test.helpers import http_server_context
+from iqpilot.system.hardware.hw import Paths
+from iqpilot.tools.lib.url_file import URLFile, prune_cache
+import iqpilot.tools.lib.url_file as url_file_module
+
+
+def concurrent_prune_cache(cache_root, entry, barrier):
+  Paths.download_cache_root = staticmethod(lambda: cache_root)
+  barrier.wait()
+  prune_cache(entry)
+
+
+class CachingTestRequestHandler(http.server.BaseHTTPRequestHandler):
+  FILE_EXISTS = True
+
+  def do_GET(self):
+    if self.FILE_EXISTS:
+      self.send_response(206 if "Range" in self.headers else 200, b'1234')
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+  def do_HEAD(self):
+    if self.FILE_EXISTS:
+      self.send_response(200)
+      self.send_header("Content-Length", "4")
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+
+@pytest.fixture
+def host():
+  with http_server_context(handler=CachingTestRequestHandler) as (host, port):
+    yield f"http://{host}:{port}"
+
+class TestFileDownload:
+
+  def test_pipeline_defaults(self, host):
+    # TODO: parameterize the defaults so we don't rely on hard-coded values in xx
+
+    assert URLFile.pool_manager().pools._maxsize == 10# PoolManager num_pools param
+    pool_manager_defaults = {
+      "maxsize": 100,
+      "socket_options": [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),],
+    }
+    for k, v in pool_manager_defaults.items():
+      assert URLFile.pool_manager().connection_pool_kw.get(k) == v
+
+    retry_defaults = {
+      "total": 6,
+      "backoff_factor": 0.75,
+      "status_forcelist": [409, 429, 500, 502, 503, 504],
+    }
+    for k, v in retry_defaults.items():
+      assert getattr(URLFile.pool_manager().connection_pool_kw["retries"], k) == v
+
+    # ensure caching on by default and cache dir gets created
+    os.environ.pop("DISABLE_FILEREADER_CACHE", None)
+    if os.path.exists(Paths.download_cache_root()):
+      shutil.rmtree(Paths.download_cache_root())
+    URLFile(f"{host}/test.txt").get_length()
+    URLFile(f"{host}/test.txt").read()
+    assert os.path.exists(Paths.download_cache_root())
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    assert file_cached.get_length() == file_downloaded.get_length()
+    assert length + start if length is not None else 0 <= file_downloaded.get_length()
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    assert response_cached == response_downloaded
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    assert file_cached.get_length() == file_downloaded.get_length()
+    assert response_cached == response_downloaded
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ.pop("DISABLE_FILEREADER_CACHE", None)
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+  @pytest.mark.parametrize("cache_enabled", [True, False])
+  def test_recover_from_missing_file(self, host, cache_enabled):
+    if cache_enabled:
+      os.environ.pop("DISABLE_FILEREADER_CACHE", None)
+    else:
+      os.environ["DISABLE_FILEREADER_CACHE"] = "1"
+
+    file_url = f"{host}/test.png"
+
+    CachingTestRequestHandler.FILE_EXISTS = False
+    length = URLFile(file_url).get_length()
+    assert length == -1
+
+    CachingTestRequestHandler.FILE_EXISTS = True
+    length = URLFile(file_url).get_length()
+    assert length == 4
+
+
+class TestCache:
+  def test_concurrent_prune_cache(self, tmp_path):
+    context = multiprocessing.get_context("fork")
+    barrier = context.Barrier(16)
+    processes = [context.Process(target=concurrent_prune_cache, args=(f"{tmp_path}/", f"entry_{i}", barrier)) for i in range(16)]
+    for process in processes:
+      process.start()
+    for process in processes:
+      process.join(10)
+      assert process.exitcode == 0
+
+    manifest = set()
+    for line in (tmp_path / "manifest.txt").read_text().splitlines():
+      parts = line.split()
+      if len(parts) == 2:
+        manifest.add(parts[0])
+    assert manifest == {f"entry_{i}" for i in range(16)}
+
+  def test_prune_cache(self, monkeypatch):
+    with tempfile.TemporaryDirectory() as tmpdir:
+      monkeypatch.setattr(Paths, 'download_cache_root', staticmethod(lambda: tmpdir + "/"))
+
+      # setup test files and manifest
+      manifest_lines = []
+      for i in range(3):
+        fname = f"hash_{i}"
+        with open(tmpdir + "/" + fname, "wb") as f:
+          f.truncate(1000)
+        manifest_lines.append(f"{fname} {1000 + i}")
+      with open(tmpdir + "/manifest.txt", "w") as f:
+        f.write('\n'.join(manifest_lines))
+
+      # under limit, shouldn't prune
+      assert len(os.listdir(tmpdir)) == 4
+      prune_cache()
+      assert len([name for name in os.listdir(tmpdir) if name != "manifest.lock"]) == 4
+
+      # set a tiny cache limit to force eviction (1.5 chunks worth)
+      monkeypatch.setattr(url_file_module, 'CACHE_SIZE', url_file_module.CHUNK_SIZE + url_file_module.CHUNK_SIZE // 2)
+
+      # prune_cache should evict oldest files to get under limit
+      prune_cache()
+      remaining = [name for name in os.listdir(tmpdir) if name != "manifest.lock"]
+      # should have evicted at least one file + manifest
+      assert len(remaining) < 4
+      # newest file should remain
+      assert manifest_lines[2].split()[0] in remaining

iqpilot/tools/lib/tests/test_logreader.py

@@ -0,0 +1,176 @@
+import capnp
+import contextlib
+import shutil
+import tempfile
+import os
+import pytest
+import requests
+
+from parameterized import parameterized
+
+from iqpilot.cereal import log as capnp_log
+from iqpilot.tools.lib.logreader import InternalUnavailableException, LogReader, parse_indirect
+from iqpilot.tools.lib.route import SegmentRange
+from iqpilot.tools.lib.url_file import URLFileException
+
+NUM_SEGS = 17  # number of segments in the test route
+ALL_SEGS = list(range(NUM_SEGS))
+TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12"
+QLOG_FILE = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+
+
+@contextlib.contextmanager
+def setup_source_scenario(mocker, is_internal=False):
+  internal_source_mock = mocker.patch("iqpilot.tools.lib.logreader.internal_source")
+  internal_source_mock.__name__ = internal_source_mock._mock_name
+
+  openpilotci_source_mock = mocker.patch("iqpilot.tools.lib.logreader.openpilotci_source")
+  openpilotci_source_mock.__name__ = openpilotci_source_mock._mock_name
+
+  comma_api_source_mock = mocker.patch("iqpilot.tools.lib.logreader.comma_api_source")
+  comma_api_source_mock.__name__ = comma_api_source_mock._mock_name
+
+  if is_internal:
+    internal_source_mock.return_value = {3: QLOG_FILE}
+  else:
+    internal_source_mock.side_effect = InternalUnavailableException
+
+  openpilotci_source_mock.return_value = {}
+  comma_api_source_mock.return_value = {3: QLOG_FILE}
+
+  yield
+
+
+class TestLogReader:
+  @pytest.mark.parametrize(("identifier", "expected"), [
+    (f"{TEST_ROUTE}", ALL_SEGS),
+    (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"{TEST_ROUTE}--0", [0]),
+    (f"{TEST_ROUTE}--5", [5]),
+    (f"{TEST_ROUTE}/0", [0]),
+    (f"{TEST_ROUTE}/5", [5]),
+    (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]),
+    (f"{TEST_ROUTE}/0:0", []),
+    (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]),
+    (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]),
+    (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]),
+    (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]),
+    (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]),
+    (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]),
+    (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]),
+    (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]),
+    (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]),
+    (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]),
+    (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '%7C')}", ALL_SEGS),
+  ])
+  def test_indirect_parsing(self, identifier, expected, mocker):
+    mocker.patch("iqpilot.tools.lib.route.get_max_seg_number_cached", return_value=NUM_SEGS - 1)
+    parsed = parse_indirect(identifier)
+    sr = SegmentRange(parsed)
+    assert list(sr.seg_idxs) == expected, identifier
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE.replace('/', '|')}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE}--5", f"{TEST_ROUTE}/5"),
+    (f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"),
+    (f"{TEST_ROUTE}/5:6/r", f"{TEST_ROUTE}/5:6/r"),
+    (f"{TEST_ROUTE}/5", f"{TEST_ROUTE}/5"),
+  ])
+  def test_canonical_name(self, identifier, expected):
+    sr = SegmentRange(identifier)
+    assert str(sr) == expected
+
+  @pytest.mark.parametrize("cache_enabled", [True, False])
+  def test_direct_parsing(self, mocker, cache_enabled):
+    file_exists_mock = mocker.patch("iqpilot.tools.lib.filereader.file_exists")
+    if cache_enabled:
+      os.environ.pop("DISABLE_FILEREADER_CACHE", None)
+    else:
+      os.environ["DISABLE_FILEREADER_CACHE"] = "1"
+    qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+
+    with requests.get(QLOG_FILE, stream=True) as r:
+      with qlog as f:
+        shutil.copyfileobj(r.raw, f)
+
+    for f in [QLOG_FILE, qlog.name]:
+      l = len(list(LogReader(f)))
+      assert l > 100
+
+    with pytest.raises(URLFileException) if not cache_enabled else pytest.raises(AssertionError):
+      l = len(list(LogReader(QLOG_FILE.replace("/3/", "/200/"))))
+
+    # file_exists should not be called for direct files
+    assert file_exists_mock.call_count == 0
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}///",),
+    (f"{TEST_ROUTE}---",),
+    (f"{TEST_ROUTE}/-4:--2",),
+    (f"{TEST_ROUTE}/-a",),
+    (f"{TEST_ROUTE}/j",),
+    (f"{TEST_ROUTE}/0:1:2:3",),
+    (f"{TEST_ROUTE}/:::3",),
+    (f"{TEST_ROUTE}3",),
+    (f"{TEST_ROUTE}-3",),
+    (f"{TEST_ROUTE}--3a",),
+  ])
+  def test_bad_ranges(self, segment_range):
+    with pytest.raises(AssertionError):
+      _ = SegmentRange(segment_range).seg_idxs
+
+  @pytest.mark.parametrize("segment_range, api_call", [
+    (f"{TEST_ROUTE}/0", False),
+    (f"{TEST_ROUTE}/:2", False),
+    (f"{TEST_ROUTE}/0:", True),
+    (f"{TEST_ROUTE}/-1", True),
+    (f"{TEST_ROUTE}", True),
+  ])
+  def test_slicing_api_call(self, mocker, segment_range, api_call):
+    max_seg_mock = mocker.patch("iqpilot.tools.lib.route.get_max_seg_number_cached")
+    max_seg_mock.return_value = NUM_SEGS
+    _ = SegmentRange(segment_range).seg_idxs
+    assert api_call == max_seg_mock.called
+
+  @pytest.mark.parametrize("is_internal", [True, False])
+  def test_auto_source_scenarios(self, mocker, is_internal):
+    lr = LogReader(QLOG_FILE)
+    qlog_len = len(list(lr))
+
+    with setup_source_scenario(mocker, is_internal=is_internal):
+      lr = LogReader(f"{TEST_ROUTE}/3/q")
+      log_len = len(list(lr))
+      assert qlog_len == log_len
+
+  def test_only_union_types(self):
+    with tempfile.NamedTemporaryFile() as qlog:
+      # write valid Event messages
+      num_msgs = 100
+      with open(qlog.name, "wb") as f:
+        f.write(b"".join(capnp_log.Event.new_message().to_bytes() for _ in range(num_msgs)))
+
+      msgs = list(LogReader(qlog.name))
+      assert len(msgs) == num_msgs
+      [m.which() for m in msgs]
+
+      # append non-union Event message
+      event_msg = capnp_log.Event.new_message()
+      non_union_bytes = bytearray(event_msg.to_bytes())
+      non_union_bytes[event_msg.total_size.word_count * 8] = 0xff  # set discriminant value out of range using Event word offset
+      with open(qlog.name, "ab") as f:
+        f.write(non_union_bytes)
+
+      # ensure new message is added, but is not a union type
+      msgs = list(LogReader(qlog.name))
+      assert len(msgs) == num_msgs + 1
+      with pytest.raises((capnp.KjException, RuntimeError)):
+        [m.which() for m in msgs]
+
+      # should not be added when only_union_types=True
+      msgs = list(LogReader(qlog.name, only_union_types=True))
+      assert len(msgs) == num_msgs
+      [m.which() for m in msgs]

iqpilot/tools/lib/tests/test_route_library.py

@@ -0,0 +1,27 @@
+from collections import namedtuple
+
+from iqpilot.tools.lib.route import SegmentName
+
+class TestRouteLibrary:
+  def test_segment_name_formats(self):
+    Case = namedtuple('Case', ['input', 'expected_route', 'expected_segment_num', 'expected_data_dir'])
+
+    cases = [ Case("a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, None),
+              Case("a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19/3", "a2a0ccea32023010|2023-07-27--13-01-19", 3, None),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, "/data/media/0/realdata") ]
+
+    def _validate(case):
+      route_or_segment_name = case.input
+
+      s = SegmentName(route_or_segment_name, allow_route_name=True)
+
+      assert str(s.route_name) == case.expected_route
+      assert s.segment_num == case.expected_segment_num
+      assert s.data_dir == case.expected_data_dir
+
+    for case in cases:
+      _validate(case)